diff --git a/extra/peg/javascript/tokenizer/tokenizer.factor b/extra/peg/javascript/tokenizer/tokenizer.factor index 56afa42cad..a4d82a1580 100644 --- a/extra/peg/javascript/tokenizer/tokenizer.factor +++ b/extra/peg/javascript/tokenizer/tokenizer.factor @@ -48,7 +48,7 @@ Name = !(Keyword) iName => [[ ast-name boa ]] Number = Digits:ws '.' Digits:fs => [[ ws "." fs 3array "" concat-as string>number ast-number boa ]] | Digits => [[ >string string>number ast-number boa ]] -SingleEscapeChar = "b" => [[ CHAR: \b ]] +SingleEscape = "b" => [[ CHAR: \b ]] | "f" => [[ CHAR: \f ]] | "n" => [[ CHAR: \n ]] | "r" => [[ CHAR: \r ]] @@ -58,10 +58,10 @@ SingleEscapeChar = "b" => [[ CHAR: \b ]] | "\"" => [[ CHAR: " ]] | "\\" => [[ CHAR: \\ ]] HexDigit = [0-9a-fA-F] -HexEscapeChar = "x" (HexDigit HexDigit):d => [[ d hex> ]] -UnicodeEscapeChar = "u" (HexDigit HexDigit HexDigit HexDigit):d => [[ d hex> ]] +HexEscape = "x" (HexDigit HexDigit):d => [[ d hex> ]] +UnicodeEscape = "u" (HexDigit HexDigit HexDigit HexDigit):d => [[ d hex> ]] | "u{" HexDigit+:d "}" => [[ d hex> ]] -EscapeChar = "\\" (SingleEscapeChar | HexEscapeChar | UnicodeEscapeChar):c => [[ c ]] +EscapeChar = "\\" (SingleEscape | HexEscape | UnicodeEscape):c => [[ c ]] StringChars1 = (EscapeChar | !('"""') .)* => [[ >string ]] StringChars2 = (EscapeChar | !('"') .)* => [[ >string ]] StringChars3 = (EscapeChar | !("'") .)* => [[ >string ]]