diff --git a/extra/peg/javascript/tokenizer/tokenizer-tests.factor b/extra/peg/javascript/tokenizer/tokenizer-tests.factor index 509ff4a0fe..a61125d08c 100644 --- a/extra/peg/javascript/tokenizer/tokenizer-tests.factor +++ b/extra/peg/javascript/tokenizer/tokenizer-tests.factor @@ -21,3 +21,7 @@ IN: peg.javascript.tokenizer.tests } [ "123; 'hello'; foo(x);" tokenize-javascript ast>> ] unit-test + +{ V{ T{ ast-regexp f "<(w+)[^>]*?)/>" "g" } } } [ + "/<(\\w+)[^>]*?)\\/>/g" tokenize-javascript ast>> +] unit-test \ No newline at end of file diff --git a/extra/peg/javascript/tokenizer/tokenizer.factor b/extra/peg/javascript/tokenizer/tokenizer.factor index 256e478571..f65b0b2ad6 100644 --- a/extra/peg/javascript/tokenizer/tokenizer.factor +++ b/extra/peg/javascript/tokenizer/tokenizer.factor @@ -57,15 +57,15 @@ StringChars3 = (EscapeChar | !("'") .)* => [[ >string ]] Str = '"""' StringChars1:cs '"""' => [[ cs ast-string boa ]] | '"' StringChars2:cs '"' => [[ cs ast-string boa ]] | "'" StringChars3:cs "'" => [[ cs ast-string boa ]] -RegExpFlags = NameRest* +RegExpFlags = NameRest* => [[ >string ]] NonTerminator = !("\n" | "\r") . -BackslashSequence = "\\" NonTerminator +BackslashSequence = "\\" NonTerminator => [[ second ]] RegExpFirstChar = !("*" | "\\" | "/") NonTerminator | BackslashSequence RegExpChar = !("\\" | "/") NonTerminator | BackslashSequence RegExpChars = RegExpChar* -RegExpBody = RegExpFirstChar RegExpChars +RegExpBody = RegExpFirstChar RegExpChars => [[ first2 swap prefix >string ]] RegExp = "/" RegExpBody:b "/" RegExpFlags:fl => [[ b fl ast-regexp boa ]] Special = "(" | ")" | "{" | "}" | "[" | "]" | "," | ";" | "?" | ":" | "!==" | "!=" | "===" | "==" | "=" | ">="