steve
/
factor
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

RegExp fix for javascript tokenizer

db4
Chris Double 2008-06-30 15:20:12 +12:00
parent acb6d3a312
commit 4394cb08f6
2 changed files with 7 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
extra/peg/javascript/tokenizer/tokenizer-tests.factor
View File

@ -21,3 +21,7 @@ IN: peg.javascript.tokenizer.tests
} [ } [
"123; 'hello'; foo(x);" tokenize-javascript ast>> "123; 'hello'; foo(x);" tokenize-javascript ast>>
] unit-test ] unit-test
{ V{ T{ ast-regexp f "<(w+)[^>]*?)/>" "g" } } } [
"/<(\\w+)[^>]*?)\\/>/g" tokenize-javascript ast>>
] unit-test

6
extra/peg/javascript/tokenizer/tokenizer.factor
View File

@ -57,15 +57,15 @@ StringChars3 = (EscapeChar | !("'") .)* => [[ >string ]]
Str = '"""' StringChars1:cs '"""' => [[ cs ast-string boa ]] Str = '"""' StringChars1:cs '"""' => [[ cs ast-string boa ]]
| '"' StringChars2:cs '"' => [[ cs ast-string boa ]] | '"' StringChars2:cs '"' => [[ cs ast-string boa ]]
| "'" StringChars3:cs "'" => [[ cs ast-string boa ]] | "'" StringChars3:cs "'" => [[ cs ast-string boa ]]
RegExpFlags = NameRest* RegExpFlags = NameRest* => [[ >string ]]
NonTerminator = !("\n" | "\r") . NonTerminator = !("\n" | "\r") .
BackslashSequence = "\\" NonTerminator BackslashSequence = "\\" NonTerminator => [[ second ]]
RegExpFirstChar = !("*" | "\\" | "/") NonTerminator RegExpFirstChar = !("*" | "\\" | "/") NonTerminator
| BackslashSequence | BackslashSequence
RegExpChar = !("\\" | "/") NonTerminator RegExpChar = !("\\" | "/") NonTerminator
| BackslashSequence | BackslashSequence
RegExpChars = RegExpChar* RegExpChars = RegExpChar*
RegExpBody = RegExpFirstChar RegExpChars RegExpBody = RegExpFirstChar RegExpChars => [[ first2 swap prefix >string ]]
RegExp = "/" RegExpBody:b "/" RegExpFlags:fl => [[ b fl ast-regexp boa ]] RegExp = "/" RegExpBody:b "/" RegExpFlags:fl => [[ b fl ast-regexp boa ]]
Special = "(" | ")" | "{" | "}" | "[" | "]" | "," | ";" Special = "(" | ")" | "{" | "}" | "[" | "]" | "," | ";"
| "?" | ":" | "!==" | "!=" | "===" | "==" | "=" | ">=" | "?" | ":" | "!==" | "!=" | "===" | "==" | "=" | ">="