steve
/
factor
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix peg.ebnf tests. Handle \ in EBNF

db4
Chris Double 2008-06-30 14:32:20 +12:00
parent cf00bc8a0c
commit acb6d3a312
3 changed files with 41 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
extra/peg/ebnf/ebnf-tests.factor
View File

@ -132,21 +132,21 @@ IN: peg.ebnf.tests
"Z" [EBNF foo=[A-Z] EBNF] call ast>> "Z" [EBNF foo=[A-Z] EBNF] call ast>>
] unit-test ] unit-test
{ f } [ [
"0" [EBNF foo=[A-Z] EBNF] call "0" [EBNF foo=[A-Z] EBNF] call
] unit-test ] must-fail
{ CHAR: 0 } [ { CHAR: 0 } [
"0" [EBNF foo=[^A-Z] EBNF] call ast>> "0" [EBNF foo=[^A-Z] EBNF] call ast>>
] unit-test ] unit-test
{ f } [ [
"A" [EBNF foo=[^A-Z] EBNF] call "A" [EBNF foo=[^A-Z] EBNF] call
] unit-test ] must-fail
{ f } [ [
"Z" [EBNF foo=[^A-Z] EBNF] call "Z" [EBNF foo=[^A-Z] EBNF] call
] unit-test ] must-fail
{ V{ "1" "+" "foo" } } [ { V{ "1" "+" "foo" } } [
"1+1" [EBNF foo='1' '+' '1' [[ drop "foo" ]] EBNF] call ast>> "1+1" [EBNF foo='1' '+' '1' [[ drop "foo" ]] EBNF] call ast>>
@ -176,17 +176,17 @@ IN: peg.ebnf.tests
{ 1 2 3 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF] call ast>> { 1 2 3 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF] call ast>>
] unit-test ] unit-test
{ f } [ [
{ "a" 2 3 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF] call { "a" 2 3 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF] call
] unit-test ] must-fail
{ 3 } [ { 3 } [
{ 1 2 "a" 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF] call ast>> { 1 2 "a" 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF] call ast>>
] unit-test ] unit-test
{ f } [ [
"ab" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF] call "ab" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF] call
] unit-test ] must-fail
{ V{ "a" " " "b" } } [ { V{ "a" " " "b" } } [
"a b" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF] call ast>> "a b" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF] call ast>>
@ -229,9 +229,9 @@ IN: peg.ebnf.tests
"a\nb" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF] call ast>> "a\nb" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF] call ast>>
] unit-test ] unit-test
{ f } [ [
"axb" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF] call "axb" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF] call
] unit-test ] must-fail
{ V{ V{ 49 } "+" V{ 49 } } } [ { V{ V{ 49 } "+" V{ 49 } } } [
#! Test direct left recursion. #! Test direct left recursion.
@ -314,41 +314,41 @@ main = Primary
"abc" [EBNF a="a" "b" foo=a "c" EBNF] call ast>> "abc" [EBNF a="a" "b" foo=a "c" EBNF] call ast>>
] unit-test ] unit-test
{ f } [ [
"a bc" [EBNF a="a" "b" foo=(a "c") EBNF] call "a bc" [EBNF a="a" "b" foo=(a "c") EBNF] call
] unit-test ] must-fail
{ f } [ [
"a bc" [EBNF a="a" "b" foo=a "c" EBNF] call "a bc" [EBNF a="a" "b" foo=a "c" EBNF] call
] unit-test ] must-fail
{ f } [ [
"a bc" [EBNF a="a" "b" foo={a "c"} EBNF] call "a bc" [EBNF a="a" "b" foo={a "c"} EBNF] call
] unit-test ] must-fail
{ f } [ [
"ab c" [EBNF a="a" "b" foo=a "c" EBNF] call "ab c" [EBNF a="a" "b" foo=a "c" EBNF] call
] unit-test ] must-fail
{ V{ V{ "a" "b" } "c" } } [ { V{ V{ "a" "b" } "c" } } [
"ab c" [EBNF a="a" "b" foo={a "c"} EBNF] call ast>> "ab c" [EBNF a="a" "b" foo={a "c"} EBNF] call ast>>
] unit-test ] unit-test
{ f } [ [
"ab c" [EBNF a="a" "b" foo=(a "c") EBNF] call "ab c" [EBNF a="a" "b" foo=(a "c") EBNF] call
] unit-test ] must-fail
{ f } [ [
"a b c" [EBNF a="a" "b" foo=a "c" EBNF] call "a b c" [EBNF a="a" "b" foo=a "c" EBNF] call
] unit-test ] must-fail
{ f } [ [
"a b c" [EBNF a="a" "b" foo=(a "c") EBNF] call "a b c" [EBNF a="a" "b" foo=(a "c") EBNF] call
] unit-test ] must-fail
{ f } [ [
"a b c" [EBNF a="a" "b" foo={a "c"} EBNF] call "a b c" [EBNF a="a" "b" foo={a "c"} EBNF] call
] unit-test ] must-fail
{ V{ V{ V{ "a" "b" } "c" } V{ V{ "a" "b" } "c" } } } [ { V{ V{ V{ "a" "b" } "c" } V{ V{ "a" "b" } "c" } } } [
"ab cab c" [EBNF a="a" "b" foo={a "c"}* EBNF] call ast>> "ab cab c" [EBNF a="a" "b" foo={a "c"}* EBNF] call ast>>
@ -516,3 +516,7 @@ Tok = Spaces (Number | Special )
{ "++" } [ { "++" } [
"++--" [EBNF tokenizer=("++" | "--") main="++" EBNF] call ast>> "++--" [EBNF tokenizer=("++" | "--") main="++" EBNF] call ast>>
] unit-test ] unit-test
{ "\\" } [
"\\" [EBNF foo="\\" EBNF] call ast>>
] unit-test

1
extra/peg/ebnf/ebnf.factor
View File

@ -99,6 +99,7 @@ PEG: escaper ( string -- ast )
"\\t" token [ drop "\t" ] action , "\\t" token [ drop "\t" ] action ,
"\\n" token [ drop "\n" ] action , "\\n" token [ drop "\n" ] action ,
"\\r" token [ drop "\r" ] action , "\\r" token [ drop "\r" ] action ,
"\\\\" token [ drop "\\" ] action ,
] choice* any-char-parser 2array choice repeat0 ; ] choice* any-char-parser 2array choice repeat0 ;
: replace-escapes ( string -- string ) : replace-escapes ( string -- string )

9
extra/peg/javascript/tokenizer/tokenizer.factor
View File

@ -58,7 +58,14 @@ Str = '"""' StringChars1:cs '"""' => [[ cs ast-string boa ]]
| '"' StringChars2:cs '"' => [[ cs ast-string boa ]] | '"' StringChars2:cs '"' => [[ cs ast-string boa ]]
| "'" StringChars3:cs "'" => [[ cs ast-string boa ]] | "'" StringChars3:cs "'" => [[ cs ast-string boa ]]
RegExpFlags = NameRest* RegExpFlags = NameRest*
RegExpBody = (!("/" | "\n" | "\r") .)* => [[ >string ]] NonTerminator = !("\n" | "\r") .
BackslashSequence = "\\" NonTerminator
RegExpFirstChar = !("*" | "\\" | "/") NonTerminator
| BackslashSequence
RegExpChar = !("\\" | "/") NonTerminator
| BackslashSequence
RegExpChars = RegExpChar*
RegExpBody = RegExpFirstChar RegExpChars
RegExp = "/" RegExpBody:b "/" RegExpFlags:fl => [[ b fl ast-regexp boa ]] RegExp = "/" RegExpBody:b "/" RegExpFlags:fl => [[ b fl ast-regexp boa ]]
Special = "(" | ")" | "{" | "}" | "[" | "]" | "," | ";" Special = "(" | ")" | "{" | "}" | "[" | "]" | "," | ";"
| "?" | ":" | "!==" | "!=" | "===" | "==" | "=" | ">=" | "?" | ":" | "!==" | "!=" | "===" | "==" | "=" | ">="