From acb6d3a312dff4450f37a4ffafc1132010a92578 Mon Sep 17 00:00:00 2001
From: Chris Double <chris@bethia.(none)>
Date: Mon, 30 Jun 2008 14:32:20 +1200
Subject: [PATCH] Fix peg.ebnf tests. Handle \ in EBNF

---
 extra/peg/ebnf/ebnf-tests.factor              | 60 ++++++++++---------
 extra/peg/ebnf/ebnf.factor                    |  1 +
 .../peg/javascript/tokenizer/tokenizer.factor |  9 ++-
 3 files changed, 41 insertions(+), 29 deletions(-)

diff --git a/extra/peg/ebnf/ebnf-tests.factor b/extra/peg/ebnf/ebnf-tests.factor
index 2269af6625..a2807d20db 100644
--- a/extra/peg/ebnf/ebnf-tests.factor
+++ b/extra/peg/ebnf/ebnf-tests.factor
@@ -132,21 +132,21 @@ IN: peg.ebnf.tests
   "Z" [EBNF foo=[A-Z] EBNF] call ast>> 
 ] unit-test
 
-{ f } [
+[
   "0" [EBNF foo=[A-Z] EBNF] call  
-] unit-test
+] must-fail
 
 { CHAR: 0 } [
   "0" [EBNF foo=[^A-Z] EBNF] call ast>> 
 ] unit-test
 
-{ f } [
+[
   "A" [EBNF foo=[^A-Z] EBNF] call  
-] unit-test
+] must-fail
 
-{ f } [
+[
   "Z" [EBNF foo=[^A-Z] EBNF] call  
-] unit-test
+] must-fail
 
 { V{ "1" "+" "foo" } } [
   "1+1" [EBNF foo='1' '+' '1' [[ drop "foo" ]] EBNF] call ast>>
@@ -176,17 +176,17 @@ IN: peg.ebnf.tests
   { 1 2 3 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF] call ast>>
 ] unit-test
 
-{ f } [
+[
   { "a" 2 3 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF] call 
-] unit-test
+] must-fail
 
 { 3 } [
   { 1 2 "a" 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF] call ast>>
 ] unit-test
 
-{ f } [
+[
   "ab" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF] call 
-] unit-test
+] must-fail
 
 { V{ "a" " " "b" } } [
   "a b" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF] call ast>>
@@ -229,9 +229,9 @@ IN: peg.ebnf.tests
   "a\nb" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF] call ast>>
 ] unit-test
 
-{ f } [
+[
   "axb" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF] call 
-] unit-test
+] must-fail
 
 { V{ V{ 49 } "+" V{ 49 } } } [ 
   #! Test direct left recursion. 
@@ -314,41 +314,41 @@ main = Primary
   "abc" [EBNF a="a" "b" foo=a "c" EBNF] call ast>>
 ] unit-test
 
-{ f } [
+[
   "a bc" [EBNF a="a" "b" foo=(a "c") EBNF] call 
-] unit-test
+] must-fail
 
-{ f } [
+[
   "a bc" [EBNF a="a" "b" foo=a "c" EBNF] call 
-] unit-test
+] must-fail
 
-{ f } [
+[
   "a bc" [EBNF a="a" "b" foo={a "c"} EBNF] call
-] unit-test
+] must-fail
 
-{ f } [
+[
   "ab c" [EBNF a="a" "b" foo=a "c" EBNF] call 
-] unit-test
+] must-fail
 
 { V{ V{ "a" "b" } "c" } } [
   "ab c" [EBNF a="a" "b" foo={a "c"} EBNF] call ast>>
 ] unit-test
 
-{ f } [
+[
   "ab c" [EBNF a="a" "b" foo=(a "c") EBNF] call 
-] unit-test
+] must-fail
 
-{ f } [
+[
   "a b c" [EBNF a="a" "b" foo=a "c" EBNF] call 
-] unit-test
+] must-fail
 
-{ f } [
+[
   "a b c" [EBNF a="a" "b" foo=(a "c") EBNF] call 
-] unit-test
+] must-fail
 
-{ f } [
+[
   "a b c" [EBNF a="a" "b" foo={a "c"} EBNF] call 
-] unit-test
+] must-fail
 
 { V{ V{ V{ "a" "b" } "c" } V{ V{ "a" "b" } "c" } } } [
   "ab cab c" [EBNF a="a" "b" foo={a "c"}* EBNF] call ast>>
@@ -515,4 +515,8 @@ Tok                = Spaces (Number | Special )
 
 { "++" } [
   "++--" [EBNF tokenizer=("++" | "--") main="++" EBNF] call ast>>
+] unit-test
+
+{ "\\" } [
+  "\\" [EBNF foo="\\" EBNF] call ast>>
 ] unit-test
\ No newline at end of file
diff --git a/extra/peg/ebnf/ebnf.factor b/extra/peg/ebnf/ebnf.factor
index 3d48665c8c..610cffd273 100644
--- a/extra/peg/ebnf/ebnf.factor
+++ b/extra/peg/ebnf/ebnf.factor
@@ -99,6 +99,7 @@ PEG: escaper ( string -- ast )
     "\\t" token [ drop "\t" ] action ,
     "\\n" token [ drop "\n" ] action ,
     "\\r" token [ drop "\r" ] action ,
+    "\\\\" token [ drop "\\" ] action ,
   ] choice* any-char-parser 2array choice repeat0 ;
 
 : replace-escapes ( string -- string )
diff --git a/extra/peg/javascript/tokenizer/tokenizer.factor b/extra/peg/javascript/tokenizer/tokenizer.factor
index 825c8f03d1..256e478571 100644
--- a/extra/peg/javascript/tokenizer/tokenizer.factor
+++ b/extra/peg/javascript/tokenizer/tokenizer.factor
@@ -58,7 +58,14 @@ Str                =   '"""' StringChars1:cs '"""' => [[ cs ast-string boa ]]
                      | '"' StringChars2:cs '"' => [[ cs ast-string boa ]]
                      | "'" StringChars3:cs "'" => [[ cs ast-string boa ]]
 RegExpFlags        = NameRest*
-RegExpBody         = (!("/" | "\n" | "\r") .)* => [[ >string ]]
+NonTerminator      = !("\n" | "\r") .
+BackslashSequence  = "\\" NonTerminator
+RegExpFirstChar    =   !("*" | "\\" | "/") NonTerminator
+                     | BackslashSequence
+RegExpChar         =   !("\\" | "/") NonTerminator
+                     | BackslashSequence
+RegExpChars        = RegExpChar*
+RegExpBody         = RegExpFirstChar RegExpChars
 RegExp             = "/" RegExpBody:b "/" RegExpFlags:fl => [[ b fl ast-regexp boa ]]
 Special            =   "("   | ")"   | "{"   | "}"   | "["   | "]"   | ","   | ";"
                      | "?"   | ":"   | "!==" | "!="  | "===" | "=="  | "="   | ">="