From 6b83ab9d9060f8972b43d7082253a3a957d1f96f Mon Sep 17 00:00:00 2001
From: Chris Double <chris@bethia.(none)>
Date: Sat, 21 Jun 2008 01:49:07 +1200
Subject: [PATCH] Fix terminal parser with non-default tokenizers

---
 extra/peg/ebnf/ebnf-tests.factor                |  4 ++++
 extra/peg/ebnf/ebnf.factor                      |  7 ++++---
 extra/peg/javascript/javascript.factor          |  2 +-
 extra/peg/javascript/parser/parser-tests.factor | 14 +++++++-------
 extra/peg/javascript/parser/parser.factor       |  5 +++--
 5 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/extra/peg/ebnf/ebnf-tests.factor b/extra/peg/ebnf/ebnf-tests.factor
index a90fa98436..2269af6625 100644
--- a/extra/peg/ebnf/ebnf-tests.factor
+++ b/extra/peg/ebnf/ebnf-tests.factor
@@ -511,4 +511,8 @@ Tok                = Spaces (Number | Special )
           tokenizer=spaces (number | operator) 
           main= . . . 
         EBNF] call ast>> 
+] unit-test
+
+{ "++" } [
+  "++--" [EBNF tokenizer=("++" | "--") main="++" EBNF] call ast>>
 ] unit-test
\ No newline at end of file
diff --git a/extra/peg/ebnf/ebnf.factor b/extra/peg/ebnf/ebnf.factor
index 564b376b29..4725534178 100644
--- a/extra/peg/ebnf/ebnf.factor
+++ b/extra/peg/ebnf/ebnf.factor
@@ -15,13 +15,14 @@ TUPLE: tokenizer any one many ;
 
 : default-tokenizer ( -- tokenizer )
   T{ tokenizer f 
-    [ [ drop t ] satisfy ]
+    [ any-char ]
     [ token ]
-    [ [ = ] curry satisfy ]
+    [ [ = ] curry any-char swap semantic ]
   } ;
 
 : parser-tokenizer ( parser -- tokenizer )
-  1quotation [ [ = ] curry satisfy ] dup tokenizer boa ;
+  [ 1quotation ] keep
+  [ swap [ = ] curry semantic ] curry dup tokenizer boa ;
 
 : rule-tokenizer ( name word -- tokenizer )
   rule parser-tokenizer ;
diff --git a/extra/peg/javascript/javascript.factor b/extra/peg/javascript/javascript.factor
index d27a06e9d2..791f63c56b 100644
--- a/extra/peg/javascript/javascript.factor
+++ b/extra/peg/javascript/javascript.factor
@@ -5,7 +5,7 @@ IN: peg.javascript
 
 : parse-javascript ( string -- ast )
   tokenize-javascript [
-    ast>> parse-javascript [
+    ast>> javascript [
       ast>>
     ] [
       "Unable to parse JavaScript" throw
diff --git a/extra/peg/javascript/parser/parser-tests.factor b/extra/peg/javascript/parser/parser-tests.factor
index b19bb314bb..fd0e27b6d4 100644
--- a/extra/peg/javascript/parser/parser-tests.factor
+++ b/extra/peg/javascript/parser/parser-tests.factor
@@ -1,11 +1,11 @@
 ! Copyright (C) 2008 Chris Double.
 ! See http://factorcode.org/license.txt for BSD license.
 !
-USING: kernel tools.test peg peg.javascript.ast peg.javascript.tokenizer  
-       peg.javascript.parser accessors multiline sequences math ;
+USING: kernel tools.test peg peg.javascript.ast peg.javascript.parser 
+       accessors multiline sequences math ;
 IN: peg.javascript.parser.tests
 
-\ parse-javascript must-infer
+\ javascript must-infer
 
 {
   T{
@@ -23,14 +23,14 @@ IN: peg.javascript.parser.tests
       }
   }
 } [
-  "123; 'hello'; foo(x);" tokenize-javascript ast>> parse-javascript ast>>
+  "123; 'hello'; foo(x);" javascript ast>>
 ] unit-test
 
 { t } [ 
 <"
 var x=5
 var y=10
-"> tokenize-javascript ast>> parse-javascript remaining>> length zero?
+"> javascript remaining>> length zero?
 ] unit-test
 
 
@@ -41,7 +41,7 @@ function foldl(f, initial, seq) {
      initial = f(initial, seq[i]);
    return initial;
 }
-"> tokenize-javascript ast>> parse-javascript remaining>> length zero?
+"> javascript remaining>> length zero?
 ] unit-test
 
 { t } [ 
@@ -52,6 +52,6 @@ ParseState.prototype.from = function(index) {
     r.length = this.length - index;
     return r;
 }
-"> tokenize-javascript ast>> parse-javascript remaining>> length zero?
+"> javascript remaining>> length zero?
 ] unit-test
 
diff --git a/extra/peg/javascript/parser/parser.factor b/extra/peg/javascript/parser/parser.factor
index 1a074090f3..0239ce882c 100644
--- a/extra/peg/javascript/parser/parser.factor
+++ b/extra/peg/javascript/parser/parser.factor
@@ -1,12 +1,13 @@
 ! Copyright (C) 2008 Chris Double.
 ! See http://factorcode.org/license.txt for BSD license.
-USING: kernel accessors peg peg.ebnf peg.javascript.ast ;
+USING: kernel accessors peg peg.ebnf peg.javascript.ast peg.javascript.tokenizer ;
 IN: peg.javascript.parser
 
 #! Grammar for JavaScript. Based on OMeta-JS example from:
 #! http://jarrett.cs.ucla.edu/ometa-js/#JavaScript_Compiler 
 
-EBNF: parse-javascript
+EBNF: javascript
+tokenizer         = <foreign tokenize-javascript Tok>
 End               = !(.)
 Space             = " " | "\t" | "\n" 
 Spaces            = Space* => [[ ignore ]]