Fix terminal parser with non-default tokenizers

extra/peg/ebnf/ebnf-tests.factor

@@ -511,4 +511,8 @@ Tok                = Spaces (Number | Special )
           tokenizer=spaces (number | operator) 
           main= . . . 
         EBNF] call ast>> 
+] unit-test
+
+{ "++" } [
+  "++--" [EBNF tokenizer=("++" | "--") main="++" EBNF] call ast>>
 ] unit-test

extra/peg/ebnf/ebnf.factor

@@ -15,13 +15,14 @@ TUPLE: tokenizer any one many ;
 
 : default-tokenizer ( -- tokenizer )
   T{ tokenizer f 
-    [ [ drop t ] satisfy ]
+    [ any-char ]
     [ token ]
-    [ [ = ] curry satisfy ]
+    [ [ = ] curry any-char swap semantic ]
   } ;
 
 : parser-tokenizer ( parser -- tokenizer )
-  1quotation [ [ = ] curry satisfy ] dup tokenizer boa ;
+  [ 1quotation ] keep
+  [ swap [ = ] curry semantic ] curry dup tokenizer boa ;
 
 : rule-tokenizer ( name word -- tokenizer )
   rule parser-tokenizer ;

extra/peg/javascript/javascript.factor

@@ -5,7 +5,7 @@ IN: peg.javascript
 
 : parse-javascript ( string -- ast )
   tokenize-javascript [
-    ast>> parse-javascript [
+    ast>> javascript [
       ast>>
     ] [
       "Unable to parse JavaScript" throw

extra/peg/javascript/parser/parser-tests.factor

@@ -1,11 +1,11 @@
 ! Copyright (C) 2008 Chris Double.
 ! See http://factorcode.org/license.txt for BSD license.
 !
-USING: kernel tools.test peg peg.javascript.ast peg.javascript.tokenizer  
-       peg.javascript.parser accessors multiline sequences math ;
+USING: kernel tools.test peg peg.javascript.ast peg.javascript.parser 
+       accessors multiline sequences math ;
 IN: peg.javascript.parser.tests
 
-\ parse-javascript must-infer
+\ javascript must-infer
 
 {
   T{
@@ -23,14 +23,14 @@ IN: peg.javascript.parser.tests
       }
   }
 } [
-  "123; 'hello'; foo(x);" tokenize-javascript ast>> parse-javascript ast>>
+  "123; 'hello'; foo(x);" javascript ast>>
 ] unit-test
 
 { t } [ 
 <"
 var x=5
 var y=10
-"> tokenize-javascript ast>> parse-javascript remaining>> length zero?
+"> javascript remaining>> length zero?
 ] unit-test
 
 
@@ -41,7 +41,7 @@ function foldl(f, initial, seq) {
      initial = f(initial, seq[i]);
    return initial;
 }
-"> tokenize-javascript ast>> parse-javascript remaining>> length zero?
+"> javascript remaining>> length zero?
 ] unit-test
 
 { t } [ 
@@ -52,6 +52,6 @@ ParseState.prototype.from = function(index) {
     r.length = this.length - index;
     return r;
 }
-"> tokenize-javascript ast>> parse-javascript remaining>> length zero?
+"> javascript remaining>> length zero?
 ] unit-test
 

extra/peg/javascript/parser/parser.factor

@@ -1,12 +1,13 @@
 ! Copyright (C) 2008 Chris Double.
 ! See http://factorcode.org/license.txt for BSD license.
-USING: kernel accessors peg peg.ebnf peg.javascript.ast ;
+USING: kernel accessors peg peg.ebnf peg.javascript.ast peg.javascript.tokenizer ;
 IN: peg.javascript.parser
 
 #! Grammar for JavaScript. Based on OMeta-JS example from:
 #! http://jarrett.cs.ucla.edu/ometa-js/#JavaScript_Compiler 
 
-EBNF: parse-javascript
+EBNF: javascript
+tokenizer         = <foreign tokenize-javascript Tok>
 End               = !(.)
 Space             = " " | "\t" | "\n" 
 Spaces            = Space* => [[ ignore ]]