Remove EBNF: and [EBNF EBNF] +tokenizer syntax

replaced with tokenizer support in ebnf grammar itself.

extra/peg/ebnf/ebnf.factor

@@ -520,20 +520,10 @@ M: ebnf-non-terminal (transform) ( ast -- parser )
   parse-result-ast transform dup dup parser [ main swap at compile ] with-variable
   [ compiled-parse ] curry [ with-scope ] curry ;
 
-: [EBNF 
-  scan {
-    { "+" [ scan-word execute "" swap ] }
-    [ " " append default-tokenizer ]
-  } case \ tokenizer set-global
-  [ "EBNF]" parse-multiline-string ] [ drop "" ] recover append ebnf>quot nip parsed 
-  reset-tokenizer ; parsing
+: [EBNF "EBNF]" reset-tokenizer parse-multiline-string ebnf>quot nip parsed reset-tokenizer ; parsing
 
 : EBNF: 
-  CREATE-WORD scan {
-    { "+" [ scan-word execute "" swap ] }
-    [ " " append default-tokenizer ]
-  } case \ tokenizer set-global
-  dupd [ ";EBNF" parse-multiline-string ] [ drop "" ] recover append 
+  reset-tokenizer CREATE-WORD dup ";EBNF" parse-multiline-string  
   ebnf>quot swapd 1 1 <effect> define-declared "ebnf-parser" set-word-prop 
   reset-tokenizer ; parsing
 

extra/peg/javascript/javascript.factor

@@ -4,8 +4,8 @@ USING: kernel accessors peg.javascript.tokenizer peg.javascript.parser ;
 IN: peg.javascript
 
 : parse-javascript ( string -- ast )
-  tokenizer [
-    ast>> javascript [
+  tokenize-javascript [
+    ast>> parse-javascript [
       ast>>
     ] [
       "Unable to parse JavaScript" throw

extra/peg/javascript/parser/parser-tests.factor

@@ -5,7 +5,7 @@ USING: kernel tools.test peg peg.javascript.ast peg.javascript.tokenizer
        peg.javascript.parser accessors multiline sequences math ;
 IN: peg.javascript.parser.tests
 
-\ javascript must-infer
+\ parse-javascript must-infer
 
 {
   T{
@@ -23,14 +23,14 @@ IN: peg.javascript.parser.tests
       }
   }
 } [
-  "123; 'hello'; foo(x);" tokenizer ast>> javascript ast>>
+  "123; 'hello'; foo(x);" tokenize-javascript ast>> parse-javascript ast>>
 ] unit-test
 
 { t } [ 
 <"
 var x=5
 var y=10
-"> tokenizer ast>> javascript remaining>> length zero?
+"> tokenize-javascript ast>> parse-javascript remaining>> length zero?
 ] unit-test
 
 
@@ -41,7 +41,7 @@ function foldl(f, initial, seq) {
      initial = f(initial, seq[i]);
    return initial;
 }
-"> tokenizer ast>> javascript remaining>> length zero?
+"> tokenize-javascript ast>> parse-javascript remaining>> length zero?
 ] unit-test
 
 { t } [ 
@@ -52,6 +52,6 @@ ParseState.prototype.from = function(index) {
     r.length = this.length - index;
     return r;
 }
-"> tokenizer ast>> javascript remaining>> length zero?
+"> tokenize-javascript ast>> parse-javascript remaining>> length zero?
 ] unit-test
 

extra/peg/javascript/parser/parser.factor

@@ -6,7 +6,7 @@ IN: peg.javascript.parser
 #! Grammar for JavaScript. Based on OMeta-JS example from:
 #! http://jarrett.cs.ucla.edu/ometa-js/#JavaScript_Compiler 
 
-EBNF: javascript
+EBNF: parse-javascript
 End               = !(.)
 Space             = " " | "\t" | "\n" 
 Spaces            = Space* => [[ ignore ]]

extra/peg/javascript/tokenizer/tokenizer-tests.factor

@@ -4,7 +4,7 @@
 USING: kernel tools.test peg peg.javascript.ast peg.javascript.tokenizer accessors ;
 IN: peg.javascript.tokenizer.tests
 
-\ tokenizer must-infer
+\ tokenize-javascript must-infer
 
 {
   V{
@@ -19,5 +19,5 @@ IN: peg.javascript.tokenizer.tests
     ";"
   }    
 } [
-  "123; 'hello'; foo(x);" tokenizer ast>>
+  "123; 'hello'; foo(x);" tokenize-javascript ast>>
 ] unit-test

extra/peg/javascript/tokenizer/tokenizer.factor

@@ -8,7 +8,7 @@ IN: peg.javascript.tokenizer
 
 USE: prettyprint
 
-EBNF: javascript-tokenizer 
+EBNF: tokenize-javascript 
 Letter            = [a-zA-Z]
 Digit             = [0-9]
 Digits            = Digit+