From f3145c5961dab694f51ba8a1845362d5dcb6a1f9 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Fri, 4 Jul 2008 15:48:52 +1200 Subject: [PATCH] [EBNF and EBNF: now return ast --- extra/peg/ebnf/ebnf-tests.factor | 112 +++++++++--------- extra/peg/ebnf/ebnf.factor | 2 +- extra/peg/javascript/javascript.factor | 6 +- .../peg/javascript/parser/parser-tests.factor | 10 +- .../tokenizer/tokenizer-tests.factor | 4 +- 5 files changed, 65 insertions(+), 69 deletions(-) diff --git a/extra/peg/ebnf/ebnf-tests.factor b/extra/peg/ebnf/ebnf-tests.factor index ef90929b79..7f14293a15 100644 --- a/extra/peg/ebnf/ebnf-tests.factor +++ b/extra/peg/ebnf/ebnf-tests.factor @@ -113,23 +113,23 @@ IN: peg.ebnf.tests ] unit-test { V{ "a" "b" } } [ - "ab" [EBNF foo='a' 'b' EBNF] ast>> + "ab" [EBNF foo='a' 'b' EBNF] ] unit-test { V{ 1 "b" } } [ - "ab" [EBNF foo=('a')[[ drop 1 ]] 'b' EBNF] ast>> + "ab" [EBNF foo=('a')[[ drop 1 ]] 'b' EBNF] ] unit-test { V{ 1 2 } } [ - "ab" [EBNF foo=('a') [[ drop 1 ]] ('b') [[ drop 2 ]] EBNF] ast>> + "ab" [EBNF foo=('a') [[ drop 1 ]] ('b') [[ drop 2 ]] EBNF] ] unit-test { CHAR: A } [ - "A" [EBNF foo=[A-Z] EBNF] ast>> + "A" [EBNF foo=[A-Z] EBNF] ] unit-test { CHAR: Z } [ - "Z" [EBNF foo=[A-Z] EBNF] ast>> + "Z" [EBNF foo=[A-Z] EBNF] ] unit-test [ @@ -137,7 +137,7 @@ IN: peg.ebnf.tests ] must-fail { CHAR: 0 } [ - "0" [EBNF foo=[^A-Z] EBNF] ast>> + "0" [EBNF foo=[^A-Z] EBNF] ] unit-test [ @@ -149,31 +149,31 @@ IN: peg.ebnf.tests ] must-fail { V{ "1" "+" "foo" } } [ - "1+1" [EBNF foo='1' '+' '1' [[ drop "foo" ]] EBNF] ast>> + "1+1" [EBNF foo='1' '+' '1' [[ drop "foo" ]] EBNF] ] unit-test { "foo" } [ - "1+1" [EBNF foo='1' '+' '1' => [[ drop "foo" ]] EBNF] ast>> + "1+1" [EBNF foo='1' '+' '1' => [[ drop "foo" ]] EBNF] ] unit-test { "foo" } [ - "1+1" [EBNF foo='1' '+' '1' => [[ drop "foo" ]] | '1' '-' '1' => [[ drop "bar" ]] EBNF] ast>> + "1+1" [EBNF foo='1' '+' '1' => [[ drop "foo" ]] | '1' '-' '1' => [[ drop "bar" ]] EBNF] ] unit-test { "bar" } [ - "1-1" [EBNF foo='1' '+' '1' => [[ drop "foo" ]] | '1' '-' '1' => [[ drop "bar" ]] EBNF] ast>> + "1-1" [EBNF foo='1' '+' '1' => [[ drop "foo" ]] | '1' '-' '1' => [[ drop "bar" ]] EBNF] ] unit-test { 6 } [ - "4+2" [EBNF num=[0-9] => [[ digit> ]] foo=num:x '+' num:y => [[ x y + ]] EBNF] ast>> + "4+2" [EBNF num=[0-9] => [[ digit> ]] foo=num:x '+' num:y => [[ x y + ]] EBNF] ] unit-test { 6 } [ - "4+2" [EBNF foo=[0-9]:x '+' [0-9]:y => [[ x digit> y digit> + ]] EBNF] ast>> + "4+2" [EBNF foo=[0-9]:x '+' [0-9]:y => [[ x digit> y digit> + ]] EBNF] ] unit-test { 10 } [ - { 1 2 3 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF] ast>> + { 1 2 3 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF] ] unit-test [ @@ -181,7 +181,7 @@ IN: peg.ebnf.tests ] must-fail { 3 } [ - { 1 2 "a" 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF] ast>> + { 1 2 "a" 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF] ] unit-test [ @@ -189,44 +189,44 @@ IN: peg.ebnf.tests ] must-fail { V{ "a" " " "b" } } [ - "a b" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF] ast>> + "a b" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF] ] unit-test { V{ "a" "\t" "b" } } [ - "a\tb" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF] ast>> + "a\tb" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF] ] unit-test { V{ "a" "\n" "b" } } [ - "a\nb" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF] ast>> + "a\nb" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF] ] unit-test { V{ "a" f "b" } } [ - "ab" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF] ast>> + "ab" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF] ] unit-test { V{ "a" " " "b" } } [ - "a b" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF] ast>> + "a b" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF] ] unit-test { V{ "a" "\t" "b" } } [ - "a\tb" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF] ast>> + "a\tb" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF] ] unit-test { V{ "a" "\n" "b" } } [ - "a\nb" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF] ast>> + "a\nb" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF] ] unit-test { V{ "a" "b" } } [ - "ab" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF] ast>> + "ab" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF] ] unit-test { V{ "a" "b" } } [ - "a\tb" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF] ast>> + "a\tb" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF] ] unit-test { V{ "a" "b" } } [ - "a\nb" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF] ast>> + "a\nb" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF] ] unit-test [ @@ -236,19 +236,19 @@ IN: peg.ebnf.tests { V{ V{ 49 } "+" V{ 49 } } } [ #! Test direct left recursion. #! Using packrat, so first part of expr fails, causing 2nd choice to be used - "1+1" [EBNF num=([0-9])+ expr=expr "+" num | num EBNF] ast>> + "1+1" [EBNF num=([0-9])+ expr=expr "+" num | num EBNF] ] unit-test { V{ V{ V{ 49 } "+" V{ 49 } } "+" V{ 49 } } } [ #! Test direct left recursion. #! Using packrat, so first part of expr fails, causing 2nd choice to be used - "1+1+1" [EBNF num=([0-9])+ expr=expr "+" num | num EBNF] ast>> + "1+1+1" [EBNF num=([0-9])+ expr=expr "+" num | num EBNF] ] unit-test { V{ V{ V{ 49 } "+" V{ 49 } } "+" V{ 49 } } } [ #! Test indirect left recursion. #! Using packrat, so first part of expr fails, causing 2nd choice to be used - "1+1+1" [EBNF num=([0-9])+ x=expr expr=x "+" num | num EBNF] ast>> + "1+1+1" [EBNF num=([0-9])+ x=expr expr=x "+" num | num EBNF] ] unit-test { t } [ @@ -281,37 +281,37 @@ main = Primary ;EBNF { "this" } [ - "this" primary ast>> + "this" primary ] unit-test { V{ "this" "." "x" } } [ - "this.x" primary ast>> + "this.x" primary ] unit-test { V{ V{ "this" "." "x" } "." "y" } } [ - "this.x.y" primary ast>> + "this.x.y" primary ] unit-test { V{ V{ "this" "." "x" } "." "m" "(" ")" } } [ - "this.x.m()" primary ast>> + "this.x.m()" primary ] unit-test { V{ V{ V{ "x" "[" "i" "]" } "[" "j" "]" } "." "y" } } [ - "x[i][j].y" primary ast>> + "x[i][j].y" primary ] unit-test 'ebnf' compile must-infer { V{ V{ "a" "b" } "c" } } [ - "abc" [EBNF a="a" "b" foo=(a "c") EBNF] ast>> + "abc" [EBNF a="a" "b" foo=(a "c") EBNF] ] unit-test { V{ V{ "a" "b" } "c" } } [ - "abc" [EBNF a="a" "b" foo={a "c"} EBNF] ast>> + "abc" [EBNF a="a" "b" foo={a "c"} EBNF] ] unit-test { V{ V{ "a" "b" } "c" } } [ - "abc" [EBNF a="a" "b" foo=a "c" EBNF] ast>> + "abc" [EBNF a="a" "b" foo=a "c" EBNF] ] unit-test [ @@ -331,7 +331,7 @@ main = Primary ] must-fail { V{ V{ "a" "b" } "c" } } [ - "ab c" [EBNF a="a" "b" foo={a "c"} EBNF] ast>> + "ab c" [EBNF a="a" "b" foo={a "c"} EBNF] ] unit-test [ @@ -351,37 +351,37 @@ main = Primary ] must-fail { V{ V{ V{ "a" "b" } "c" } V{ V{ "a" "b" } "c" } } } [ - "ab cab c" [EBNF a="a" "b" foo={a "c"}* EBNF] ast>> + "ab cab c" [EBNF a="a" "b" foo={a "c"}* EBNF] ] unit-test { V{ } } [ - "ab cab c" [EBNF a="a" "b" foo=(a "c")* EBNF] ast>> + "ab cab c" [EBNF a="a" "b" foo=(a "c")* EBNF] ] unit-test { V{ V{ V{ "a" "b" } "c" } V{ V{ "a" "b" } "c" } } } [ - "ab c ab c" [EBNF a="a" "b" foo={a "c"}* EBNF] ast>> + "ab c ab c" [EBNF a="a" "b" foo={a "c"}* EBNF] ] unit-test { V{ } } [ - "ab c ab c" [EBNF a="a" "b" foo=(a "c")* EBNF] ast>> + "ab c ab c" [EBNF a="a" "b" foo=(a "c")* EBNF] ] unit-test { V{ "a" "a" "a" } } [ - "aaa" [EBNF a=('a')* b=!('b') a:x => [[ x ]] EBNF] ast>> + "aaa" [EBNF a=('a')* b=!('b') a:x => [[ x ]] EBNF] ] unit-test { t } [ - "aaa" [EBNF a=('a')* b=!('b') a:x => [[ x ]] EBNF] ast>> - "aaa" [EBNF a=('a')* b=!('b') (a):x => [[ x ]] EBNF] ast>> = + "aaa" [EBNF a=('a')* b=!('b') a:x => [[ x ]] EBNF] + "aaa" [EBNF a=('a')* b=!('b') (a):x => [[ x ]] EBNF] = ] unit-test { V{ "a" "a" "a" } } [ - "aaa" [EBNF a=('a')* b=a:x => [[ x ]] EBNF] ast>> + "aaa" [EBNF a=('a')* b=a:x => [[ x ]] EBNF] ] unit-test { t } [ - "aaa" [EBNF a=('a')* b=a:x => [[ x ]] EBNF] ast>> - "aaa" [EBNF a=('a')* b=(a):x => [[ x ]] EBNF] ast>> = + "aaa" [EBNF a=('a')* b=a:x => [[ x ]] EBNF] + "aaa" [EBNF a=('a')* b=(a):x => [[ x ]] EBNF] = ] unit-test { t } [ @@ -429,19 +429,19 @@ foo= 'd' ;EBNF { "a" } [ - "a" parser1 ast>> + "a" parser1 ] unit-test { V{ "a" "b" } } [ - "ab" parser2 ast>> + "ab" parser2 ] unit-test { V{ "a" "c" } } [ - "ac" parser3 ast>> + "ac" parser3 ] unit-test { V{ CHAR: a "d" } } [ - "ad" parser4 ast>> + "ad" parser4 ] unit-test { t } [ @@ -460,7 +460,7 @@ foo= 'd' #! Tokenizer tests { V{ "a" CHAR: b } } [ - "ab" [EBNF tokenizer=default foo="a" . EBNF] ast>> + "ab" [EBNF tokenizer=default foo="a" . EBNF] ] unit-test TUPLE: ast-number value ; @@ -488,7 +488,7 @@ Tok = Spaces (Number | Special ) tokenizer = foo=. tokenizer=default baz=. main = bar foo foo baz - EBNF] ast>> + EBNF] ] unit-test { V{ CHAR: 5 "+" CHAR: 2 } } [ @@ -499,7 +499,7 @@ Tok = Spaces (Number | Special ) spaces=space* => [[ ignore ]] tokenizer=spaces (number | operator) main= . . . - EBNF] ast>> + EBNF] ] unit-test { V{ CHAR: 5 "+" CHAR: 2 } } [ @@ -510,13 +510,13 @@ Tok = Spaces (Number | Special ) spaces=space* => [[ ignore ]] tokenizer=spaces (number | operator) main= . . . - EBNF] ast>> + EBNF] ] unit-test { "++" } [ - "++--" [EBNF tokenizer=("++" | "--") main="++" EBNF] ast>> + "++--" [EBNF tokenizer=("++" | "--") main="++" EBNF] ] unit-test { "\\" } [ - "\\" [EBNF foo="\\" EBNF] ast>> + "\\" [EBNF foo="\\" EBNF] ] unit-test \ No newline at end of file diff --git a/extra/peg/ebnf/ebnf.factor b/extra/peg/ebnf/ebnf.factor index 2a57015fa6..2a75fcccc0 100644 --- a/extra/peg/ebnf/ebnf.factor +++ b/extra/peg/ebnf/ebnf.factor @@ -523,7 +523,7 @@ M: ebnf-non-terminal (transform) ( ast -- parser ) : ebnf>quot ( string -- hashtable quot ) parse-ebnf dup dup parser [ main swap at compile ] with-variable - [ compiled-parse ] curry [ with-scope ] curry ; + [ compiled-parse ] curry [ with-scope ast>> ] curry ; : " reset-tokenizer parse-multiline-string parse-ebnf main swap at parsed reset-tokenizer ; parsing diff --git a/extra/peg/javascript/javascript.factor b/extra/peg/javascript/javascript.factor index 8fe0538eae..4a919cf39f 100644 --- a/extra/peg/javascript/javascript.factor +++ b/extra/peg/javascript/javascript.factor @@ -4,8 +4,4 @@ USING: kernel accessors peg.javascript.tokenizer peg.javascript.parser ; IN: peg.javascript : parse-javascript ( string -- ast ) - javascript [ - ast>> - ] [ - "Unable to parse JavaScript" throw - ] if* ; + javascript ; diff --git a/extra/peg/javascript/parser/parser-tests.factor b/extra/peg/javascript/parser/parser-tests.factor index fd0e27b6d4..769dc41f78 100644 --- a/extra/peg/javascript/parser/parser-tests.factor +++ b/extra/peg/javascript/parser/parser-tests.factor @@ -2,7 +2,7 @@ ! See http://factorcode.org/license.txt for BSD license. ! USING: kernel tools.test peg peg.javascript.ast peg.javascript.parser - accessors multiline sequences math ; + accessors multiline sequences math peg.ebnf ; IN: peg.javascript.parser.tests \ javascript must-infer @@ -23,14 +23,14 @@ IN: peg.javascript.parser.tests } } } [ - "123; 'hello'; foo(x);" javascript ast>> + "123; 'hello'; foo(x);" javascript ] unit-test { t } [ <" var x=5 var y=10 -"> javascript remaining>> length zero? +"> main \ javascript rule (parse) remaining>> length zero? ] unit-test @@ -41,7 +41,7 @@ function foldl(f, initial, seq) { initial = f(initial, seq[i]); return initial; } -"> javascript remaining>> length zero? +"> main \ javascript rule (parse) remaining>> length zero? ] unit-test { t } [ @@ -52,6 +52,6 @@ ParseState.prototype.from = function(index) { r.length = this.length - index; return r; } -"> javascript remaining>> length zero? +"> main \ javascript rule (parse) remaining>> length zero? ] unit-test diff --git a/extra/peg/javascript/tokenizer/tokenizer-tests.factor b/extra/peg/javascript/tokenizer/tokenizer-tests.factor index a61125d08c..f0080a31b2 100644 --- a/extra/peg/javascript/tokenizer/tokenizer-tests.factor +++ b/extra/peg/javascript/tokenizer/tokenizer-tests.factor @@ -19,9 +19,9 @@ IN: peg.javascript.tokenizer.tests ";" } } [ - "123; 'hello'; foo(x);" tokenize-javascript ast>> + "123; 'hello'; foo(x);" tokenize-javascript ] unit-test { V{ T{ ast-regexp f "<(w+)[^>]*?)/>" "g" } } } [ - "/<(\\w+)[^>]*?)\\/>/g" tokenize-javascript ast>> + "/<(\\w+)[^>]*?)\\/>/g" tokenize-javascript ] unit-test \ No newline at end of file