steve
/
factor
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[EBNF and EBNF: now return ast

db4
Chris Double 2008-07-04 15:48:52 +12:00
parent 72bd6b4dc8
commit f3145c5961
5 changed files with 65 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
extra/peg/ebnf/ebnf-tests.factor
View File

@ -113,23 +113,23 @@ IN: peg.ebnf.tests
] unit-test ] unit-test
{ V{ "a" "b" } } [ { V{ "a" "b" } } [
"ab" [EBNF foo='a' 'b' EBNF] ast>> "ab" [EBNF foo='a' 'b' EBNF]
] unit-test ] unit-test
{ V{ 1 "b" } } [ { V{ 1 "b" } } [
"ab" [EBNF foo=('a')[[ drop 1 ]] 'b' EBNF] ast>> "ab" [EBNF foo=('a')[[ drop 1 ]] 'b' EBNF]
] unit-test ] unit-test
{ V{ 1 2 } } [ { V{ 1 2 } } [
"ab" [EBNF foo=('a') [[ drop 1 ]] ('b') [[ drop 2 ]] EBNF] ast>> "ab" [EBNF foo=('a') [[ drop 1 ]] ('b') [[ drop 2 ]] EBNF]
] unit-test ] unit-test
{ CHAR: A } [ { CHAR: A } [
"A" [EBNF foo=[A-Z] EBNF] ast>> "A" [EBNF foo=[A-Z] EBNF]
] unit-test ] unit-test
{ CHAR: Z } [ { CHAR: Z } [
"Z" [EBNF foo=[A-Z] EBNF] ast>> "Z" [EBNF foo=[A-Z] EBNF]
] unit-test ] unit-test
[ [
@ -137,7 +137,7 @@ IN: peg.ebnf.tests
] must-fail ] must-fail
{ CHAR: 0 } [ { CHAR: 0 } [
"0" [EBNF foo=[^A-Z] EBNF] ast>> "0" [EBNF foo=[^A-Z] EBNF]
] unit-test ] unit-test
[ [
@ -149,31 +149,31 @@ IN: peg.ebnf.tests
] must-fail ] must-fail
{ V{ "1" "+" "foo" } } [ { V{ "1" "+" "foo" } } [
"1+1" [EBNF foo='1' '+' '1' [[ drop "foo" ]] EBNF] ast>> "1+1" [EBNF foo='1' '+' '1' [[ drop "foo" ]] EBNF]
] unit-test ] unit-test
{ "foo" } [ { "foo" } [
"1+1" [EBNF foo='1' '+' '1' => [[ drop "foo" ]] EBNF] ast>> "1+1" [EBNF foo='1' '+' '1' => [[ drop "foo" ]] EBNF]
] unit-test ] unit-test
{ "foo" } [ { "foo" } [
"1+1" [EBNF foo='1' '+' '1' => [[ drop "foo" ]] | '1' '-' '1' => [[ drop "bar" ]] EBNF] ast>> "1+1" [EBNF foo='1' '+' '1' => [[ drop "foo" ]] | '1' '-' '1' => [[ drop "bar" ]] EBNF]
] unit-test ] unit-test
{ "bar" } [ { "bar" } [
"1-1" [EBNF foo='1' '+' '1' => [[ drop "foo" ]] | '1' '-' '1' => [[ drop "bar" ]] EBNF] ast>> "1-1" [EBNF foo='1' '+' '1' => [[ drop "foo" ]] | '1' '-' '1' => [[ drop "bar" ]] EBNF]
] unit-test ] unit-test
{ 6 } [ { 6 } [
"4+2" [EBNF num=[0-9] => [[ digit> ]] foo=num:x '+' num:y => [[ x y + ]] EBNF] ast>> "4+2" [EBNF num=[0-9] => [[ digit> ]] foo=num:x '+' num:y => [[ x y + ]] EBNF]
] unit-test ] unit-test
{ 6 } [ { 6 } [
"4+2" [EBNF foo=[0-9]:x '+' [0-9]:y => [[ x digit> y digit> + ]] EBNF] ast>> "4+2" [EBNF foo=[0-9]:x '+' [0-9]:y => [[ x digit> y digit> + ]] EBNF]
] unit-test ] unit-test
{ 10 } [ { 10 } [
{ 1 2 3 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF] ast>> { 1 2 3 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF]
] unit-test ] unit-test
[ [
@ -181,7 +181,7 @@ IN: peg.ebnf.tests
] must-fail ] must-fail
{ 3 } [ { 3 } [
{ 1 2 "a" 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF] ast>> { 1 2 "a" 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF]
] unit-test ] unit-test
[ [
@ -189,44 +189,44 @@ IN: peg.ebnf.tests
] must-fail ] must-fail
{ V{ "a" " " "b" } } [ { V{ "a" " " "b" } } [
"a b" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF] ast>> "a b" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF]
] unit-test ] unit-test
{ V{ "a" "\t" "b" } } [ { V{ "a" "\t" "b" } } [
"a\tb" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF] ast>> "a\tb" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF]
] unit-test ] unit-test
{ V{ "a" "\n" "b" } } [ { V{ "a" "\n" "b" } } [
"a\nb" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF] ast>> "a\nb" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF]
] unit-test ] unit-test
{ V{ "a" f "b" } } [ { V{ "a" f "b" } } [
"ab" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF] ast>> "ab" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF]
] unit-test ] unit-test
{ V{ "a" " " "b" } } [ { V{ "a" " " "b" } } [
"a b" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF] ast>> "a b" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF]
] unit-test ] unit-test
{ V{ "a" "\t" "b" } } [ { V{ "a" "\t" "b" } } [
"a\tb" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF] ast>> "a\tb" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF]
] unit-test ] unit-test
{ V{ "a" "\n" "b" } } [ { V{ "a" "\n" "b" } } [
"a\nb" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF] ast>> "a\nb" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF]
] unit-test ] unit-test
{ V{ "a" "b" } } [ { V{ "a" "b" } } [
"ab" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF] ast>> "ab" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF]
] unit-test ] unit-test
{ V{ "a" "b" } } [ { V{ "a" "b" } } [
"a\tb" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF] ast>> "a\tb" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF]
] unit-test ] unit-test
{ V{ "a" "b" } } [ { V{ "a" "b" } } [
"a\nb" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF] ast>> "a\nb" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF]
] unit-test ] unit-test
[ [
@ -236,19 +236,19 @@ IN: peg.ebnf.tests
{ V{ V{ 49 } "+" V{ 49 } } } [ { V{ V{ 49 } "+" V{ 49 } } } [
#! Test direct left recursion. #! Test direct left recursion.
#! Using packrat, so first part of expr fails, causing 2nd choice to be used #! Using packrat, so first part of expr fails, causing 2nd choice to be used
"1+1" [EBNF num=([0-9])+ expr=expr "+" num | num EBNF] ast>> "1+1" [EBNF num=([0-9])+ expr=expr "+" num | num EBNF]
] unit-test ] unit-test
{ V{ V{ V{ 49 } "+" V{ 49 } } "+" V{ 49 } } } [ { V{ V{ V{ 49 } "+" V{ 49 } } "+" V{ 49 } } } [
#! Test direct left recursion. #! Test direct left recursion.
#! Using packrat, so first part of expr fails, causing 2nd choice to be used #! Using packrat, so first part of expr fails, causing 2nd choice to be used
"1+1+1" [EBNF num=([0-9])+ expr=expr "+" num | num EBNF] ast>> "1+1+1" [EBNF num=([0-9])+ expr=expr "+" num | num EBNF]
] unit-test ] unit-test
{ V{ V{ V{ 49 } "+" V{ 49 } } "+" V{ 49 } } } [ { V{ V{ V{ 49 } "+" V{ 49 } } "+" V{ 49 } } } [
#! Test indirect left recursion. #! Test indirect left recursion.
#! Using packrat, so first part of expr fails, causing 2nd choice to be used #! Using packrat, so first part of expr fails, causing 2nd choice to be used
"1+1+1" [EBNF num=([0-9])+ x=expr expr=x "+" num | num EBNF] ast>> "1+1+1" [EBNF num=([0-9])+ x=expr expr=x "+" num | num EBNF]
] unit-test ] unit-test
{ t } [ { t } [
@ -281,37 +281,37 @@ main = Primary
;EBNF ;EBNF
{ "this" } [ { "this" } [
"this" primary ast>> "this" primary
] unit-test ] unit-test
{ V{ "this" "." "x" } } [ { V{ "this" "." "x" } } [
"this.x" primary ast>> "this.x" primary
] unit-test ] unit-test
{ V{ V{ "this" "." "x" } "." "y" } } [ { V{ V{ "this" "." "x" } "." "y" } } [
"this.x.y" primary ast>> "this.x.y" primary
] unit-test ] unit-test
{ V{ V{ "this" "." "x" } "." "m" "(" ")" } } [ { V{ V{ "this" "." "x" } "." "m" "(" ")" } } [
"this.x.m()" primary ast>> "this.x.m()" primary
] unit-test ] unit-test
{ V{ V{ V{ "x" "[" "i" "]" } "[" "j" "]" } "." "y" } } [ { V{ V{ V{ "x" "[" "i" "]" } "[" "j" "]" } "." "y" } } [
"x[i][j].y" primary ast>> "x[i][j].y" primary
] unit-test ] unit-test
'ebnf' compile must-infer 'ebnf' compile must-infer
{ V{ V{ "a" "b" } "c" } } [ { V{ V{ "a" "b" } "c" } } [
"abc" [EBNF a="a" "b" foo=(a "c") EBNF] ast>> "abc" [EBNF a="a" "b" foo=(a "c") EBNF]
] unit-test ] unit-test
{ V{ V{ "a" "b" } "c" } } [ { V{ V{ "a" "b" } "c" } } [
"abc" [EBNF a="a" "b" foo={a "c"} EBNF] ast>> "abc" [EBNF a="a" "b" foo={a "c"} EBNF]
] unit-test ] unit-test
{ V{ V{ "a" "b" } "c" } } [ { V{ V{ "a" "b" } "c" } } [
"abc" [EBNF a="a" "b" foo=a "c" EBNF] ast>> "abc" [EBNF a="a" "b" foo=a "c" EBNF]
] unit-test ] unit-test
[ [
@ -331,7 +331,7 @@ main = Primary
] must-fail ] must-fail
{ V{ V{ "a" "b" } "c" } } [ { V{ V{ "a" "b" } "c" } } [
"ab c" [EBNF a="a" "b" foo={a "c"} EBNF] ast>> "ab c" [EBNF a="a" "b" foo={a "c"} EBNF]
] unit-test ] unit-test
[ [
@ -351,37 +351,37 @@ main = Primary
] must-fail ] must-fail
{ V{ V{ V{ "a" "b" } "c" } V{ V{ "a" "b" } "c" } } } [ { V{ V{ V{ "a" "b" } "c" } V{ V{ "a" "b" } "c" } } } [
"ab cab c" [EBNF a="a" "b" foo={a "c"}* EBNF] ast>> "ab cab c" [EBNF a="a" "b" foo={a "c"}* EBNF]
] unit-test ] unit-test
{ V{ } } [ { V{ } } [
"ab cab c" [EBNF a="a" "b" foo=(a "c")* EBNF] ast>> "ab cab c" [EBNF a="a" "b" foo=(a "c")* EBNF]
] unit-test ] unit-test
{ V{ V{ V{ "a" "b" } "c" } V{ V{ "a" "b" } "c" } } } [ { V{ V{ V{ "a" "b" } "c" } V{ V{ "a" "b" } "c" } } } [
"ab c ab c" [EBNF a="a" "b" foo={a "c"}* EBNF] ast>> "ab c ab c" [EBNF a="a" "b" foo={a "c"}* EBNF]
] unit-test ] unit-test
{ V{ } } [ { V{ } } [
"ab c ab c" [EBNF a="a" "b" foo=(a "c")* EBNF] ast>> "ab c ab c" [EBNF a="a" "b" foo=(a "c")* EBNF]
] unit-test ] unit-test
{ V{ "a" "a" "a" } } [ { V{ "a" "a" "a" } } [
"aaa" [EBNF a=('a')* b=!('b') a:x => [[ x ]] EBNF] ast>> "aaa" [EBNF a=('a')* b=!('b') a:x => [[ x ]] EBNF]
] unit-test ] unit-test
{ t } [ { t } [
"aaa" [EBNF a=('a')* b=!('b') a:x => [[ x ]] EBNF] ast>> "aaa" [EBNF a=('a')* b=!('b') a:x => [[ x ]] EBNF]
"aaa" [EBNF a=('a')* b=!('b') (a):x => [[ x ]] EBNF] ast>> = "aaa" [EBNF a=('a')* b=!('b') (a):x => [[ x ]] EBNF] =
] unit-test ] unit-test
{ V{ "a" "a" "a" } } [ { V{ "a" "a" "a" } } [
"aaa" [EBNF a=('a')* b=a:x => [[ x ]] EBNF] ast>> "aaa" [EBNF a=('a')* b=a:x => [[ x ]] EBNF]
] unit-test ] unit-test
{ t } [ { t } [
"aaa" [EBNF a=('a')* b=a:x => [[ x ]] EBNF] ast>> "aaa" [EBNF a=('a')* b=a:x => [[ x ]] EBNF]
"aaa" [EBNF a=('a')* b=(a):x => [[ x ]] EBNF] ast>> = "aaa" [EBNF a=('a')* b=(a):x => [[ x ]] EBNF] =
] unit-test ] unit-test
{ t } [ { t } [
@ -429,19 +429,19 @@ foo=<foreign any-char> 'd'
;EBNF ;EBNF
{ "a" } [ { "a" } [
"a" parser1 ast>> "a" parser1
] unit-test ] unit-test
{ V{ "a" "b" } } [ { V{ "a" "b" } } [
"ab" parser2 ast>> "ab" parser2
] unit-test ] unit-test
{ V{ "a" "c" } } [ { V{ "a" "c" } } [
"ac" parser3 ast>> "ac" parser3
] unit-test ] unit-test
{ V{ CHAR: a "d" } } [ { V{ CHAR: a "d" } } [
"ad" parser4 ast>> "ad" parser4
] unit-test ] unit-test
{ t } [ { t } [
@ -460,7 +460,7 @@ foo=<foreign any-char> 'd'
#! Tokenizer tests #! Tokenizer tests
{ V{ "a" CHAR: b } } [ { V{ "a" CHAR: b } } [
"ab" [EBNF tokenizer=default foo="a" . EBNF] ast>> "ab" [EBNF tokenizer=default foo="a" . EBNF]
] unit-test ] unit-test
TUPLE: ast-number value ; TUPLE: ast-number value ;
@ -488,7 +488,7 @@ Tok = Spaces (Number | Special )
tokenizer = <foreign a-tokenizer Tok> foo=. tokenizer = <foreign a-tokenizer Tok> foo=.
tokenizer=default baz=. tokenizer=default baz=.
main = bar foo foo baz main = bar foo foo baz
EBNF] ast>> EBNF]
] unit-test ] unit-test
{ V{ CHAR: 5 "+" CHAR: 2 } } [ { V{ CHAR: 5 "+" CHAR: 2 } } [
@ -499,7 +499,7 @@ Tok = Spaces (Number | Special )
spaces=space* => [[ ignore ]] spaces=space* => [[ ignore ]]
tokenizer=spaces (number | operator) tokenizer=spaces (number | operator)
main= . . . main= . . .
EBNF] ast>> EBNF]
] unit-test ] unit-test
{ V{ CHAR: 5 "+" CHAR: 2 } } [ { V{ CHAR: 5 "+" CHAR: 2 } } [
@ -510,13 +510,13 @@ Tok = Spaces (Number | Special )
spaces=space* => [[ ignore ]] spaces=space* => [[ ignore ]]
tokenizer=spaces (number | operator) tokenizer=spaces (number | operator)
main= . . . main= . . .
EBNF] ast>> EBNF]
] unit-test ] unit-test
{ "++" } [ { "++" } [
"++--" [EBNF tokenizer=("++" | "--") main="++" EBNF] ast>> "++--" [EBNF tokenizer=("++" | "--") main="++" EBNF]
] unit-test ] unit-test
{ "\\" } [ { "\\" } [
"\\" [EBNF foo="\\" EBNF] ast>> "\\" [EBNF foo="\\" EBNF]
] unit-test ] unit-test

2
extra/peg/ebnf/ebnf.factor
View File

@ -523,7 +523,7 @@ M: ebnf-non-terminal (transform) ( ast -- parser )
: ebnf>quot ( string -- hashtable quot ) : ebnf>quot ( string -- hashtable quot )
parse-ebnf dup dup parser [ main swap at compile ] with-variable parse-ebnf dup dup parser [ main swap at compile ] with-variable
[ compiled-parse ] curry [ with-scope ] curry ; [ compiled-parse ] curry [ with-scope ast>> ] curry ;
: <EBNF "EBNF>" reset-tokenizer parse-multiline-string parse-ebnf main swap at : <EBNF "EBNF>" reset-tokenizer parse-multiline-string parse-ebnf main swap at
parsed reset-tokenizer ; parsing parsed reset-tokenizer ; parsing

6
extra/peg/javascript/javascript.factor
View File

@ -4,8 +4,4 @@ USING: kernel accessors peg.javascript.tokenizer peg.javascript.parser ;
IN: peg.javascript IN: peg.javascript
: parse-javascript ( string -- ast ) : parse-javascript ( string -- ast )
javascript [ javascript ;
ast>>
] [
"Unable to parse JavaScript" throw
] if* ;

10
extra/peg/javascript/parser/parser-tests.factor
View File

@ -2,7 +2,7 @@
! See http://factorcode.org/license.txt for BSD license. ! See http://factorcode.org/license.txt for BSD license.
! !
USING: kernel tools.test peg peg.javascript.ast peg.javascript.parser USING: kernel tools.test peg peg.javascript.ast peg.javascript.parser
accessors multiline sequences math ; accessors multiline sequences math peg.ebnf ;
IN: peg.javascript.parser.tests IN: peg.javascript.parser.tests
\ javascript must-infer \ javascript must-infer
@ -23,14 +23,14 @@ IN: peg.javascript.parser.tests
} }
} }
} [ } [
"123; 'hello'; foo(x);" javascript ast>> "123; 'hello'; foo(x);" javascript
] unit-test ] unit-test
{ t } [ { t } [
<" <"
var x=5 var x=5
var y=10 var y=10
"> javascript remaining>> length zero? "> main \ javascript rule (parse) remaining>> length zero?
] unit-test ] unit-test
@ -41,7 +41,7 @@ function foldl(f, initial, seq) {
initial = f(initial, seq[i]); initial = f(initial, seq[i]);
return initial; return initial;
} }
"> javascript remaining>> length zero? "> main \ javascript rule (parse) remaining>> length zero?
] unit-test ] unit-test
{ t } [ { t } [
@ -52,6 +52,6 @@ ParseState.prototype.from = function(index) {
r.length = this.length - index; r.length = this.length - index;
return r; return r;
} }
"> javascript remaining>> length zero? "> main \ javascript rule (parse) remaining>> length zero?
] unit-test ] unit-test

4
extra/peg/javascript/tokenizer/tokenizer-tests.factor
View File

@ -19,9 +19,9 @@ IN: peg.javascript.tokenizer.tests
";" ";"
} }
} [ } [
"123; 'hello'; foo(x);" tokenize-javascript ast>> "123; 'hello'; foo(x);" tokenize-javascript
] unit-test ] unit-test
{ V{ T{ ast-regexp f "<(w+)[^>]*?)/>" "g" } } } [ { V{ T{ ast-regexp f "<(w+)[^>]*?)/>" "g" } } } [
"/<(\\w+)[^>]*?)\\/>/g" tokenize-javascript ast>> "/<(\\w+)[^>]*?)\\/>/g" tokenize-javascript
] unit-test ] unit-test