simple-tokenizer: consider \t \n \r spaces also.
parent
56bb141257
commit
a30c0c0946
|
@ -31,3 +31,8 @@ USING: simple-tokenizer tools.test ;
|
|||
] [
|
||||
"\"Hello world.app/Contents/MacOS/hello-ui\" -i=boot.macosx-ppc.image \"-include= math compiler ui\" -deploy-vocab=hello-ui \"-output-image=Hello world.app/Contents/Resources/hello-ui.image\" -no-stack-traces -no-user-init" tokenize
|
||||
] unit-test
|
||||
|
||||
{ V{ "ls" "-l" } } [ "ls -l" tokenize ] unit-test
|
||||
{ V{ "ls" "-l" } } [ "ls -l\n" tokenize ] unit-test
|
||||
{ V{ "ls" "-l" } } [ "\nls -l" tokenize ] unit-test
|
||||
{ V{ "ls" "-l" } } [ "\nls -l\n" tokenize ] unit-test
|
||||
|
|
|
@ -4,10 +4,10 @@ USING: peg.ebnf strings ;
|
|||
IN: simple-tokenizer
|
||||
|
||||
EBNF: tokenize
|
||||
space = " "
|
||||
space = [ \t\n\r]
|
||||
escaped-char = "\" .:ch => [[ ch ]]
|
||||
quoted = '"' (escaped-char | [^"])*:a '"' => [[ a ]]
|
||||
unquoted = (escaped-char | [^ "])+
|
||||
unquoted = (escaped-char | [^ \t\n\r"])+
|
||||
argument = (quoted | unquoted) => [[ >string ]]
|
||||
command = space* (argument:a space* => [[ a ]])+:c !(.) => [[ c ]]
|
||||
;EBNF
|
||||
|
|
Loading…
Reference in New Issue