From a30c0c0946b54f60d13f289d7c9531533c24dd83 Mon Sep 17 00:00:00 2001 From: John Benediktsson Date: Sun, 19 Apr 2015 20:47:55 -0700 Subject: [PATCH] simple-tokenizer: consider \t \n \r spaces also. --- basis/simple-tokenizer/simple-tokenizer-tests.factor | 5 +++++ basis/simple-tokenizer/simple-tokenizer.factor | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/basis/simple-tokenizer/simple-tokenizer-tests.factor b/basis/simple-tokenizer/simple-tokenizer-tests.factor index 3b44f03650..5f71903313 100644 --- a/basis/simple-tokenizer/simple-tokenizer-tests.factor +++ b/basis/simple-tokenizer/simple-tokenizer-tests.factor @@ -31,3 +31,8 @@ USING: simple-tokenizer tools.test ; ] [ "\"Hello world.app/Contents/MacOS/hello-ui\" -i=boot.macosx-ppc.image \"-include= math compiler ui\" -deploy-vocab=hello-ui \"-output-image=Hello world.app/Contents/Resources/hello-ui.image\" -no-stack-traces -no-user-init" tokenize ] unit-test + +{ V{ "ls" "-l" } } [ "ls -l" tokenize ] unit-test +{ V{ "ls" "-l" } } [ "ls -l\n" tokenize ] unit-test +{ V{ "ls" "-l" } } [ "\nls -l" tokenize ] unit-test +{ V{ "ls" "-l" } } [ "\nls -l\n" tokenize ] unit-test diff --git a/basis/simple-tokenizer/simple-tokenizer.factor b/basis/simple-tokenizer/simple-tokenizer.factor index f7de0822c1..cf6b5a6091 100644 --- a/basis/simple-tokenizer/simple-tokenizer.factor +++ b/basis/simple-tokenizer/simple-tokenizer.factor @@ -4,10 +4,10 @@ USING: peg.ebnf strings ; IN: simple-tokenizer EBNF: tokenize -space = " " +space = [ \t\n\r] escaped-char = "\" .:ch => [[ ch ]] quoted = '"' (escaped-char | [^"])*:a '"' => [[ a ]] -unquoted = (escaped-char | [^ "])+ +unquoted = (escaped-char | [^ \t\n\r"])+ argument = (quoted | unquoted) => [[ >string ]] command = space* (argument:a space* => [[ a ]])+:c !(.) => [[ c ]] ;EBNF