From a30c0c0946b54f60d13f289d7c9531533c24dd83 Mon Sep 17 00:00:00 2001
From: John Benediktsson <mrjbq7@gmail.com>
Date: Sun, 19 Apr 2015 20:47:55 -0700
Subject: [PATCH] simple-tokenizer: consider \t \n \r spaces also.

---
 basis/simple-tokenizer/simple-tokenizer-tests.factor | 5 +++++
 basis/simple-tokenizer/simple-tokenizer.factor       | 4 ++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/basis/simple-tokenizer/simple-tokenizer-tests.factor b/basis/simple-tokenizer/simple-tokenizer-tests.factor
index 3b44f03650..5f71903313 100644
--- a/basis/simple-tokenizer/simple-tokenizer-tests.factor
+++ b/basis/simple-tokenizer/simple-tokenizer-tests.factor
@@ -31,3 +31,8 @@ USING: simple-tokenizer tools.test ;
 ] [
     "\"Hello world.app/Contents/MacOS/hello-ui\" -i=boot.macosx-ppc.image \"-include= math compiler ui\" -deploy-vocab=hello-ui \"-output-image=Hello world.app/Contents/Resources/hello-ui.image\" -no-stack-traces -no-user-init" tokenize
 ] unit-test
+
+{ V{ "ls" "-l" } } [ "ls -l" tokenize ] unit-test
+{ V{ "ls" "-l" } } [ "ls -l\n" tokenize ] unit-test
+{ V{ "ls" "-l" } } [ "\nls -l" tokenize ] unit-test
+{ V{ "ls" "-l" } } [ "\nls -l\n" tokenize ] unit-test
diff --git a/basis/simple-tokenizer/simple-tokenizer.factor b/basis/simple-tokenizer/simple-tokenizer.factor
index f7de0822c1..cf6b5a6091 100644
--- a/basis/simple-tokenizer/simple-tokenizer.factor
+++ b/basis/simple-tokenizer/simple-tokenizer.factor
@@ -4,10 +4,10 @@ USING: peg.ebnf strings ;
 IN: simple-tokenizer
 
 EBNF: tokenize
-space = " "
+space = [ \t\n\r]
 escaped-char = "\" .:ch => [[ ch ]]
 quoted = '"' (escaped-char | [^"])*:a '"' => [[ a ]]
-unquoted = (escaped-char | [^ "])+
+unquoted = (escaped-char | [^ \t\n\r"])+
 argument = (quoted | unquoted) => [[ >string ]]
 command = space* (argument:a space* => [[ a ]])+:c !(.) => [[ c ]]
 ;EBNF