Rename io.launcher.unix.parser to simple-tokenizer since ftp.server uses it

basis/ftp/server/server.factor

@@ -1,15 +1,14 @@
 ! Copyright (C) 2008 Doug Coleman.
 ! See http://factorcode.org/license.txt for BSD license.
-USING: accessors assocs byte-arrays calendar classes
+USING: accessors assocs byte-arrays calendar classes combinators
-combinators combinators.short-circuit concurrency.promises
+combinators.short-circuit concurrency.promises continuations
-continuations destructors ftp io io.backend io.directories
+destructors ftp io io.backend io.directories io.encodings
-io.encodings io.encodings.binary
+io.encodings.binary tools.files io.encodings.utf8 io.files
-tools.files io.encodings.utf8 io.files io.files.info
+io.files.info io.pathnames io.servers.connection io.sockets
-io.pathnames io.launcher.unix.parser io.servers.connection
+io.streams.duplex io.streams.string io.timeouts kernel make math
-io.sockets io.streams.duplex io.streams.string io.timeouts
+math.bitwise math.parser namespaces sequences splitting threads
-kernel make math math.bitwise math.parser namespaces sequences
+unicode.case logging calendar.format strings io.files.links
-splitting threads unicode.case logging calendar.format
+io.files.types io.encodings.8-bit.latin1 simple-tokenizer ;
-strings io.files.links io.files.types io.encodings.8-bit.latin1 ;
 IN: ftp.server
 
 SYMBOL: server
@@ -24,7 +23,7 @@ TUPLE: ftp-command raw tokenized ;
     dup \ <ftp-command> DEBUG log-message
     ftp-command new
         over >>raw
-        swap tokenize-command >>tokenized ;
+        swap tokenize >>tokenized ;
 
 TUPLE: ftp-get path ;
 : <ftp-get> ( path -- obj )

basis/io/launcher/unix/parser/parser-tests.factor

@@ -1,33 +0,0 @@
-IN: io.launcher.unix.parser.tests
-USING: io.launcher.unix.parser tools.test ;
-
-[ "" tokenize-command ] must-fail
-[ "   " tokenize-command ] must-fail
-[ V{ "a" } ] [ "a" tokenize-command ] unit-test
-[ V{ "abc" } ] [ "abc" tokenize-command ] unit-test
-[ V{ "abc" } ] [ "abc   " tokenize-command ] unit-test
-[ V{ "abc" } ] [ "   abc" tokenize-command ] unit-test
-[ V{ "abc" "def" } ] [ "abc def" tokenize-command ] unit-test
-[ V{ "abc def" } ] [ "abc\\ def" tokenize-command ] unit-test
-[ V{ "abc\\" "def" } ] [ "abc\\\\ def" tokenize-command ] unit-test
-[ V{ "abc\\ def" } ] [ "\"abc\\\\ def\"" tokenize-command ] unit-test
-[ V{ "abc\\ def" } ] [ "  \"abc\\\\ def\"" tokenize-command ] unit-test
-[ V{ "abc\\ def" "hey" } ] [ "\"abc\\\\ def\" hey" tokenize-command ] unit-test
-[ V{ "abc def" "hey" } ] [ "\"abc def\" \"hey\"" tokenize-command ] unit-test
-[ "\"abc def\" \"hey" tokenize-command ] must-fail
-[ "\"abc def" tokenize-command ] must-fail
-[ V{ "abc def" "h\"ey" } ] [ "\"abc def\" \"h\\\"ey\"  " tokenize-command ] unit-test
-
-[
-    V{
-        "Hello world.app/Contents/MacOS/hello-ui"
-        "-i=boot.macosx-ppc.image"
-        "-include= math compiler ui"
-        "-deploy-vocab=hello-ui"
-        "-output-image=Hello world.app/Contents/Resources/hello-ui.image"
-        "-no-stack-traces"
-        "-no-user-init"
-    }
-] [
-    "\"Hello world.app/Contents/MacOS/hello-ui\" -i=boot.macosx-ppc.image \"-include= math compiler ui\" -deploy-vocab=hello-ui \"-output-image=Hello world.app/Contents/Resources/hello-ui.image\" -no-stack-traces -no-user-init" tokenize-command
-] unit-test

basis/io/launcher/unix/parser/platforms.txt

@@ -1 +0,0 @@
-unix

basis/io/launcher/unix/unix.factor

@@ -1,15 +1,14 @@
-! Copyright (C) 2007, 2008 Slava Pestov.
+! Copyright (C) 2007, 2010 Slava Pestov.
 ! See http://factorcode.org/license.txt for BSD license.
 USING: accessors alien.c-types arrays assocs combinators
 continuations environment io io.backend io.backend.unix
-io.files io.files.private io.files.unix io.launcher
+io.files io.files.private io.files.unix io.launcher io.pathnames
-io.launcher.unix.parser io.pathnames io.ports kernel math
+io.ports kernel math namespaces sequences strings system threads
-namespaces sequences strings system threads unix
+unix unix.process unix.ffi simple-tokenizer ;
-unix.process unix.ffi ;
 IN: io.launcher.unix
 
 : get-arguments ( process -- seq )
-    command>> dup string? [ tokenize-command ] when ;
+    command>> dup string? [ tokenize ] when ;
 
 : assoc>env ( assoc -- env )
     [ "=" glue ] { } assoc>map ;

basis/simple-tokenizer/authors.txt

@@ -0,0 +1 @@
+Slava Pestov

basis/simple-tokenizer/simple-tokenizer-docs.factor

@@ -0,0 +1,13 @@
+USING: help.markup help.syntax strings ;
+IN: simple-tokenizer
+
+HELP: tokenize
+{ $values { "input" string } { "ast" "a sequence of strings" } }
+{ $description
+    "Tokenize a string. Supported syntax:"
+    { $list
+        { { $snippet "foo bar baz" } " - simple tokens" }
+        { { $snippet "foo\\ bar" } " - token with an escaped space"}
+        { { $snippet "\"foo bar\"" } " - quoted token" }
+    }
+} ;

basis/simple-tokenizer/simple-tokenizer-tests.factor

@@ -0,0 +1,33 @@
+IN: simple-tokenizer.tests
+USING: simple-tokenizer tools.test ;
+
+[ "" tokenize ] must-fail
+[ "   " tokenize ] must-fail
+[ V{ "a" } ] [ "a" tokenize ] unit-test
+[ V{ "abc" } ] [ "abc" tokenize ] unit-test
+[ V{ "abc" } ] [ "abc   " tokenize ] unit-test
+[ V{ "abc" } ] [ "   abc" tokenize ] unit-test
+[ V{ "abc" "def" } ] [ "abc def" tokenize ] unit-test
+[ V{ "abc def" } ] [ "abc\\ def" tokenize ] unit-test
+[ V{ "abc\\" "def" } ] [ "abc\\\\ def" tokenize ] unit-test
+[ V{ "abc\\ def" } ] [ "\"abc\\\\ def\"" tokenize ] unit-test
+[ V{ "abc\\ def" } ] [ "  \"abc\\\\ def\"" tokenize ] unit-test
+[ V{ "abc\\ def" "hey" } ] [ "\"abc\\\\ def\" hey" tokenize ] unit-test
+[ V{ "abc def" "hey" } ] [ "\"abc def\" \"hey\"" tokenize ] unit-test
+[ "\"abc def\" \"hey" tokenize ] must-fail
+[ "\"abc def" tokenize ] must-fail
+[ V{ "abc def" "h\"ey" } ] [ "\"abc def\" \"h\\\"ey\"  " tokenize ] unit-test
+
+[
+    V{
+        "Hello world.app/Contents/MacOS/hello-ui"
+        "-i=boot.macosx-ppc.image"
+        "-include= math compiler ui"
+        "-deploy-vocab=hello-ui"
+        "-output-image=Hello world.app/Contents/Resources/hello-ui.image"
+        "-no-stack-traces"
+        "-no-user-init"
+    }
+] [
+    "\"Hello world.app/Contents/MacOS/hello-ui\" -i=boot.macosx-ppc.image \"-include= math compiler ui\" -deploy-vocab=hello-ui \"-output-image=Hello world.app/Contents/Resources/hello-ui.image\" -no-stack-traces -no-user-init" tokenize
+] unit-test

basis/simple-tokenizer/simple-tokenizer.factor

@@ -1,13 +1,9 @@
-! Copyright (C) 2008 Slava Pestov
+! Copyright (C) 2008, 2010 Slava Pestov
 ! See http://factorcode.org/license.txt for BSD license.
 USING: peg peg.ebnf arrays sequences strings kernel ;
-IN: io.launcher.unix.parser
+IN: simple-tokenizer
 
-! Our command line parser. Supported syntax:
+EBNF: tokenize
-! foo bar baz -- simple tokens
-! foo\ bar -- escaping the space
-! "foo bar" -- quotation
-EBNF: tokenize-command
 space = " "
 escaped-char = "\" .:ch => [[ ch ]]
 quoted = '"' (escaped-char | [^"])*:a '"' => [[ a ]]

basis/simple-tokenizer/tags.txt

@@ -0,0 +1 @@
+text