2010-02-20 23:00:48 -05:00
|
|
|
! Copyright (C) 2008, 2010 Slava Pestov
|
2008-03-03 17:45:18 -05:00
|
|
|
! See http://factorcode.org/license.txt for BSD license.
|
2017-08-05 23:23:57 -04:00
|
|
|
USING: peg.ebnf multiline strings ;
|
2010-02-20 23:00:48 -05:00
|
|
|
IN: simple-tokenizer
|
2008-03-03 17:45:18 -05:00
|
|
|
|
2017-08-05 23:23:57 -04:00
|
|
|
EBNF: tokenize [=[
|
2015-04-19 23:47:55 -04:00
|
|
|
space = [ \t\n\r]
|
2015-04-20 00:12:40 -04:00
|
|
|
escaped-char = "\\" .:ch => [[ ch ]]
|
2009-04-18 22:53:22 -04:00
|
|
|
quoted = '"' (escaped-char | [^"])*:a '"' => [[ a ]]
|
2015-04-19 23:47:55 -04:00
|
|
|
unquoted = (escaped-char | [^ \t\n\r"])+
|
2009-04-18 22:53:22 -04:00
|
|
|
argument = (quoted | unquoted) => [[ >string ]]
|
|
|
|
command = space* (argument:a space* => [[ a ]])+:c !(.) => [[ c ]]
|
2017-08-05 23:23:57 -04:00
|
|
|
]=]
|