steve
/
factor
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

remove tokenize-line

db4
Doug Coleman 2009-04-01 17:28:36 -05:00
parent 5ec23cdde0
commit 7b6260ca8c
2 changed files with 12 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
extra/html/parser/state/state-tests.factor
View File

@ -88,5 +88,8 @@ IN: html.parser.state.tests
[ "c" ] [ "c" ]
[ "c" <state-parser> take-token ] unit-test [ "c" <state-parser> take-token ] unit-test
[ { "a" "b" "c" "abcd e \\\"f g" } ] [ f ]
[ "a b c \"abcd e \\\"f g\"" CHAR: \ CHAR: " tokenize-line ] unit-test [ "" <state-parser> take-token ] unit-test
[ "abcd e \\\"f g" ]
[ "\"abcd e \\\"f g\"" <state-parser> CHAR: \ CHAR: " take-token* ] unit-test

17
extra/html/parser/state/state.factor
View File

@ -98,19 +98,16 @@ TUPLE: state-parser sequence n ;
start-n state-parser (>>n) f start-n state-parser (>>n) f
] if ; ] if ;
: take-token ( state-parser -- string ) : (take-token) ( state-parser -- string )
skip-whitespace [ current { [ blank? ] [ f = ] } 1|| ] take-until ; skip-whitespace [ current { [ blank? ] [ f = ] } 1|| ] take-until ;
:: (tokenize-line) ( state-parser escape-char quote-char -- ) :: take-token* ( state-parser escape-char quote-char -- string/f )
state-parser skip-whitespace state-parser skip-whitespace
dup current { dup current {
{ quote-char [ { quote-char [ escape-char quote-char take-quoted-string ] }
[ escape-char quote-char take-quoted-string , ] { f [ drop f ] }
[ escape-char quote-char (tokenize-line) ] bi [ drop (take-token) ]
] }
{ f [ drop ] }
[ drop [ take-token , ] [ escape-char quote-char (tokenize-line) ] bi ]
} case ; } case ;
: tokenize-line ( line escape-char quote-char -- seq ) : take-token ( state-parser -- string/f )
[ <state-parser> ] 2dip [ (tokenize-line) ] { } make ; CHAR: \ CHAR: " take-token* ;