144 lines
3.2 KiB
Factor
Executable File
144 lines
3.2 KiB
Factor
Executable File
USING: xmode.tokens xmode.catalog
|
|
xmode.marker tools.test kernel ;
|
|
IN: xmode.marker.tests
|
|
|
|
[
|
|
{
|
|
T{ token f "int" KEYWORD3 }
|
|
T{ token f " " f }
|
|
T{ token f "x" f }
|
|
}
|
|
] [ f "int x" "c" load-mode tokenize-line nip ] unit-test
|
|
|
|
[
|
|
{
|
|
T{ token f "\"" LITERAL1 }
|
|
T{ token f "hello\\\"" LITERAL1 }
|
|
T{ token f " " LITERAL1 }
|
|
T{ token f "world" LITERAL1 }
|
|
T{ token f "\"" LITERAL1 }
|
|
}
|
|
] [ f "\"hello\\\" world\"" "c" load-mode tokenize-line nip ] unit-test
|
|
|
|
[
|
|
{
|
|
T{ token f "\"" LITERAL1 }
|
|
T{ token f "hello\\\ world" LITERAL1 }
|
|
T{ token f "\"" LITERAL1 }
|
|
}
|
|
] [ f "\"hello\\\ world\"" "c" load-mode tokenize-line nip ] unit-test
|
|
|
|
[
|
|
{
|
|
T{ token f "int" KEYWORD3 }
|
|
T{ token f " " f }
|
|
T{ token f "x" f }
|
|
}
|
|
] [ f "int x" "java" load-mode tokenize-line nip ] unit-test
|
|
|
|
[
|
|
{
|
|
T{ token f "//" COMMENT2 }
|
|
T{ token f " " COMMENT2 }
|
|
T{ token f "hello" COMMENT2 }
|
|
T{ token f " " COMMENT2 }
|
|
T{ token f "world" COMMENT2 }
|
|
}
|
|
] [ f "// hello world" "java" load-mode tokenize-line nip ] unit-test
|
|
|
|
|
|
[
|
|
{
|
|
T{ token f "hello" f }
|
|
T{ token f " " f }
|
|
T{ token f "world" f }
|
|
T{ token f ":" f }
|
|
}
|
|
] [ f "hello world:" "java" load-mode tokenize-line nip ] unit-test
|
|
|
|
[
|
|
{
|
|
T{ token f "hello_world" LABEL }
|
|
T{ token f ":" OPERATOR }
|
|
}
|
|
] [ f "hello_world:" "java" load-mode tokenize-line nip ] unit-test
|
|
|
|
[
|
|
{
|
|
T{ token f "\t" f }
|
|
T{ token f "hello_world" LABEL }
|
|
T{ token f ":" OPERATOR }
|
|
}
|
|
] [ f "\thello_world:" "java" load-mode tokenize-line nip ] unit-test
|
|
|
|
[
|
|
{
|
|
T{ token f "<!" KEYWORD2 }
|
|
T{ token f "ELEMENT" KEYWORD2 }
|
|
T{ token f " " KEYWORD2 }
|
|
T{ token f "%" LITERAL2 }
|
|
T{ token f "hello" LITERAL2 }
|
|
T{ token f ";" LITERAL2 }
|
|
T{ token f " " KEYWORD2 }
|
|
T{ token f ">" KEYWORD2 }
|
|
}
|
|
] [
|
|
f "<!ELEMENT %hello; >" "xml" load-mode tokenize-line nip
|
|
] unit-test
|
|
|
|
[
|
|
{
|
|
T{ token f "<!" KEYWORD2 }
|
|
T{ token f "ELEMENT" KEYWORD2 }
|
|
T{ token f " " KEYWORD2 }
|
|
T{ token f "%" LITERAL2 }
|
|
T{ token f "hello-world" LITERAL2 }
|
|
T{ token f ";" LITERAL2 }
|
|
T{ token f " " KEYWORD2 }
|
|
T{ token f ">" KEYWORD2 }
|
|
}
|
|
] [
|
|
f "<!ELEMENT %hello-world; >" "xml" load-mode tokenize-line nip
|
|
] unit-test
|
|
|
|
[
|
|
{
|
|
T{ token f "$" KEYWORD2 }
|
|
T{ token f "FOO" KEYWORD2 }
|
|
}
|
|
] [
|
|
f "$FOO" "shellscript" load-mode tokenize-line nip
|
|
] unit-test
|
|
|
|
[
|
|
{
|
|
T{ token f "AND" KEYWORD1 }
|
|
}
|
|
] [
|
|
f "AND" "pascal" load-mode tokenize-line nip
|
|
] unit-test
|
|
|
|
[
|
|
{
|
|
T{ token f "Comment {" COMMENT1 }
|
|
T{ token f "XXX" COMMENT1 }
|
|
T{ token f "}" COMMENT1 }
|
|
}
|
|
] [
|
|
f "Comment {XXX}" "rebol" load-mode tokenize-line nip
|
|
] unit-test
|
|
|
|
[
|
|
|
|
] [
|
|
f "font:75%/1.6em \"Lucida Grande\", \"Lucida Sans Unicode\", verdana, geneva, sans-serif;" "css" load-mode tokenize-line 2drop
|
|
] unit-test
|
|
|
|
[
|
|
{
|
|
T{ token f "<" MARKUP }
|
|
T{ token f "aaa" MARKUP }
|
|
T{ token f ">" MARKUP }
|
|
}
|
|
] [ f "<aaa>" "html" load-mode tokenize-line nip ] unit-test
|