2008-12-17 20:39:02 -05:00
|
|
|
USING: accessors parser words words.symbol
|
|
|
|
|
sequences namespaces kernel assocs
|
2008-01-16 01:04:42 -05:00
|
|
|
compiler.units ;
|
2007-11-28 23:34:11 -05:00
|
|
|
IN: xmode.tokens
|
|
|
|
|
|
|
|
|
|
! Based on org.gjt.sp.jedit.syntax.Token
|
2008-03-11 17:03:22 -04:00
|
|
|
<<
|
2007-11-28 23:34:11 -05:00
|
|
|
SYMBOL: tokens
|
|
|
|
|
|
2014-11-30 22:26:23 -05:00
|
|
|
{
|
|
|
|
|
"COMMENT1" "COMMENT2" "COMMENT3" "COMMENT4" "DIGIT"
|
|
|
|
|
"FUNCTION" "INVALID" "KEYWORD1" "KEYWORD2" "KEYWORD3"
|
|
|
|
|
"KEYWORD4" "LABEL" "LITERAL1" "LITERAL2" "LITERAL3"
|
|
|
|
|
"LITERAL4" "MARKUP" "OPERATOR" "END" "NULL"
|
|
|
|
|
} [
|
2015-06-08 15:38:38 -04:00
|
|
|
dup create-word-in dup define-symbol
|
2008-03-08 05:27:19 -05:00
|
|
|
] H{ } map>assoc tokens set-global
|
|
|
|
|
>>
|
2007-11-28 23:34:11 -05:00
|
|
|
|
2008-01-16 01:04:42 -05:00
|
|
|
: string>token ( string -- id ) tokens get at ;
|
2007-11-28 23:34:11 -05:00
|
|
|
|
|
|
|
|
TUPLE: token str id ;
|
|
|
|
|
|
|
|
|
|
C: <token> token
|
