diff --git a/core/lexer/lexer.factor b/core/lexer/lexer.factor index 36ea984b2f..2e23e7352b 100644 --- a/core/lexer/lexer.factor +++ b/core/lexer/lexer.factor @@ -94,9 +94,22 @@ M: lexer skip-word : still-parsing-line? ( lexer -- ? ) check-lexer [ column>> ] [ line-length>> ] bi < ; -DEFER: parse-token +: (parse-raw) ( lexer -- str ) + check-lexer { + [ column>> ] + [ skip-word ] + [ column>> ] + [ line-text>> ] + } cleave subseq ; - - -: (parse-token) ( lexer -- str ) - dup check-lexer { - [ column>> ] - [ skip-word ] - [ column>> ] - [ line-text>> ] - } cleave subseq skip-comments ; - : parse-token ( lexer -- str/f ) - dup still-parsing? [ - dup skip-blank - dup still-parsing-line? - [ (parse-token) ] [ dup next-line parse-token ] if - ] [ drop f ] if ; + dup parse-raw [ skip-comments ] [ drop f ] if* ; : ?scan-token ( -- str/f ) lexer get parse-token ; diff --git a/core/syntax/syntax.factor b/core/syntax/syntax.factor index 78b41e2116..13f07fcddc 100644 --- a/core/syntax/syntax.factor +++ b/core/syntax/syntax.factor @@ -81,7 +81,7 @@ IN: bootstrap.syntax "f" [ f suffix! ] define-core-syntax "CHAR:" [ - scan-token { + lexer get parse-raw [ "token" throw-unexpected-eof ] unless* { { [ dup length 1 = ] [ first ] } { [ "\\" ?head ] [ next-escape >string "" assert= ] } [ name>char-hook get call( name -- char ) ]