lexer/parser: rename (scan-token) to ?scan-token, (scan-datum) to ?scan-datum.

db4
John Benediktsson 2014-05-20 09:13:58 -07:00
parent b244b3f731
commit a38524eb99
4 changed files with 18 additions and 18 deletions

View File

@ -57,14 +57,14 @@ HELP: parse-token
{ $values { "lexer" lexer } { "str/f" { $maybe string } } } { $values { "lexer" lexer } { "str/f" { $maybe string } } }
{ $description "Reads the next token from the lexer. Tokens are delimited by whitespace, with the exception that " { $snippet "\"" } " is treated like a single token even when not followed by whitespace." } ; { $description "Reads the next token from the lexer. Tokens are delimited by whitespace, with the exception that " { $snippet "\"" } " is treated like a single token even when not followed by whitespace." } ;
HELP: (scan-token) HELP: ?scan-token
{ $values { "str/f" { $maybe string } } } { $values { "str/f" { $maybe string } } }
{ $description "Reads the next token from the lexer. Tokens are delimited by whitespace, with the exception that " { $snippet "\"" } " is treated like a single token even when not followed by whitespace. This word outputs " { $link f } " on end of input. To throw an error on end of input, use " { $link scan-token } " instead." } { $description "Reads the next token from the lexer. Tokens are delimited by whitespace, with the exception that " { $snippet "\"" } " is treated like a single token even when not followed by whitespace. This word outputs " { $link f } " on end of input. To throw an error on end of input, use " { $link scan-token } " instead." }
$parsing-note ; $parsing-note ;
HELP: scan-token HELP: scan-token
{ $values { "str" string } } { $values { "str" string } }
{ $description "Reads the next token from the lexer. Tokens are delimited by whitespace, with the exception that " { $snippet "\"" } " is treated like a single token even when not followed by whitespace. This word throws " { $link unexpected-eof } " on end of input. To output " { $link f } " on end of input, use " { $link (scan-token) } " instead." } { $description "Reads the next token from the lexer. Tokens are delimited by whitespace, with the exception that " { $snippet "\"" } " is treated like a single token even when not followed by whitespace. This word throws " { $link unexpected-eof } " on end of input. To output " { $link f } " on end of input, use " { $link ?scan-token } " instead." }
$parsing-note ; $parsing-note ;
HELP: still-parsing? HELP: still-parsing?

View File

@ -117,14 +117,14 @@ M: lexer skip-word
[ (parse-token) ] [ dup next-line parse-token ] if [ (parse-token) ] [ dup next-line parse-token ] if
] [ drop f ] if ; ] [ drop f ] if ;
: (scan-token) ( -- str/f ) lexer get parse-token ; : ?scan-token ( -- str/f ) lexer get parse-token ;
PREDICATE: unexpected-eof < unexpected got>> not ; PREDICATE: unexpected-eof < unexpected got>> not ;
: throw-unexpected-eof ( word -- * ) f unexpected ; : throw-unexpected-eof ( word -- * ) f unexpected ;
: scan-token ( -- str ) : scan-token ( -- str )
(scan-token) [ "token" throw-unexpected-eof ] unless* ; ?scan-token [ "token" throw-unexpected-eof ] unless* ;
: expect ( token -- ) : expect ( token -- )
scan-token 2dup = [ 2drop ] [ unexpected ] if ; scan-token 2dup = [ 2drop ] [ unexpected ] if ;

View File

@ -18,8 +18,8 @@ ARTICLE: "reading-ahead" "Reading ahead"
} }
"Parsing words that return " { $link f } " on end of file:" "Parsing words that return " { $link f } " on end of file:"
{ $subsections { $subsections
(scan-token) ?scan-token
(scan-datum) ?scan-datum
} }
"A simple example is the " { $link POSTPONE: \ } " word:" "A simple example is the " { $link POSTPONE: \ } " word:"
{ $see POSTPONE: \ } ; { $see POSTPONE: \ } ;
@ -160,7 +160,7 @@ HELP: parse-datum
{ $values { "string" string } { "word/number" "a word or number" } } { $values { "string" string } { "word/number" "a word or number" } }
{ $description "If " { $snippet "string" } " is a valid number literal, it is converted to a number, otherwise the current vocabulary search path is searched for a word named by the string." } { $description "If " { $snippet "string" } " is a valid number literal, it is converted to a number, otherwise the current vocabulary search path is searched for a word named by the string." }
{ $errors "Throws an error if the token does not name a word, and does not parse as a number." } { $errors "Throws an error if the token does not name a word, and does not parse as a number." }
{ $notes "This word is used to implement " { $link (scan-datum) } " and " { $link scan-datum } "." } ; { $notes "This word is used to implement " { $link ?scan-datum } " and " { $link scan-datum } "." } ;
HELP: scan-word HELP: scan-word
{ $values { "word" "a word" } } { $values { "word" "a word" } }
@ -176,25 +176,25 @@ HELP: scan-word-name
{ $errors "Throws an error if the scanned token is a number or upon finding end of file." } { $errors "Throws an error if the scanned token is a number or upon finding end of file." }
$parsing-note ; $parsing-note ;
HELP: (scan-datum) HELP: ?scan-datum
{ $values { "word/number/f" "a word, a number, or " { $link f } } } { $values { "word/number/f" { $maybe word number } } }
{ $description "Reads the next token from parser input. If the token is found in the vocabulary search path, returns the word named by the token. If the token does not find a word, it is next converted to a number. If this conversion fails, too, this word returns " { $link f } "." } { $description "Reads the next token from parser input. If the token is found in the vocabulary search path, returns the word named by the token. If the token does not find a word, it is next converted to a number. If this conversion fails, too, this word returns " { $link f } "." }
$parsing-note ; $parsing-note ;
HELP: scan-datum HELP: scan-datum
{ $values { "word/number" "a word or a number" } } { $values { "word/number" { $or word number } } }
{ $description "Reads the next token from parser input. If the token is found in the vocabulary search path, returns the word named be the token. If the token is not found in the vocabulary search path, it is converted to a number. If this conversion fails, an error is thrown." } { $description "Reads the next token from parser input. If the token is found in the vocabulary search path, returns the word named be the token. If the token is not found in the vocabulary search path, it is converted to a number. If this conversion fails, an error is thrown." }
{ $errors "Throws an error if the token is not a number or end of file is reached." } { $errors "Throws an error if the token is not a number or end of file is reached." }
$parsing-note ; $parsing-note ;
HELP: scan-number HELP: scan-number
{ $values { "number" "a number" } } { $values { "number" number } }
{ $description "Reads the next token from parser input. If the token is a number literal, it is converted to a number. Otherwise, it throws an error." } { $description "Reads the next token from parser input. If the token is a number literal, it is converted to a number. Otherwise, it throws an error." }
{ $errors "Throws an error if the token is not a number or end of file is reached." } { $errors "Throws an error if the token is not a number or end of file is reached." }
$parsing-note ; $parsing-note ;
HELP: parse-until-step HELP: parse-until-step
{ $values { "accum" vector } { "end" word } { "?" "a boolean" } } { $values { "accum" vector } { "end" word } { "?" boolean } }
{ $description "Parses a token. If the token is a number or an ordinary word, it is added to the accumulator. If it is a parsing word, calls the parsing word with the accumulator on the stack. Outputs " { $link f } " if " { $snippet "end" } " is encountered, " { $link t } " otherwise." } { $description "Parses a token. If the token is a number or an ordinary word, it is added to the accumulator. If it is a parsing word, calls the parsing word with the accumulator on the stack. Outputs " { $link f } " if " { $snippet "end" } " is encountered, " { $link t } " otherwise." }
$parsing-note ; $parsing-note ;

View File

@ -59,17 +59,17 @@ ERROR: number-expected ;
dup string>number [ ] [ no-word ] ?if dup string>number [ ] [ no-word ] ?if
] ?if ; ] ?if ;
: (scan-datum) ( -- word/number/f ) : ?scan-datum ( -- word/number/f )
(scan-token) dup [ parse-datum ] when ; ?scan-token dup [ parse-datum ] when ;
: scan-datum ( -- word/number ) : scan-datum ( -- word/number )
(scan-datum) [ \ word throw-unexpected-eof ] unless* ; ?scan-datum [ \ word throw-unexpected-eof ] unless* ;
: scan-word ( -- word ) : scan-word ( -- word )
(scan-token) parse-word ; ?scan-token parse-word ;
: scan-number ( -- number ) : scan-number ( -- number )
(scan-token) parse-number ; ?scan-token parse-number ;
ERROR: invalid-word-name string ; ERROR: invalid-word-name string ;
@ -105,7 +105,7 @@ ERROR: staging-violation word ;
scan-object \ f or ; scan-object \ f or ;
: parse-until-step ( accum end -- accum ? ) : parse-until-step ( accum end -- accum ? )
(scan-datum) { ?scan-datum {
{ [ 2dup eq? ] [ 2drop f ] } { [ 2dup eq? ] [ 2drop f ] }
{ [ dup not ] [ drop throw-unexpected-eof t ] } { [ dup not ] [ drop throw-unexpected-eof t ] }
{ [ dup delimiter? ] [ unexpected t ] } { [ dup delimiter? ] [ unexpected t ] }