From a282a8dd2689d6bfb18728d198452d2ba79cf7ab Mon Sep 17 00:00:00 2001
From: Joe Groff <arcata@gmail.com>
Date: Mon, 1 Mar 2010 11:47:57 -0800
Subject: [PATCH] lexer, parser: update docs on "parse-tokens" and add docs for
 "each-token", "map-tokens"

---
 core/lexer/lexer-docs.factor   | 13 ++++++++++++-
 core/parser/parser-docs.factor | 10 +++++++---
 2 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/core/lexer/lexer-docs.factor b/core/lexer/lexer-docs.factor
index 30888b76d8..b9af0dc65c 100644
--- a/core/lexer/lexer-docs.factor
+++ b/core/lexer/lexer-docs.factor
@@ -66,10 +66,21 @@ HELP: still-parsing?
 { $values { "lexer" lexer } { "?" "a boolean" } }
 { $description "Outputs " { $link f } " if end of input has been reached, " { $link t } " otherwise." } ;
 
+HELP: each-token
+{ $values { "end" string } { "quot" { $quotation "( token -- )" } } }
+{ $description "Reads a sequence of tokens until the first occurrence of " { $snippet "end" } ". " { $snippet "quot" } " is called on each token as it is read." }
+{ $examples "This word is used to implement " { $link POSTPONE: USING: } "." }
+$parsing-note ;
+
+HELP: map-tokens
+{ $values { "end" string } { "quot" { $quotation "( token -- object )" } } { "seq" "a new sequence of " { $snippet "object" } "s" } }
+{ $description "Reads a sequence of tokens until the first occurrence of " { $snippet "end" } ". " { $snippet "quot" } " is called on each token as it is read, and the results are collected into a new output sequence." }
+$parsing-note ;
+
 HELP: parse-tokens
 { $values { "end" string } { "seq" "a new sequence of strings" } }
 { $description "Reads a sequence of tokens until the first occurrence of " { $snippet "end" } ". The tokens remain as strings and are not processed in any way." }
-{ $examples "This word is used to implement " { $link POSTPONE: USING: } "." }
+{ $notes "This word is equivalent to " { $link map-tokens } " with an empty quotation." }
 $parsing-note ;
 
 HELP: unexpected
diff --git a/core/parser/parser-docs.factor b/core/parser/parser-docs.factor
index b024d1d968..c04a0f568e 100644
--- a/core/parser/parser-docs.factor
+++ b/core/parser/parser-docs.factor
@@ -52,8 +52,12 @@ ARTICLE: "parsing-tokens" "Parsing raw tokens"
 $nl
 "One example is the " { $link POSTPONE: USING: } " parsing word."
 { $see POSTPONE: USING: } 
-"It reads a list of vocabularies terminated by " { $link POSTPONE: ; } ". However, the vocabulary names do not name words, except by coincidence; so " { $link parse-until } " cannot be used here. Instead, a lower-level word is called:"
-{ $subsections parse-tokens } ;
+"It reads a list of vocabularies terminated by " { $link POSTPONE: ; } ". However, the vocabulary names do not name words, except by coincidence; so " { $link parse-until } " cannot be used here. Instead, a set of lower-level combinators can be used:"
+{ $subsections
+    each-token
+    map-tokens
+    parse-tokens
+} ;
 
 ARTICLE: "parsing-words" "Parsing words"
 "The Factor parser follows a simple recursive-descent design. The parser reads successive tokens from the input; if the token identifies a number or an ordinary word, it is added to an accumulator vector. Otherwise if the token identifies a parsing word, the parsing word is executed immediately."
@@ -164,7 +168,7 @@ HELP: parse-until
 { $examples "This word is used to implement " { $link POSTPONE: ARTICLE: } "." }
 $parsing-note ;
 
-{ parse-tokens (parse-until) parse-until } related-words
+{ parse-tokens each-token map-tokens (parse-until) parse-until } related-words
 
 HELP: (parse-lines)
 { $values { "lexer" lexer } { "quot" "a new " { $link quotation } } }