diff --git a/extra/peg-lexer/peg-lexer-docs.factor b/extra/peg-lexer/peg-lexer-docs.factor index 7b45ffb87d..22e620235d 100644 --- a/extra/peg-lexer/peg-lexer-docs.factor +++ b/extra/peg-lexer/peg-lexer-docs.factor @@ -8,4 +8,7 @@ HELP: ON-BNF: HELP: create-bnf { $values { "word" string } { "parser" parser } } -{ $description "Runtime equivalent of ON-BNF- also useful with manually constructed parsers." } ; \ No newline at end of file +{ $description "Runtime equivalent of " { $link POSTPONE: ON-BNF: } " also useful with manually constructed parsers." } ; + +HELP: factor +{ $description "Tokenizer that acts like standard factor lexer, separating tokens by whitespace." } ; \ No newline at end of file diff --git a/extra/peg-lexer/peg-lexer.factor b/extra/peg-lexer/peg-lexer.factor index bc191fd878..f4bed6c5ef 100644 --- a/extra/peg-lexer/peg-lexer.factor +++ b/extra/peg-lexer/peg-lexer.factor @@ -30,15 +30,11 @@ M: lex-hash at* swap { H{ } clone \ packrat set ] f make-assoc swap bind ; inline -! Usage: -! ON-BNF: word expr= [1-9] ;ON-BNF -! << name parser create-bnf >> - : parse* ( parser -- ast ) compile [ execute [ error-stack get first throw ] unless* ] with-global-lexer ast>> ; -: create-bnf ( name parser -- ) reset-tokenizer [ lexer get skip-blank parse* dup V{ } = [ parsed ] unless ] curry +: create-bnf ( name parser -- ) reset-tokenizer [ lexer get skip-blank parse* parsed ] curry define word make-parsing ; : ON-BNF: CREATE-WORD reset-tokenizer ";ON-BNF" parse-multiline-string parse-ebnf diff --git a/extra/peg-lexer/test-parsers/test-parsers.factor b/extra/peg-lexer/test-parsers/test-parsers.factor new file mode 100644 index 0000000000..83c9f85933 --- /dev/null +++ b/extra/peg-lexer/test-parsers/test-parsers.factor @@ -0,0 +1,17 @@ +USING: peg-lexer math.parser strings ; +IN: peg-lexer.test-parsers + +ON-BNF: test1 + num = [1-4]* => [[ >string ]] + expr = num ( "-end" | "-done" ) +;ON-BNF + +ON-BNF: test2 + num = [1-4]* => [[ >string string>number ]] + expr= num [5-9] +;ON-BNF + +ON-BNF: test3 + tokenizer = + expr= "heavy" "duty" "testing" +;ON-BNF \ No newline at end of file