peg.ebnf: support escaped double quote in a string

db4
Jon Harper 2015-08-24 23:41:02 +02:00
parent b2a51dfeb2
commit 03630e8100
3 changed files with 37 additions and 3 deletions

View File

@ -60,6 +60,9 @@ HELP: EBNF:
ARTICLE: "peg.ebnf.strings" "Strings" ARTICLE: "peg.ebnf.strings" "Strings"
"A string in a rule will match that sequence of characters from the input string. " "A string in a rule will match that sequence of characters from the input string. "
"The string is delimited by matching single or double quotes. "
"Factor's escape sequences are interpreted: " { $link "escape" } ". "
"For double quotes delimiters, an escaped double quote doesn't terminate the string. "
"The AST result from the match is the string itself." "The AST result from the match is the string itself."
{ $examples { $examples
{ $example { $example
@ -67,6 +70,21 @@ ARTICLE: "peg.ebnf.strings" "Strings"
"\"helloworld\" [EBNF rule=\"hello\" \"world\" EBNF] ." "\"helloworld\" [EBNF rule=\"hello\" \"world\" EBNF] ."
"V{ \"hello\" \"world\" }" "V{ \"hello\" \"world\" }"
} }
{ $example
"USING: prettyprint peg.ebnf ;"
"\"AΣ𝄞\" [EBNF rule='\\x41' '\\u{greek-capital-letter-sigma}' '\\u01D11E' EBNF] ."
"V{ \"A\" \"Σ\" \"𝄞\" }"
}
{ $example
"USING: io peg.ebnf ;"
"\"A double quote: \\\"\" [EBNF rule='A double quote: \"' EBNF] print"
"A double quote: \""
}
{ $example
"USING: io peg.ebnf ;"
"\"' and \\\"\" [EBNF rule=\"' and \\\"\" EBNF] print"
"' and \""
}
} ; } ;
ARTICLE: "peg.ebnf.any" "Any" ARTICLE: "peg.ebnf.any" "Any"

View File

@ -124,6 +124,18 @@ IN: peg.ebnf.tests
"'foo'" identifier-parser parse "'foo'" identifier-parser parse
] unit-test ] unit-test
{ "\"" } [
"\"\\\"\"" identifier-parser parse
] unit-test
{ "\\" } [
"\"\\\\\"" identifier-parser parse
] unit-test
{ "AΣ𝄞" } [
"'\\x41\\u{greek-capital-letter-sigma}\\u01D11E'" identifier-parser parse
] unit-test
{ "foo" } [ { "foo" } [
"foo" non-terminal-parser parse symbol>> "foo" non-terminal-parser parse symbol>>
] unit-test ] unit-test

View File

@ -3,7 +3,7 @@
USING: accessors arrays assocs combinators USING: accessors arrays assocs combinators
combinators.short-circuit effects io.streams.string kernel make combinators.short-circuit effects io.streams.string kernel make
math.parser multiline namespaces parser peg peg.parsers math.parser multiline namespaces parser peg peg.parsers
peg.search quotations sequences splitting stack-checker strings peg.search quotations sequences sequences.deep splitting stack-checker strings
strings.parser summary unicode.categories words ; strings.parser summary unicode.categories words ;
FROM: vocabs.parser => search ; FROM: vocabs.parser => search ;
FROM: peg.search => replace ; FROM: peg.search => replace ;
@ -120,9 +120,13 @@ C: <ebnf> ebnf
#! or double quotes. The AST produced is the identifier #! or double quotes. The AST produced is the identifier
#! between the quotes. #! between the quotes.
[ [
[ CHAR: " = not ] satisfy repeat1 "\"" "\"" surrounded-by , [
[ CHAR: \ = ] satisfy
[ [ CHAR: " = ] [ CHAR: \ = ] bi or ] satisfy 2seq ,
[ CHAR: " = not ] satisfy ,
] choice* repeat1 "\"" "\"" surrounded-by ,
[ CHAR: ' = not ] satisfy repeat1 "'" "'" surrounded-by , [ CHAR: ' = not ] satisfy repeat1 "'" "'" surrounded-by ,
] choice* [ >string unescape-string ] action ; ] choice* [ flatten >string unescape-string ] action ;
: non-terminal-parser ( -- parser ) : non-terminal-parser ( -- parser )
#! A non-terminal is the name of another rule. It can #! A non-terminal is the name of another rule. It can