Merge commit 'doublec/master'

db4
Slava Pestov 2007-12-20 00:43:16 -05:00
commit f2b4a04a44
17 changed files with 258 additions and 228 deletions

View File

@ -1,54 +1,54 @@
! Copyright (C) 2006 Chris Double. All Rights Reserved.
! See http://factorcode.org/license.txt for BSD license.
USING: kernel tools.test parser-combinators lazy-lists fjsc ;
USING: kernel tools.test peg fjsc ;
IN: temporary
{ T{ ast-expression f { T{ ast-number f 55 } T{ ast-identifier f "2abc1" } T{ ast-number f 100 } } } } [
"55 2abc1 100" 'expression' parse-1
{ T{ ast-expression f V{ T{ ast-number f 55 } T{ ast-identifier f "2abc1" } T{ ast-number f 100 } } } } [
"55 2abc1 100" 'expression' parse parse-result-ast
] unit-test
{ T{ ast-quotation f { T{ ast-number f 55 } T{ ast-identifier f "2abc1" } T{ ast-number f 100 } } } } [
"[ 55 2abc1 100 ]" 'quotation' parse-1
{ T{ ast-quotation f V{ T{ ast-number f 55 } T{ ast-identifier f "2abc1" } T{ ast-number f 100 } } } } [
"[ 55 2abc1 100 ]" 'quotation' parse parse-result-ast
] unit-test
{ T{ ast-array f { T{ ast-number f 55 } T{ ast-identifier f "2abc1" } T{ ast-number f 100 } } } } [
"{ 55 2abc1 100 }" 'array' parse-1
{ T{ ast-array f V{ T{ ast-number f 55 } T{ ast-identifier f "2abc1" } T{ ast-number f 100 } } } } [
"{ 55 2abc1 100 }" 'array' parse parse-result-ast
] unit-test
{ T{ ast-stack-effect f { } { "d" "e" "f" } } } [
"( -- d e f )" 'stack-effect' parse-1
{ T{ ast-stack-effect f V{ } V{ "d" "e" "f" } } } [
"( -- d e f )" 'stack-effect' parse parse-result-ast
] unit-test
{ T{ ast-stack-effect f { "a" "b" "c" } { "d" "e" "f" } } } [
"( a b c -- d e f )" 'stack-effect' parse-1
{ T{ ast-stack-effect f V{ "a" "b" "c" } V{ "d" "e" "f" } } } [
"( a b c -- d e f )" 'stack-effect' parse parse-result-ast
] unit-test
{ T{ ast-stack-effect f { "a" "b" "c" } { } } } [
"( a b c -- )" 'stack-effect' parse-1
{ T{ ast-stack-effect f V{ "a" "b" "c" } V{ } } } [
"( a b c -- )" 'stack-effect' parse parse-result-ast
] unit-test
{ T{ ast-stack-effect f { } { } } } [
"( -- )" 'stack-effect' parse-1
{ T{ ast-stack-effect f V{ } V{ } } } [
"( -- )" 'stack-effect' parse parse-result-ast
] unit-test
{ } [
": foo ( a b -- c d ) abcdefghijklmn 123 ;" 'expression' parse car drop
{ f } [
": foo ( a b -- c d ) abcdefghijklmn 123 ;" 'expression' parse not
] unit-test
{ T{ ast-expression f { T{ ast-string f "abcd" } } } } [
"\"abcd\"" 'statement' parse-1
{ T{ ast-expression f V{ T{ ast-string f "abcd" } } } } [
"\"abcd\"" 'statement' parse parse-result-ast
] unit-test
{ T{ ast-expression f { T{ ast-use f "foo" } } } } [
"USE: foo" 'statement' parse-1
{ T{ ast-expression f V{ T{ ast-use f "foo" } } } } [
"USE: foo" 'statement' parse parse-result-ast
] unit-test
{ T{ ast-expression f { T{ ast-in f "foo" } } } } [
"IN: foo" 'statement' parse-1
{ T{ ast-expression f V{ T{ ast-in f "foo" } } } } [
"IN: foo" 'statement' parse parse-result-ast
] unit-test
{ T{ ast-expression f { T{ ast-using f { "foo" "bar" } } } } } [
"USING: foo bar ;" 'statement' parse-1
{ T{ ast-expression f V{ T{ ast-using f V{ "foo" "bar" } } } } } [
"USING: foo bar ;" 'statement' parse parse-result-ast
] unit-test

View File

@ -1,50 +1,38 @@
! Copyright (C) 2006 Chris Double. All Rights Reserved.
! See http://factorcode.org/license.txt for BSD license.
USING: kernel lazy-lists parser-combinators parser-combinators.simple
strings promises sequences math math.parser namespaces words
quotations arrays hashtables io io.streams.string assocs ;
USING: kernel peg strings promises sequences math math.parser
namespaces words quotations arrays hashtables io
io.streams.string assocs memoize ;
IN: fjsc
TUPLE: ast-number value ;
C: <ast-number> ast-number
TUPLE: ast-identifier value vocab ;
C: <ast-identifier> ast-identifier
TUPLE: ast-string value ;
C: <ast-string> ast-string
TUPLE: ast-quotation values ;
C: <ast-quotation> ast-quotation
TUPLE: ast-array elements ;
C: <ast-array> ast-array
TUPLE: ast-define name stack-effect expression ;
C: <ast-define> ast-define
TUPLE: ast-expression values ;
C: <ast-expression> ast-expression
TUPLE: ast-word value vocab ;
C: <ast-word> ast-word
TUPLE: ast-comment ;
C: <ast-comment> ast-comment
TUPLE: ast-stack-effect in out ;
C: <ast-stack-effect> ast-stack-effect
TUPLE: ast-use name ;
C: <ast-use> ast-use
TUPLE: ast-using names ;
C: <ast-using> ast-using
TUPLE: ast-in name ;
C: <ast-in> ast-in
TUPLE: ast-hashtable elements ;
C: <ast-number> ast-number
C: <ast-identifier> ast-identifier
C: <ast-string> ast-string
C: <ast-quotation> ast-quotation
C: <ast-array> ast-array
C: <ast-define> ast-define
C: <ast-expression> ast-expression
C: <ast-word> ast-word
C: <ast-comment> ast-comment
C: <ast-stack-effect> ast-stack-effect
C: <ast-use> ast-use
C: <ast-using> ast-using
C: <ast-in> ast-in
C: <ast-hashtable> ast-hashtable
: identifier-middle? ( ch -- bool )
@ -56,7 +44,7 @@ C: <ast-hashtable> ast-hashtable
digit? not
and and and and and ;
LAZY: 'identifier-ends' ( -- parser )
MEMO: 'identifier-ends' ( -- parser )
[
[ blank? not ] keep
[ CHAR: " = not ] keep
@ -65,99 +53,137 @@ LAZY: 'identifier-ends' ( -- parser )
[ letter? not ] keep
identifier-middle? not
and and and and and
] satisfy <!*> ;
] satisfy repeat0 ;
LAZY: 'identifier-middle' ( -- parser )
[ identifier-middle? ] satisfy <!+> ;
MEMO: 'identifier-middle' ( -- parser )
[ identifier-middle? ] satisfy repeat1 ;
LAZY: 'identifier' ( -- parser )
'identifier-ends'
'identifier-middle' <&>
'identifier-ends' <:&>
[ concat >string f <ast-identifier> ] <@ ;
MEMO: 'identifier' ( -- parser )
[
'identifier-ends' ,
'identifier-middle' ,
'identifier-ends' ,
] { } make seq [
concat >string f <ast-identifier>
] action ;
DEFER: 'expression'
LAZY: 'effect-name' ( -- parser )
MEMO: 'effect-name' ( -- parser )
[
[ blank? not ] keep
[ CHAR: ) = not ] keep
CHAR: - = not
and
] satisfy <!+> [ >string ] <@ ;
and and
] satisfy repeat1 [ >string ] action ;
LAZY: 'stack-effect' ( -- parser )
"(" token sp
'effect-name' sp <*> &>
"--" token sp <&
'effect-name' sp <*> <&>
")" token sp <& [ first2 <ast-stack-effect> ] <@ ;
MEMO: 'stack-effect' ( -- parser )
[
"(" token hide ,
'effect-name' sp repeat0 ,
"--" token sp hide ,
'effect-name' sp repeat0 ,
")" token sp hide ,
] { } make seq [
first2 <ast-stack-effect>
] action ;
LAZY: 'define' ( -- parser )
":" token sp
'identifier' sp [ ast-identifier-value ] <@ &>
'stack-effect' sp <!?> <&>
'expression' <:&>
";" token sp <& [ first3 <ast-define> ] <@ ;
MEMO: 'define' ( -- parser )
[
":" token sp hide ,
'identifier' sp [ ast-identifier-value ] action ,
'stack-effect' sp optional ,
'expression' ,
";" token sp hide ,
] { } make seq [ first3 <ast-define> ] action ;
LAZY: 'quotation' ( -- parser )
"[" token sp
'expression' [ ast-expression-values ] <@ &>
"]" token sp <& [ <ast-quotation> ] <@ ;
MEMO: 'quotation' ( -- parser )
[
"[" token sp hide ,
'expression' [ ast-expression-values ] action ,
"]" token sp hide ,
] { } make seq [ first <ast-quotation> ] action ;
LAZY: 'array' ( -- parser )
"{" token sp
'expression' [ ast-expression-values ] <@ &>
"}" token sp <& [ <ast-array> ] <@ ;
MEMO: 'array' ( -- parser )
[
"{" token sp hide ,
'expression' [ ast-expression-values ] action ,
"}" token sp hide ,
] { } make seq [ first <ast-array> ] action ;
LAZY: 'word' ( -- parser )
"\\" token sp
'identifier' sp &> [ ast-identifier-value f <ast-word> ] <@ ;
MEMO: 'word' ( -- parser )
[
"\\" token sp hide ,
'identifier' sp ,
] { } make seq [ first ast-identifier-value f <ast-word> ] action ;
LAZY: 'atom' ( -- parser )
'identifier' 'integer' [ <ast-number> ] <@ <|> 'string' [ <ast-string> ] <@ <|> ;
MEMO: 'atom' ( -- parser )
[
'identifier' ,
'integer' [ <ast-number> ] action ,
'string' [ <ast-string> ] action ,
] { } make choice ;
LAZY: 'comment' ( -- parser )
"#!" token sp
"!" token sp <|> [
dup CHAR: \n = swap CHAR: \r = or not
] satisfy <*> <&> [ drop <ast-comment> ] <@ ;
MEMO: 'comment' ( -- parser )
[
[
"#!" token sp ,
"!" token sp ,
] { } make choice hide ,
[
dup CHAR: \n = swap CHAR: \r = or not
] satisfy repeat0 ,
] { } make seq [ drop <ast-comment> ] action ;
LAZY: 'USE:' ( -- parser )
"USE:" token sp
'identifier' sp &> [ ast-identifier-value <ast-use> ] <@ ;
MEMO: 'USE:' ( -- parser )
[
"USE:" token sp hide ,
'identifier' sp ,
] { } make seq [ first ast-identifier-value <ast-use> ] action ;
LAZY: 'IN:' ( -- parser )
"IN:" token sp
'identifier' sp &> [ ast-identifier-value <ast-in> ] <@ ;
MEMO: 'IN:' ( -- parser )
[
"IN:" token sp hide ,
'identifier' sp ,
] { } make seq [ first ast-identifier-value <ast-in> ] action ;
LAZY: 'USING:' ( -- parser )
"USING:" token sp
'identifier' sp [ ast-identifier-value ] <@ <+> &>
";" token sp <& [ <ast-using> ] <@ ;
MEMO: 'USING:' ( -- parser )
[
"USING:" token sp hide ,
'identifier' sp [ ast-identifier-value ] action repeat1 ,
";" token sp hide ,
] { } make seq [ first <ast-using> ] action ;
LAZY: 'hashtable' ( -- parser )
"H{" token sp
'expression' [ ast-expression-values ] <@ &>
"}" token sp <& [ <ast-hashtable> ] <@ ;
MEMO: 'hashtable' ( -- parser )
[
"H{" token sp hide ,
'expression' [ ast-expression-values ] action ,
"}" token sp hide ,
] { } make seq [ first <ast-hashtable> ] action ;
LAZY: 'parsing-word' ( -- parser )
'USE:'
'USING:' <|>
'IN:' <|> ;
MEMO: 'parsing-word' ( -- parser )
[
'USE:' ,
'USING:' ,
'IN:' ,
] { } make choice ;
LAZY: 'expression' ( -- parser )
'comment'
'parsing-word' sp <|>
'quotation' sp <|>
'define' sp <|>
'array' sp <|>
'hashtable' sp <|>
'word' sp <|>
'atom' sp <|>
<*> [ <ast-expression> ] <@ ;
MEMO: 'expression' ( -- parser )
[
[
'comment' ,
'parsing-word' sp ,
'quotation' sp ,
'define' sp ,
'array' sp ,
'hashtable' sp ,
'word' sp ,
'atom' sp ,
] { } make choice repeat0 [ <ast-expression> ] action
] delay ;
LAZY: 'statement' ( -- parser )
MEMO: 'statement' ( -- parser )
'expression' ;
GENERIC: (compile) ( ast -- )
@ -328,7 +354,7 @@ M: wrapper (parse-factor-quotation) ( object -- ast )
GENERIC: fjsc-parse ( object -- ast )
M: string fjsc-parse ( object -- ast )
'expression' parse-1 ;
'expression' parse parse-result-ast ;
M: quotation fjsc-parse ( object -- ast )
[
@ -345,11 +371,11 @@ M: quotation fjsc-parse ( object -- ast )
] string-out ;
: fjsc-compile* ( string -- string )
'statement' parse-1 fjsc-compile ;
'statement' parse parse-result-ast fjsc-compile ;
: fc* ( string -- string )
[
'statement' parse-1 ast-expression-values do-expressions
'statement' parse parse-result-ast ast-expression-values do-expressions
] { } make [ write ] each ;

View File

@ -3,10 +3,6 @@ USE: kernel-internals
: bind ( ns quot -- )
swap >n call n> drop ;
: alert ( string -- )
#! Display the string in an alert box
window { } "" "alert" { "string" } alien-invoke ;
"browser-dom" set-in
: elements ( string -- result )
@ -38,3 +34,6 @@ USE: kernel-internals
drop "Click done!" alert
] callcc0 ;
: alert ( string -- )
#! Display the string in an alert box
window { } "" "alert" { "string" } alien-invoke ;

View File

@ -513,6 +513,12 @@ factor.add_word("alien", "set-alien-property", "primitive", function(next) {
factor.call_next(next);
});
factor.add_word("alien", "uneval", "primitive", function(next) {
var stack = factor.cont.data_stack;
stack.push(uneval(stack.pop()));
factor.call_next(next);
});
factor.add_word("words", "vocabs", "primitive", function(next) {
var stack = factor.cont.data_stack;
var result = [];

View File

@ -1,76 +0,0 @@
! Copyright (C) 2006 Chris Double.
! See http://factorcode.org/license.txt for BSD license.
USING: help.syntax help.markup parser-combinators
parser-combinators.replace ;
HELP: tree-write
{ $values
{ "object" "an object" } }
{ $description
"Write the object to the standard output stream, unless "
"it is an array, in which case recurse through the array "
"writing each object to the stream." }
{ $example "USE: parser-combinators" "{ 65 \"bc\" { 68 \"ef\" } } tree-write" "AbcDef" } ;
HELP: search
{ $values
{ "string" "a string" }
{ "parser" "a parser combinator based parser" }
{ "seq" "a sequence" }
}
{ $description
"Returns a sequence containing the parse results of all substrings "
"from the input string that successfully parse using the "
"parser."
}
{ $example "USE: parser-combinators" "\"one 123 two 456\" 'integer' search ." "{ 123 456 }" }
{ $example "USE: parser-combinators" "\"one 123 \\\"hello\\\" two 456\" 'integer' 'string' <|> search ." "{ 123 \"hello\" 456 }" }
{ $see-also search* replace replace* } ;
HELP: search*
{ $values
{ "string" "a string" }
{ "parsers" "a sequence of parser combinator based parsers" }
{ "seq" "a sequence" }
}
{ $description
"Returns a sequence containing the parse results of all substrings "
"from the input string that successfully parse using any of the "
"parsers in the 'parsers' sequence."
}
{ $example "USE: parser-combinators" "\"one 123 \\\"hello\\\" two 456\" 'integer' 'string' 2array search* ." "{ 123 \"hello\" 456 }" }
{ $see-also search replace replace* } ;
HELP: replace
{ $values
{ "string" "a string" }
{ "parser" "a parser combinator based parser" }
{ "result" "a string" }
}
{ $description
"Returns a copy of the original string but with all substrings that "
"successfully parse with the given parser replaced with "
"the result of that parser."
}
{ $example "USING: parser-combinators math.parser ;" "\"one 123 two 456\" 'integer' [ 2 * number>string ] <@ replace ." "\"one 246 two 912\"" }
{ $example "USE: parser-combinators" "\"hello *world* from *factor*\" 'bold' [ \"<strong>\" swap \"</strong>\" 3append ] <@ replace ." "\"hello <strong>world</strong> from <strong>factor</strong>\"" }
{ $example "USE: parser-combinators" "\"hello *world* from _factor_\"\n 'bold' [ \"<strong>\" swap \"</strong>\" 3append ] <@\n 'italic' [ \"<emphasis>\" swap \"</emphasis>\" 3append ] <@ <|>\n replace ." "\"hello <strong>world</strong> from <emphasis>factor</emphasis>\"" }
{ $see-also search search* replace* } ;
HELP: replace*
{ $values
{ "string" "a string" }
{ "parsers" "a sequence of parser combinator based parsers" }
{ "result" "a string" }
}
{ $description
"Returns a copy of the original string but with all substrings that "
"successfully parse with the given parsers replaced with "
"the result of that parser. Each parser is done in sequence so that "
"the parse results of the first parser can be replaced by later parsers."
}
{ $example "USE: parser-combinators" "\"*hello _world_*\"\n 'bold' [ \"<strong>\" swap \"</strong>\" 3append ] <@\n 'italic' [ \"<emphasis>\" swap \"</emphasis>\" 3append ] <@ 2array\n replace* ." "\"<strong>hello <emphasis>world</emphasis></strong>\"" }
{ $see-also search search* replace* } ;

View File

@ -1,8 +1,8 @@
! Copyright (C) 2006 Chris Double.
! See http://factorcode.org/license.txt for BSD license.
USING: kernel strings math sequences lazy-lists words
math.parser promises ;
IN: parser-combinators
math.parser promises parser-combinators ;
IN: parser-combinators.simple
: 'digit' ( -- parser )
[ digit? ] satisfy [ digit> ] <@ ;

1
extra/peg/ebnf/tags.txt Normal file
View File

@ -0,0 +1 @@
parsing

View File

@ -1,7 +1,7 @@
! Copyright (C) 2007 Chris Double.
! See http://factorcode.org/license.txt for BSD license.
USING: kernel sequences strings namespaces math assocs shuffle
vectors arrays combinators.lib memoize ;
vectors arrays combinators.lib memoize math.parser ;
IN: peg
TUPLE: parse-result remaining ast ;
@ -265,3 +265,16 @@ MEMO: delay ( parser -- parser )
MEMO: list-of ( items separator -- parser )
hide over 2array seq repeat0 [ concat ] action 2array seq [ unclip 1vector swap first append ] action ;
MEMO: 'digit' ( -- parser )
[ digit? ] satisfy [ digit> ] action ;
MEMO: 'integer' ( -- parser )
'digit' repeat1 [ 10 swap digits>integer ] action ;
MEMO: 'string' ( -- parser )
[
[ CHAR: " = ] satisfy hide ,
[ CHAR: " = not ] satisfy repeat0 ,
[ CHAR: " = ] satisfy hide ,
] { } make seq [ first >string ] action ;

1
extra/peg/pl0/tags.txt Normal file
View File

@ -0,0 +1 @@
parsing

View File

@ -0,0 +1 @@
Chris Double

View File

@ -0,0 +1,43 @@
! Copyright (C) 2006 Chris Double.
! See http://factorcode.org/license.txt for BSD license.
USING: help.syntax help.markup peg peg.search ;
HELP: tree-write
{ $values
{ "object" "an object" } }
{ $description
"Write the object to the standard output stream, unless "
"it is an array, in which case recurse through the array "
"writing each object to the stream." }
{ $example "{ 65 \"bc\" { 68 \"ef\" } } tree-write" "AbcDef" } ;
HELP: search
{ $values
{ "string" "a string" }
{ "parser" "a peg based parser" }
{ "seq" "a sequence" }
}
{ $description
"Returns a sequence containing the parse results of all substrings "
"from the input string that successfully parse using the "
"parser."
}
{ $example "\"one 123 two 456\" 'integer' search" "V{ 123 456 }" }
{ $example "\"one 123 \\\"hello\\\" two 456\" 'integer' 'string' 2array choice search" "V{ 123 \"hello\" 456 }" }
{ $see-also replace } ;
HELP: replace
{ $values
{ "string" "a string" }
{ "parser" "a peg based parser" }
{ "result" "a string" }
}
{ $description
"Returns a copy of the original string but with all substrings that "
"successfully parse with the given parser replaced with "
"the result of that parser."
}
{ $example "\"one 123 two 456\" 'integer' [ 2 * number>string ] action replace" "\"one 246 two 912\"" }
{ $see-also search } ;

View File

@ -0,0 +1,18 @@
! Copyright (C) 2007 Chris Double.
! See http://factorcode.org/license.txt for BSD license.
!
USING: kernel math math.parser arrays tools.test peg peg.search ;
IN: temporary
{ V{ 123 456 } } [
"abc 123 def 456" 'integer' search
] unit-test
{ V{ 123 "hello" 456 } } [
"one 123 \"hello\" two 456" 'integer' 'string' 2array choice search
] unit-test
{ "abc 246 def 912" } [
"abc 123 def 456" 'integer' [ 2 * number>string ] action replace
] unit-test

View File

@ -1,8 +1,8 @@
! Copyright (C) 2006 Chris Double.
! See http://factorcode.org/license.txt for BSD license.
USING: kernel math io io.streams.string sequences strings
lazy-lists combinators parser-combinators.simple ;
IN: parser-combinators
combinators peg memoize arrays ;
IN: peg.search
: tree-write ( object -- )
{
@ -12,26 +12,21 @@ IN: parser-combinators
{ [ t ] [ write ] }
} cond ;
MEMO: any-char-parser ( -- parser )
[ drop t ] satisfy ;
: search ( string parser -- seq )
any-char-parser [ drop f ] <@ <|> <*> parse dup nil? [
drop { }
any-char-parser [ drop f ] action 2array choice repeat0 parse dup [
parse-result-ast [ ] subset
] [
car parse-result-parsed [ ] subset
drop { }
] if ;
: search* ( string parsers -- seq )
unclip [ <|> ] reduce any-char-parser [ drop f ] <@ <|> <*> parse dup nil? [
drop { }
] [
car parse-result-parsed [ ] subset
] if ;
: (replace) ( string parser -- seq )
any-char-parser <|> <*> parse-1 ;
any-char-parser 2array choice repeat0 parse parse-result-ast [ ] subset ;
: replace ( string parser -- result )
[ (replace) [ tree-write ] each ] string-out ;
: replace* ( string parsers -- result )
swap [ replace ] reduce ;

View File

@ -0,0 +1 @@
Search and replace using parsing expression grammars

View File

@ -0,0 +1 @@
parsing

1
extra/peg/tags.txt Normal file
View File

@ -0,0 +1 @@
parsing

View File

@ -1,7 +1,7 @@
! Copyright (C) 2006 Chris Double. All Rights Reserved.
! See http://factorcode.org/license.txt for BSD license.
!
USING: kernel furnace fjsc parser-combinators namespaces
USING: kernel furnace fjsc peg namespaces
lazy-lists io io.files furnace.validator sequences
http.client http.server http.server.responders
webapps.file html ;
@ -11,7 +11,7 @@ IN: webapps.fjsc
#! Compile the factor code as a string, outputting the http
#! response containing the javascript.
serving-text
'expression' parse-1 fjsc-compile
'expression' parse parse-result-ast fjsc-compile
write flush ;
! The 'compile' action results in an URL that looks like
@ -25,7 +25,7 @@ IN: webapps.fjsc
: compile-url ( url -- )
#! Compile the factor code at the given url, return the javascript.
dup "http:" head? [ "Unable to access remote sites." throw ] when
"http://" host rot 3append http-get 2nip compile "();" write flush ;
"http://" "Host" header-param rot 3append http-get 2nip compile "();" write flush ;
\ compile-url {
{ "url" v-required }