Merge branch 'master' of factorcode.org:/git/factor

core/alien/alien-docs.factor

@@ -12,9 +12,7 @@ HELP: dll
 
 HELP: expired?
 { $values { "c-ptr" "an alien, byte array, or " { $link f } } { "?" "a boolean" } }
-{ $description "Tests if the alien is a relic from an earlier session. When an image is loaded, any alien objects which persisted in the image are marked as being expired."
+{ $description "Tests if the alien is a relic from an earlier session. A byte array is never considered to have expired, whereas passing " { $link f } " always yields true." } ;
-$nl
-"A byte array is never considered to be expired, whereas passing " { $link f } " always yields true." } ;
 
 HELP: <displaced-alien> ( displacement c-ptr -- alien )
 { $values { "displacement" "an integer" } { "c-ptr" "an alien, byte array, or " { $link f } } { "alien" "a new alien" } }
@@ -146,16 +144,22 @@ HELP: alien-callback
 
 { alien-invoke alien-indirect alien-callback } related-words
 
+ARTICLE: "alien-expiry" "Alien expiry"
+"When an image is loaded, any alien objects which persisted from the previous session are marked as having expired. This is because the C pointers they contain are almost certainly no longer valid."
+$nl
+"For this reason, the " { $link POSTPONE: ALIEN: } " word should not be used in source files, since loading the source file then saving the image will result in the literal becoming expired. Use " { $link <alien> } " instead, and ensure the word calling " { $link <alien> } " is not declared " { $link POSTPONE: flushable } "."
+{ $subsection expired? } ;
+
 ARTICLE: "aliens" "Alien addresses"
 "Instances of the " { $link alien } " class represent pointers to C data outside the Factor heap:"
 { $subsection <alien> }
 { $subsection <displaced-alien> }
 { $subsection alien-address }
-{ $subsection expired? }
 "Anywhere that a " { $link alien } " instance is accepted, the " { $link f } " singleton may be passed in to denote a null pointer."
 $nl
 "Usually alien objects do not have to created and dereferenced directly; instead declaring C function parameters and return values as having a pointer type such as " { $snippet "void*" } " takes care of the details."
 { $subsection "syntax-aliens" }
+{ $subsection "alien-expiry" }
 "When higher-level abstractions won't do:"
 { $subsection "reading-writing-memory" }
 { $see-also "c-data" "c-types-specs" } ;

core/alien/strings/strings-docs.factor

@@ -1,5 +1,5 @@
 USING: help.markup help.syntax strings byte-arrays alien libc
-debugger ;
+debugger io.encodings.string sequences ;
 IN: alien.strings
 
 HELP: string>alien
@@ -38,7 +38,11 @@ HELP: utf16n
 ARTICLE: "c-strings" "C strings"
 "C string types are arrays with shape " { $snippet "{ \"char*\" encoding }" } ", where " { $snippet "encoding" } " is an encoding descriptor. The type " { $snippet "\"char*\"" } " is an alias for " { $snippet "{ \"char*\" utf8 }" } ". See " { $link "encodings-descriptors" } " for information about encoding descriptors."
 $nl
-"Passing a Factor string to a C function expecting a C string allocates a " { $link byte-array } " in the Factor heap; the string is then converted to the requested format and a raw pointer is passed to the function. If the conversion fails, for example if the string contains null bytes or characters with values higher than 255, a " { $link c-string-error. } " is thrown."
+"Passing a Factor string to a C function expecting a C string allocates a " { $link byte-array } " in the Factor heap; the string is then converted to the requested format and a raw pointer is passed to the function."
+$nl
+"If the conversion fails, for example if the string contains null bytes or characters with values higher than 255, a " { $link c-string-error. } " is thrown."
+$nl
+"Care must be taken if the C function expects a " { $snippet "char*" } " with a length in bytes, rather than a null-terminated " { $snippet "char*" } "; passing the result of calling " { $link length } " on the string object will not suffice. This is because a Factor string of " { $emphasis "n" } " characters will not necessarily encode to " { $emphasis "n" } " bytes. The correct idiom for C functions which take a string with a length is to first encode the string using " { $link encode } ", and then pass the resulting byte array together with the length of this byte array."
 $nl
 "Sometimes a C function has a parameter type of " { $snippet "void*" } ", and various data types, among them strings, can be passed in. In this case, strings are not automatically converted to aliens, and instead you must call one of these words:"
 { $subsection string>alien }

core/alien/syntax/syntax-docs.factor

@@ -11,7 +11,7 @@ HELP: ALIEN:
 { $syntax "ALIEN: address" }
 { $values { "address" "a non-negative integer" } }
 { $description "Creates an alien object at parse time." }
-{ $notes "Alien objects are invalidated between image saves and loads." } ;
+{ $notes "Alien objects are invalidated between image saves and loads, and hence source files should not contain alien literals; this word is for interactive use only. See " { $link "alien-expiry" } " for details." } ;
 
 ARTICLE: "syntax-aliens" "Alien object literal syntax"
 { $subsection POSTPONE: ALIEN: }

core/bootstrap/compiler/compiler.factor

@@ -5,8 +5,8 @@ sequences namespaces parser kernel kernel.private classes
 classes.private arrays hashtables vectors classes.tuple sbufs
 inference.dataflow hashtables.private sequences.private math
 classes.tuple.private growable namespaces.private assocs words
-generator command-line vocabs io prettyprint libc compiler.units
+generator command-line vocabs io io.encodings.string
-math.order ;
+prettyprint libc compiler.units math.order ;
 IN: bootstrap.compiler
 
 ! Don't bring this in when deploying, since it will store a

core/bootstrap/image/image.factor

@@ -250,7 +250,7 @@ GENERIC: ' ( obj -- ptr )
     #! n is positive or zero.
     [ dup 0 > ]
     [ [ bignum-bits neg shift ] [ bignum-radix bitand ] bi ]
-    [ ] unfold nip ;
+    [ ] produce nip ;
 
 : emit-bignum ( n -- )
     dup dup 0 < [ neg ] when bignum>seq

core/classes/algebra/algebra.factor

@@ -194,7 +194,7 @@ M: anonymous-complement (classes-intersect?)
     [ [ name>> ] compare ] sort >vector
     [ dup empty? not ]
     [ dup largest-class >r over delete-nth r> ]
-    [ ] unfold nip ;
+    [ ] produce nip ;
 
 : min-class ( class seq -- class/f )
     over [ classes-intersect? ] curry filter

core/classes/tuple/tuple-docs.factor

@@ -393,8 +393,14 @@ HELP: >tuple
 { $values { "seq" sequence } { "tuple" tuple } }
 { $description "Creates a tuple with slot values taken from a sequence. The first element of the sequence must be a tuple class word and the remainder the declared slots."
 $nl
-"If the sequence has too many elements, they are ignored, and if it has too few, the remaining slots in the tuple are set to " { $link f } "." }
+"If the sequence has too few elements, the remaining slots in the tuple are set to their initial values." }
-{ $errors "Throws an error if the first element of the sequence is not a tuple class word." } ;
+{ $errors "Throws an error if one of the following occurs:"
+    { $list
+        "the first element of the sequence is not a tuple class word"
+        "the values in the sequence do not satisfy the slot class predicates"
+        "the sequence is too long"
+    }
+} ;
 
 HELP: tuple>array ( tuple -- array )
 { $values { "tuple" tuple } { "array" array } }

core/classes/tuple/tuple-tests.factor

@@ -683,3 +683,17 @@ DEFER: error-y
 [ t ] [ \ error-y tuple-class? ] unit-test
 
 [ f ] [ \ error-y generic? ] unit-test
+
+[ ] [
+    "IN: classes.tuple.tests TUPLE: forget-subclass-test ; TUPLE: forget-subclass-test' < forget-subclass-test ;"
+    <string-reader> "forget-subclass-test" parse-stream
+    drop
+] unit-test
+
+[ ] [ "forget-subclass-test'" "classes.tuple.tests" lookup new "bad-object" set ] unit-test
+
+[ ] [
+    "IN: classes.tuple.tests TUPLE: forget-subclass-test a ;"
+    <string-reader> "forget-subclass-test" parse-stream
+    drop
+] unit-test

core/classes/tuple/tuple.factor

@@ -194,13 +194,17 @@ ERROR: bad-superclass class ;
     [ permute-slots ] [ class>> ] bi
     slots>tuple ;
 
+: outdated-tuple? ( tuple assoc -- ? )
+    over tuple? [
+        [ [ layout-of ] dip key? ]
+        [ drop class "forgotten" word-prop not ]
+        2bi and
+    ] [ 2drop f ] if ;
+
 : update-tuples ( -- )
     outdated-tuples get
     dup assoc-empty? [ drop ] [
-        [
+        [ outdated-tuple? ] curry instances
-            over tuple?
-            [ >r layout-of r> key? ] [ 2drop f ] if
-        ] curry instances
         dup [ update-tuple ] map become
     ] if ;
 

core/heaps/heaps.factor

@@ -191,4 +191,4 @@ M: priority-queue heap-pop ( heap -- value key )
 : heap-pop-all ( heap -- alist )
     [ dup heap-empty? not ]
     [ dup heap-pop swap 2array ]
-    [ ] unfold nip ;
+    [ ] produce nip ;

core/inference/inference-tests.factor

@@ -540,7 +540,7 @@ ERROR: custom-error ;
 { 1 0 } [ [ ] map-children ] must-infer-as
 
 ! Corner case
-[ [ [ f dup ] [ dup ] [ ] unfold ] infer ] must-fail
+[ [ [ f dup ] [ dup ] [ ] produce ] infer ] must-fail
 
 [ [ [ f dup ] [ ] [ ] while ] infer ] must-fail
 

core/io/io.factor

@@ -100,9 +100,9 @@ SYMBOL: error-stream
     presented associate format ;
 
 : lines ( stream -- seq )
-    [ [ readln dup ] [ ] [ drop ] unfold ] with-input-stream ;
+    [ [ readln dup ] [ ] [ drop ] produce ] with-input-stream ;
 
 : contents ( stream -- str )
     [
-        [ 65536 read dup ] [ ] [ drop ] unfold concat f like
+        [ 65536 read dup ] [ ] [ drop ] produce concat f like
     ] with-input-stream ;

core/sequences/sequences-docs.factor

@@ -116,7 +116,7 @@ ARTICLE: "sequences-slices" "Subsequences and slices"
 "Taking a sequence apart into a head and a tail:"
 { $subsection unclip-slice }
 { $subsection cut-slice }
-"A utility for words which use slices as mutable iterators:"
+"A utility for words which use slices as iterators:"
 { $subsection <flat-slice> } ;
 
 ARTICLE: "sequences-combinators" "Sequence combinators"
@@ -130,7 +130,7 @@ ARTICLE: "sequences-combinators" "Sequence combinators"
 { $subsection map }
 { $subsection 2map }
 { $subsection accumulate }
-{ $subsection unfold }
+{ $subsection produce }
 "Filtering:"
 { $subsection push-if }
 { $subsection filter } ;
@@ -748,8 +748,9 @@ HELP: slice-error
 } ;
 
 HELP: slice
-{ $class-description "A virtual sequence which presents a subrange of the elements of an underlying sequence. New instances can be created by calling " { $link <slice> } ". Slices are mutable if the underlying sequence is mutable, and mutating a slice changes the underlying sequence." }
+{ $class-description "A virtual sequence which presents a subrange of the elements of an underlying sequence. New instances can be created by calling " { $link <slice> } "."
-{ $notes "The slots of a slice should not be changed after the slice has been created, because this can break invariants." } ;
+$nl
+"Slices are mutable if the underlying sequence is mutable, and mutating a slice changes the underlying sequence. However, slices cannot be resized after creation." } ;
 
 HELP: check-slice
 { $values { "from" "a non-negative integer" } { "to" "a non-negative integer" } { "seq" sequence } }
@@ -764,10 +765,10 @@ HELP: collapse-slice
 HELP: <flat-slice>
 { $values { "seq" sequence } { "slice" slice } }
 { $description "Outputs a slice with the same elements as " { $snippet "seq" } ", and " { $link slice-from } " equal to 0 and " { $link slice-to } " equal to the length of " { $snippet "seq" } "." }
-{ $notes "Some words create slices then proceed to read and write the " { $link slice-from } " and " { $link slice-to } " slots of the slice. To behave predictably when they are themselves given a slice as input, they apply this word first to get a canonical slice." } ;
+{ $notes "Some words create slices then proceed to read the " { $snippet "to" } " and " { $snippet "from" } " slots of the slice. To behave predictably when they are themselves given a slice as input, they apply this word first to get a canonical slice." } ;
 
 HELP: <slice>
-{ $values { "from" "a non-negative integer" } { "to" "a non-negative integer" } { "seq" sequence } { "slice" "a slice" } }
+{ $values { "from" "a non-negative integer" } { "to" "a non-negative integer" } { "seq" sequence } { "slice" slice } }
 { $description "Outputs a new virtual sequence sharing storage with the subrange of elements in " { $snippet "seq" } " with indices starting from and including " { $snippet "m" } ", and up to but not including " { $snippet "n" } "." }
 { $errors "Throws an error if " { $snippet "m" } " or " { $snippet "n" } " is out of bounds." }
 { $notes "Taking the slice of a slice outputs a slice of the underlying sequence of the original slice. Keep this in mind when writing code which depends on the values of " { $link slice-from } " and " { $link slice-to } " being equal to the inputs to this word. The " { $link <flat-slice> } " word might be helpful in such situations." } ;
@@ -950,14 +951,14 @@ HELP: supremum
 { $description "Outputs the greatest element of " { $snippet "seq" } "." }
 { $errors "Throws an error if the sequence is empty." } ;
 
-HELP: unfold
+HELP: produce
 { $values { "pred" "a quotation with stack effect " { $snippet "( -- ? )" } } { "quot" "a quotation with stack effect " { $snippet "( -- obj )" } } { "tail" "a quotation" } { "seq" "a sequence" } }
 { $description "Calls " { $snippet "pred" } " repeatedly. If the predicate yields " { $link f } ", stops, otherwise, calls " { $snippet "quot" } " to yield a value. Values are accumulated and returned in a sequence at the end." }
 { $examples
     "The following example divides a number by two until we reach zero, and accumulates intermediate results:"
-    { $example "USING: kernel math prettyprint sequences ;" "1337 [ dup 0 > ] [ 2/ dup ] [ ] unfold nip ." "{ 668 334 167 83 41 20 10 5 2 1 0 }" }
+    { $example "USING: kernel math prettyprint sequences ;" "1337 [ dup 0 > ] [ 2/ dup ] [ ] produce nip ." "{ 668 334 167 83 41 20 10 5 2 1 0 }" }
-    "The " { $snippet "tail" } " quotation is used when the predicate produces more than one output value. In this case, we have to drop this value even if the predicate fails in order for stack inference to calculate a stack effect for the " { $link unfold } " call:"
+    "The " { $snippet "tail" } " quotation is used when the predicate produces more than one output value. In this case, we have to drop this value even if the predicate fails in order for stack inference to calculate a stack effect for the " { $link produce } " call:"
-    { $unchecked-example "USING: kernel prettyprint random sequences ;" "[ 10 random dup 1 > ] [ ] [ drop ] unfold ." "{ 8 2 2 9 }" }
+    { $unchecked-example "USING: kernel prettyprint random sequences ;" "[ 10 random dup 1 > ] [ ] [ drop ] produce ." "{ 8 2 2 9 }" }
 } ;
 
 HELP: sigma

core/sequences/sequences.factor

@@ -420,11 +420,11 @@ PRIVATE>
 : accumulator ( quot -- quot' vec )
     V{ } clone [ [ push ] curry compose ] keep ; inline
 
-: unfold ( pred quot tail -- seq )
+: produce ( pred quot tail -- seq )
     swap accumulator >r swap while r> { } like ; inline
 
 : follow ( obj quot -- seq )
-    >r [ dup ] r> [ keep ] curry [ ] unfold nip ; inline
+    >r [ dup ] r> [ keep ] curry [ ] produce nip ; inline
 
 : prepare-index ( seq quot -- seq n quot )
     >r dup length r> ; inline

extra/benchmark/reverse-complement/reverse-complement.factor

@@ -1,30 +1,20 @@
 USING: io io.files io.streams.duplex kernel sequences
 sequences.private strings vectors words memoize splitting
-grouping hints unicode.case continuations io.encodings.ascii ;
+grouping hints tr continuations io.encodings.ascii
+unicode.case ;
 IN: benchmark.reverse-complement
 
-MEMO: trans-map ( -- str )
+TR: trans-map ch>upper "ACGTUMRYKVHDB" "TGCAAKYRMBDHV" ;
-    256 >string
-    "TGCAAKYRMBDHV" "ACGTUMRYKVHDB"
-    [ pick set-nth ] 2each ;
-
-: do-trans-map ( str -- )
-    [ ch>upper trans-map nth ] change-each ;
-
-HINTS: do-trans-map string ;
 
 : translate-seq ( seq -- str )
-    concat dup reverse-here dup do-trans-map ;
+    concat dup reverse-here dup trans-map-fast ;
 
 : show-seq ( seq -- )
     translate-seq 60 <groups> [ print ] each ;
 
 : do-line ( seq line -- seq )
-    dup first ">;" memq? [
+    dup first ">;" memq?
-        over show-seq print dup delete-all
+    [ over show-seq print dup delete-all ] [ over push ] if ;
-    ] [
-        over push
-    ] if ;
 
 HINTS: do-line vector string ;
 

extra/concurrency/mailboxes/mailboxes.factor

@@ -52,7 +52,7 @@ M: mailbox dispose* threads>> notify-all ;
     block-if-empty
     [ dup mailbox-empty? ]
     [ dup data>> pop-back ]
-    [ ] unfold nip ;
+    [ ] produce nip ;
 
 : mailbox-get-all ( mailbox -- array )
     f mailbox-get-all-timeout ;

extra/db/sqlite/lib/lib.factor

@@ -4,7 +4,8 @@ USING: alien.c-types arrays assocs kernel math math.parser
 namespaces sequences db.sqlite.ffi db combinators
 continuations db.types calendar.format serialize
 io.streams.byte-array byte-arrays io.encodings.binary
-io.backend db.errors present urls ;
+io.backend db.errors present urls io.encodings.utf8
+io.encodings.string ;
 IN: db.sqlite.lib
 
 ERROR: sqlite-error < db-error n string ;
@@ -33,7 +34,7 @@ ERROR: sqlite-sql-error < sql-error n string ;
     sqlite3_close sqlite-check-result ;
 
 : sqlite-prepare ( db sql -- handle )
-    dup length "void*" <c-object> "void*" <c-object>
+    utf8 encode dup length "void*" <c-object> "void*" <c-object>
     [ sqlite3_prepare_v2 sqlite-check-result ] 2keep
     drop *void* ;
 
@@ -44,7 +45,7 @@ ERROR: sqlite-sql-error < sql-error n string ;
     >r dupd sqlite-bind-parameter-index r> ;
 
 : sqlite-bind-text ( handle index text -- )
-    dup length SQLITE_TRANSIENT
+    utf8 encode dup length SQLITE_TRANSIENT
     sqlite3_bind_text sqlite-check-result ;
 
 : sqlite-bind-int ( handle i n -- )

extra/furnace/actions/actions.factor

@@ -7,7 +7,8 @@ xml.entities
 http.server
 http.server.responses
 furnace
-furnace.flash
+furnace.redirection
+furnace.conversations
 html.forms
 html.elements
 html.components
@@ -38,20 +39,23 @@ TUPLE: action rest authorize init display validate submit ;
 : <action> ( -- action )
     action new-action ;
 
+: merge-forms ( form -- )
+    form get
+    [ [ errors>> ] bi@ push-all ]
+    [ [ values>> ] bi@ swap update ]
+    [ swap validation-failed>> >>validation-failed drop ]
+    2tri ;
+
 : set-nested-form ( form name -- )
     dup empty? [
-        drop form set
+        drop merge-forms
     ] [
-        dup length 1 = [
+        unclip [ set-nested-form ] nest-form
-            first set-value
-        ] [
-            unclip [ set-nested-form ] nest-form
-        ] if
     ] if ;
 
 : restore-validation-errors ( -- )
-    form fget [
+    form cget [
-        nested-forms fget set-nested-form
+        nested-forms cget set-nested-form
     ] when* ;
 
 : handle-get ( action -- response )
@@ -76,10 +80,11 @@ TUPLE: action rest authorize init display validate submit ;
     dup [ >url [ same-host? ] keep and ] when ;
 
 : validation-failed ( -- * )
-    post-request? revalidate-url and
+    post-request? revalidate-url and [
-    [
+        begin-conversation
-        nested-forms-key param " " split harvest nested-forms set
+        nested-forms-key param " " split harvest nested-forms cset
-        { form nested-forms } <flash-redirect>
+        form get form cset
+        <redirect>
     ] [ <400> ] if*
     exit-with ;
 
@@ -110,7 +115,7 @@ M: action call-responder* ( path action -- response )
     } case ;
 
 M: action modify-form
-    drop request get url>> revalidate-url-key hidden-form-field ;
+    drop url get revalidate-url-key hidden-form-field ;
 
 : check-validation ( -- )
     validation-failed? [ validation-failed ] when ;

extra/furnace/alloy/alloy.factor

@@ -1,26 +1,24 @@
 ! Copyright (C) 2008 Slava Pestov.
 ! See http://factorcode.org/license.txt for BSD license.
 USING: kernel sequences db.tuples alarms calendar db fry
-furnace.cache
-furnace.asides
-furnace.flash
-furnace.sessions
-furnace.referrer
 furnace.db
+furnace.cache
+furnace.referrer
+furnace.sessions
+furnace.conversations
 furnace.auth.providers
 furnace.auth.login.permits ;
 IN: furnace.alloy
 
 : <alloy> ( responder db params -- responder' )
     '[
-        <asides>
+        <conversations>
-        <flash-scopes>
         <sessions>
         , , <db-persistence>
         <check-form-submissions>
     ] call ;
 
-: state-classes { session flash-scope aside permit } ; inline
+: state-classes { session conversation permit } ; inline
 
 : init-furnace-tables ( -- )
     state-classes ensure-tables

extra/furnace/asides/asides.factor

@@ -1,95 +0,0 @@
-! Copyright (C) 2008 Slava Pestov.
-! See http://factorcode.org/license.txt for BSD license.
-USING: accessors namespaces sequences arrays kernel
-assocs assocs.lib hashtables math.parser urls combinators
-html.elements html.templates.chloe.syntax db.types db.tuples
-http http.server http.server.filters 
-furnace furnace.cache furnace.sessions furnace.redirection ;
-IN: furnace.asides
-
-TUPLE: aside < server-state session method url post-data ;
-
-: <aside> ( id -- aside )
-    aside new-server-state ;
-
-aside "ASIDES"
-{
-    { "session" "SESSION" BIG-INTEGER +not-null+ }
-    { "method" "METHOD" { VARCHAR 10 } +not-null+ }
-    { "url" "URL" URL +not-null+ }
-    { "post-data" "POST_DATA" FACTOR-BLOB }
-} define-persistent
-
-TUPLE: asides < server-state-manager ;
-
-: <asides> ( responder -- responder' )
-    asides new-server-state-manager ;
-
-: begin-aside* ( -- id )
-    f <aside>
-        session get id>> >>session
-        request get
-        [ method>> >>method ]
-        [ url>> >>url ]
-        [ post-data>> >>post-data ]
-        tri
-    [ asides get touch-state ] [ insert-tuple ] [ id>> ] tri ;
-
-: end-aside-post ( aside -- response )
-    request [
-        clone
-            over post-data>> >>post-data
-            over url>> >>url
-    ] change
-    url>> path>> split-path
-    asides get responder>> call-responder ;
-
-ERROR: end-aside-in-get-error ;
-
-: get-aside ( id -- aside )
-    dup [ aside get-state ] when
-    dup [ dup session>> session get id>> = [ drop f ] unless ] when ;
-
-: end-aside* ( url id -- response )
-    post-request? [ end-aside-in-get-error ] unless
-    aside get-state [
-        dup method>> {
-            { "GET" [ url>> <redirect> ] }
-            { "HEAD" [ url>> <redirect> ] }
-            { "POST" [ end-aside-post ] }
-        } case
-    ] [ <redirect> ] ?if ;
-
-SYMBOL: aside-id
-
-: aside-id-key "__a" ;
-
-: begin-aside ( -- )
-    begin-aside* aside-id set ;
-
-: end-aside ( default -- response )
-    aside-id [ f ] change end-aside* ;
-
-: request-aside-id ( request -- aside-id )
-    aside-id-key swap request-params at string>number ;
-
-M: asides call-responder*
-    dup asides set
-    request get request-aside-id aside-id set
-    call-next-method ;
-
-M: asides link-attr ( tag -- )
-    drop
-    "aside" optional-attr {
-        { "none" [ aside-id off ] }
-        { "begin" [ begin-aside ] }
-        { "current" [ ] }
-        { f [ ] }
-    } case ;
-
-M: asides modify-query ( query responder -- query' )
-    drop
-    aside-id get [ aside-id-key associate assoc-union ] when* ;
-
-M: asides modify-form ( responder -- )
-    drop aside-id get aside-id-key hidden-form-field ;

extra/furnace/auth/auth.factor

@@ -1,7 +1,7 @@
 ! Copyright (c) 2008 Slava Pestov
 ! See http://factorcode.org/license.txt for BSD license.
 USING: accessors assocs namespaces kernel sequences sets
-destructors combinators fry
+destructors combinators fry logging
 io.encodings.utf8 io.encodings.string io.binary random
 checksums checksums.sha2
 html.forms
@@ -18,7 +18,11 @@ IN: furnace.auth
 
 SYMBOL: logged-in-user
 
-: logged-in? ( -- ? ) logged-in-user get >boolean ;
+: logged-in? ( -- ? )
+    logged-in-user get >boolean ;
+
+: username ( -- string/f )
+    logged-in-user get dup [ username>> ] when ;
 
 GENERIC: init-user-profile ( responder -- )
 
@@ -30,9 +34,6 @@ M: dispatcher init-user-profile
 M: filter-responder init-user-profile
     responder>> init-user-profile ;
 
-: have-capability? ( capability -- ? )
-    logged-in-user get capabilities>> member? ;
-
 : profile ( -- assoc ) logged-in-user get profile>> ;
 
 : user-changed ( -- )
@@ -57,11 +58,14 @@ V{ } clone capabilities set-global
 
 TUPLE: realm < dispatcher name users checksum secure ;
 
-GENERIC: login-required* ( realm -- response )
+GENERIC: login-required* ( description capabilities realm -- response )
+
+GENERIC: init-realm ( realm -- )
 
 GENERIC: logged-in-username ( realm -- username )
 
-: login-required ( -- * ) realm get login-required* exit-with ;
+: login-required ( description capabilities -- * )
+    realm get login-required* exit-with ;
 
 : new-realm ( responder name class -- realm )
     new-dispatcher
@@ -87,9 +91,16 @@ M: user-saver dispose
 : init-user ( user -- )
     [ [ logged-in-user set ] [ save-user-after ] bi ] when* ;
 
+\ init-user DEBUG add-input-logging
+
 M: realm call-responder* ( path responder -- response )
     dup realm set
-    dup logged-in-username dup [ users get-user ] when init-user
+    logged-in? [
+        dup init-realm
+        dup logged-in-username
+        dup [ users get-user ] when
+        init-user
+    ] unless
     call-next-method ;
 
 : encode-password ( string salt -- bytes )
@@ -122,19 +133,22 @@ TUPLE: protected < filter-responder description capabilities ;
     protected new
         swap >>responder ;
 
-: check-capabilities ( responder user/f -- ? )
+: have-capabilities? ( capabilities -- ? )
-    {
+    logged-in-user get {
         { [ dup not ] [ 2drop f ] }
         { [ dup deleted>> 1 = ] [ 2drop f ] }
-        [ [ capabilities>> ] bi@ subset? ]
+        [ capabilities>> subset? ]
     } cond ;
 
 M: protected call-responder* ( path responder -- response )
     '[
         , ,
         dup protected set
-        dup logged-in-user get check-capabilities
+        dup capabilities>> have-capabilities?
-        [ call-next-method ] [ 2drop realm get login-required* ] if
+        [ call-next-method ] [
+            [ drop ] [ [ description>> ] [ capabilities>> ] bi ] bi*
+            realm get login-required*
+        ] if
     ] if-secure-realm ;
 
 : <auth-boilerplate> ( responder -- responder' )

extra/furnace/auth/basic/basic.factor

@@ -20,8 +20,8 @@ TUPLE: basic-auth-realm < realm ;
     401 "Invalid username or password" <trivial-response>
     [ "Basic realm=\"" % swap % "\"" % ] "" make "WWW-Authenticate" set-header ;
 
-M: basic-auth-realm login-required* ( realm -- response )
+M: basic-auth-realm login-required* ( description capabilities realm -- response )
-    name>> <401> ;
+    2nip name>> <401> ;
 
 M: basic-auth-realm logged-in-username ( realm -- uid )
     drop

extra/furnace/auth/features/deactivate-user/deactivate-user.factor

@@ -2,7 +2,10 @@
 ! See http://factorcode.org/license.txt for BSD license.
 USING: kernel assocs namespaces accessors db db.tuples urls
 http.server.dispatchers
-furnace.asides furnace.actions furnace.auth furnace.auth.providers ;
+furnace.conversations
+furnace.actions
+furnace.auth
+furnace.auth.providers ;
 IN: furnace.auth.features.deactivate-user
 
 : <deactivate-user-action> ( -- action )

extra/furnace/auth/features/edit-profile/edit-profile.factor

@@ -1,12 +1,10 @@
 ! Copyright (c) 2008 Slava Pestov.
 ! See http://factorcode.org/license.txt for BSD license.
 USING: kernel accessors namespaces sequences assocs
-validators urls
+validators urls html.forms http.server.dispatchers
-html.forms
-http.server.dispatchers
 furnace.auth
-furnace.asides
+furnace.actions
-furnace.actions ;
+furnace.conversations ;
 IN: furnace.auth.features.edit-profile
 
 : <edit-profile-action> ( -- action )
@@ -22,7 +20,7 @@ IN: furnace.auth.features.edit-profile
         { realm "features/edit-profile/edit-profile" } >>template
 
         [
-            logged-in-user get username>> "username" set-value
+            username "username" set-value
 
             {
                 { "realname" [ [ v-one-line ] v-optional ] }
@@ -34,7 +32,7 @@ IN: furnace.auth.features.edit-profile
 
             { "password" "new-password" "verify-password" }
             [ value empty? not ] contains? [
-                "password" value logged-in-user get username>> check-login
+                "password" value username check-login
                 [ "incorrect password" validation-error ] unless
 
                 same-password-twice
@@ -54,7 +52,7 @@ IN: furnace.auth.features.edit-profile
 
             drop
 
-            URL" $login" end-aside
+            URL" $realm" end-aside
         ] >>submit
 
     <protected>

extra/furnace/auth/features/recover-password/recover-password.factor

@@ -11,7 +11,7 @@ IN: furnace.auth.features.recover-password
 SYMBOL: lost-password-from
 
 : current-host ( -- string )
-    request get url>> host>> host-name or ;
+    url get host>> host-name or ;
 
 : new-password-url ( user -- url )
     URL" recover-3" clone

extra/furnace/auth/login/login.factor

@@ -1,16 +1,15 @@
 ! Copyright (c) 2008 Slava Pestov
 ! See http://factorcode.org/license.txt for BSD license.
 USING: kernel accessors namespaces sequences math.parser
-calendar validators urls html.forms
+calendar validators urls logging html.forms
 http http.server http.server.dispatchers
 furnace
 furnace.auth
-furnace.flash
-furnace.asides
 furnace.actions
 furnace.sessions
 furnace.utilities
 furnace.redirection
+furnace.conversations
 furnace.auth.login.permits ;
 IN: furnace.auth.login
 
@@ -25,10 +24,8 @@ SYMBOL: permit-id
 
 TUPLE: login-realm < realm timeout domain ;
 
-M: login-realm call-responder*
+M: login-realm init-realm
-    [ name>> client-permit-id permit-id set ]
+    name>> client-permit-id permit-id set ;
-    [ call-next-method ]
-    bi ;
 
 M: login-realm logged-in-username
     drop permit-id get dup [ get-permit-uid ] when ;
@@ -47,11 +44,15 @@ M: login-realm modify-form ( responder -- )
 : put-permit-cookie ( response -- response' )
     <permit-cookie> put-cookie ;
 
+\ put-permit-cookie DEBUG add-input-logging
+
 : successful-login ( user -- response )
     [ username>> make-permit permit-id set ] [ init-user ] bi
     URL" $realm" end-aside
     put-permit-cookie ;
 
+\ successful-login DEBUG add-input-logging
+
 : logout ( -- )
     permit-id get [ delete-permit ] when*
     URL" $realm" end-aside ;
@@ -68,9 +69,8 @@ SYMBOL: capabilities
 : <login-action> ( -- action )
     <page-action>
         [
-            flashed-variables restore-flash
+            description cget "description" set-value
-            description get "description" set-value
+            capabilities cget words>strings "capabilities" set-value
-            capabilities get words>strings "capabilities" set-value
         ] >>init
 
         { login-realm "login" } >>template
@@ -90,16 +90,12 @@ SYMBOL: capabilities
 
 : <logout-action> ( -- action )
     <action>
-        [ logout ] >>submit
+        [ logout ] >>submit ;
-    <protected>
-        "logout" >>description ;
 
-M: login-realm login-required*
+M: login-realm login-required* ( description capabilities login -- response )
-    drop
     begin-aside
-    protected get description>> description set
+    [ description cset ] [ capabilities cset ] [ drop ] tri*
-    protected get capabilities>> capabilities set
+    URL" $realm/login" >secure-url <redirect> ;
-    URL" $realm/login" >secure-url flashed-variables <flash-redirect> ;
 
 : <login-realm> ( responder name -- auth )
     login-realm new-realm

extra/furnace/auth/login/permits/permits.factor

@@ -1,7 +1,5 @@
-USING: accessors namespaces combinators.lib kernel
+USING: accessors namespaces kernel combinators.short-circuit
-db.tuples db.types
+db.tuples db.types furnace.auth furnace.sessions furnace.cache ;
-furnace.auth furnace.sessions furnace.cache
-combinators.short-circuit ;
 
 IN: furnace.auth.login.permits
 

extra/furnace/boilerplate/boilerplate.factor

@@ -1,13 +1,12 @@
 ! Copyright (c) 2008 Slava Pestov
 ! See http://factorcode.org/license.txt for BSD license.
-USING: accessors kernel math.order namespaces combinators.lib
+USING: accessors kernel math.order namespaces furnace combinators.short-circuit
 html.forms
 html.templates
 html.templates.chloe
 locals
 http.server
-http.server.filters
+http.server.filters ;
-furnace combinators.short-circuit ;
 IN: furnace.boilerplate
 
 TUPLE: boilerplate < filter-responder template init ;

extra/furnace/conversations/conversations.factor

@@ -0,0 +1,178 @@
+! Copyright (C) 2008 Slava Pestov.
+! See http://factorcode.org/license.txt for BSD license.
+USING: namespaces assocs kernel sequences accessors hashtables
+urls db.types db.tuples math.parser fry logging combinators
+html.templates.chloe.syntax
+http http.server http.server.filters http.server.redirection
+furnace
+furnace.cache
+furnace.scopes
+furnace.sessions
+furnace.redirection ;
+IN: furnace.conversations
+
+TUPLE: conversation < scope
+session
+method url post-data ;
+
+: <conversation> ( id -- aside )
+    conversation new-server-state ;
+
+conversation "CONVERSATIONS" {
+    { "session" "SESSION" BIG-INTEGER +not-null+ }
+    { "method" "METHOD" { VARCHAR 10 } }
+    { "url" "URL" URL }
+    { "post-data" "POST_DATA" FACTOR-BLOB }
+} define-persistent
+
+: conversation-id-key "__c" ;
+
+TUPLE: conversations < server-state-manager ;
+
+: <conversations> ( responder -- responder' )
+    conversations new-server-state-manager ;
+
+SYMBOL: conversation
+
+SYMBOL: conversation-id
+
+: cget ( key -- value )
+    conversation get scope-get ;
+
+: cset ( value key -- )
+    conversation get scope-set ;
+
+: cchange ( key quot -- )
+    conversation get scope-change ; inline
+
+: get-conversation ( id -- conversation )
+    dup [ conversation get-state ] when
+    dup [ dup session>> session get id>> = [ drop f ] unless ] when ;
+
+: request-conversation-id ( request -- id )
+    conversation-id-key swap request-params at string>number ;
+
+: request-conversation ( request -- conversation )
+    request-conversation-id get-conversation ;
+
+: save-conversation-after ( conversation -- )
+    conversations get save-scope-after ;
+
+: set-conversation ( conversation -- )
+    [
+        [ conversation set ]
+        [ id>> conversation-id set ]
+        [ save-conversation-after ]
+        tri
+    ] when* ;
+
+: init-conversations ( conversations -- )
+    conversations set
+    request get request-conversation-id
+    get-conversation
+    set-conversation ;
+
+M: conversations call-responder*
+    [ init-conversations ]
+    [ conversations set ]
+    [ call-next-method ]
+    tri ;
+
+: empty-conversastion ( -- conversation )
+    conversation empty-scope
+        session get id>> >>session ;
+
+: touch-conversation ( conversation -- )
+    conversations get touch-state ;
+
+: add-conversation ( conversation -- )
+    [ touch-conversation ] [ insert-tuple ] bi ;
+
+: begin-conversation* ( -- conversation )
+    empty-conversastion dup add-conversation ;
+
+: begin-conversation ( -- )
+    conversation get [
+        begin-conversation*
+        set-conversation
+    ] unless ;
+
+: end-conversation ( -- )
+    conversation off
+    conversation-id off ;
+
+: <conversation-redirect> ( url seq -- response )
+    begin-conversation
+    [ [ get ] keep cset ] each
+    <redirect> ;
+
+: restore-conversation ( seq -- )
+    conversation get dup [
+        namespace>>
+        [ '[ , key? ] filter ]
+        [ '[ [ , at ] keep set ] each ]
+        bi
+    ] [ 2drop ] if ;
+
+: begin-aside ( -- )
+    begin-conversation
+    conversation get
+        request get
+        [ method>> >>method ]
+        [ url>> >>url ]
+        [ post-data>> >>post-data ]
+        tri
+    touch-conversation ;
+
+: end-aside-post ( aside -- response )
+    request [
+        clone
+            over post-data>> >>post-data
+            over url>> >>url
+    ] change
+    url>> path>> split-path
+    conversations get responder>> call-responder ;
+
+\ end-aside-post DEBUG add-input-logging
+
+ERROR: end-aside-in-get-error ;
+
+: move-on ( id -- response )
+    post-request? [ end-aside-in-get-error ] unless
+    dup method>> {
+        { "GET" [ url>> <redirect> ] }
+        { "HEAD" [ url>> <redirect> ] }
+        { "POST" [ end-aside-post ] }
+    } case ;
+
+: get-aside ( id -- conversation )
+    get-conversation dup [ dup method>> [ drop f ] unless ] when ;
+
+: end-aside* ( url id -- response )
+    get-aside [ move-on ] [ <redirect> ] ?if ;
+
+: end-aside ( default -- response )
+    conversation-id get
+    end-conversation
+    end-aside* ;
+
+M: conversations link-attr ( tag -- )
+    drop
+    "aside" optional-attr {
+        { "none" [ conversation-id off ] }
+        { "begin" [ begin-aside ] }
+        { "current" [ ] }
+        { f [ ] }
+    } case ;
+
+M: conversations modify-query ( query conversations -- query' )
+    drop
+    conversation-id get [
+        conversation-id-key associate assoc-union
+    ] when* ;
+
+M: conversations modify-form ( conversations -- )
+    drop
+    conversation-id get
+    conversation-id-key
+    hidden-form-field ;

extra/furnace/flash/flash.factor

@@ -1,61 +0,0 @@
-! Copyright (C) 2008 Slava Pestov.
-! See http://factorcode.org/license.txt for BSD license.
-USING: namespaces assocs assocs.lib kernel sequences accessors
-urls db.types db.tuples math.parser fry
-http http.server http.server.filters http.server.redirection
-furnace furnace.cache furnace.sessions furnace.redirection ;
-IN: furnace.flash
-
-TUPLE: flash-scope < server-state session namespace ;
-
-: <flash-scope> ( id -- aside )
-    flash-scope new-server-state ;
-
-flash-scope "FLASH_SCOPES" {
-    { "session" "SESSION" BIG-INTEGER +not-null+ }
-    { "namespace" "NAMESPACE" FACTOR-BLOB +not-null+ }
-} define-persistent
-
-: flash-id-key "__f" ;
-
-TUPLE: flash-scopes < server-state-manager ;
-
-: <flash-scopes> ( responder -- responder' )
-    flash-scopes new-server-state-manager ;
-
-SYMBOL: flash-scope
-
-: fget ( key -- value )
-    flash-scope get dup
-    [ namespace>> at ] [ 2drop f ] if ;
-
-: get-flash-scope ( id -- flash-scope )
-    dup [ flash-scope get-state ] when
-    dup [ dup session>> session get id>> = [ drop f ] unless ] when ;
-
-: request-flash-scope ( request -- flash-scope )
-    flash-id-key swap request-params at string>number get-flash-scope ;
-
-M: flash-scopes call-responder*
-    dup flash-scopes set
-    request get request-flash-scope flash-scope set
-    call-next-method ;
-
-: make-flash-scope ( seq -- id )
-    f <flash-scope>
-        session get id>> >>session
-        swap [ dup get ] H{ } map>assoc >>namespace
-    [ flash-scopes get touch-state ] [ insert-tuple ] [ id>> ] tri ;
-
-: <flash-redirect> ( url seq -- response )
-    [ clone ] dip
-    make-flash-scope flash-id-key set-query-param
-    <redirect> ;
-
-: restore-flash ( seq -- )
-    flash-scope get dup [
-        namespace>>
-        [ '[ , key? ] filter ]
-        [ '[ [ , at ] keep set ] each ]
-        bi
-    ] [ 2drop ] if ;

extra/furnace/furnace.factor

@@ -86,7 +86,7 @@ M: object modify-form drop ;
     "user-agent" request get header>> at "" or ;
 
 : same-host? ( url -- ? )
-    request get url>>
+    url get
     [ [ protocol>> ] [ host>> ] [ port>> ] tri 3array ] bi@ = ;
 
 : cookie-client-state ( key request -- value/f )

extra/furnace/redirection/redirection.factor

@@ -1,7 +1,7 @@
 ! Copyright (C) 2008 Slava Pestov.
 ! See http://factorcode.org/license.txt for BSD license.
 USING: kernel accessors combinators namespaces fry
-io.servers.connection
+io.servers.connection urls
 http http.server http.server.redirection http.server.filters
 furnace ;
 IN: furnace.redirection
@@ -33,8 +33,8 @@ TUPLE: secure-only < filter-responder ;
 C: <secure-only> secure-only
 
 : if-secure ( quot -- )
-    >r request get url>> protocol>> "http" =
+    >r url get protocol>> "http" =
-    [ request get url>> <secure-redirect> ]
+    [ url get <secure-redirect> ]
     r> if ; inline
 
 M: secure-only call-responder*

extra/furnace/scopes/scopes.factor

@@ -0,0 +1,42 @@
+! Copyright (C) 2008 Slava Pestov.
+! See http://factorcode.org/license.txt for BSD license.
+USING: kernel accessors assocs destructors
+db.tuples db.types furnace.cache ;
+IN: furnace.scopes
+
+TUPLE: scope < server-state namespace changed? ;
+
+: empty-scope ( class -- scope )
+    f swap new-server-state
+        H{ } clone >>namespace ; inline
+
+scope f
+{
+    { "namespace" "NAMESPACE" FACTOR-BLOB +not-null+ }
+} define-persistent
+
+: scope-changed ( scope -- )
+    t >>changed? drop ;
+
+: scope-get ( key scope -- value )
+    dup [ namespace>> at ] [ 2drop f ] if ;
+
+: scope-set ( value key scope -- )
+    [ namespace>> set-at ] [ scope-changed ] bi ;
+
+: scope-change ( key quot scope -- )
+    [ namespace>> swap change-at ] [ scope-changed ] bi ; inline
+
+! Destructor
+TUPLE: scope-saver scope manager ;
+
+C: <scope-saver> scope-saver
+
+M: scope-saver dispose
+    [ manager>> ] [ scope>> ] bi
+    dup changed?>> [
+        [ swap touch-state ] [ update-tuple ] bi
+    ] [ 2drop ] if ;
+
+: save-scope-after ( scope manager -- )
+    <scope-saver> &dispose drop ;

extra/furnace/sessions/sessions.factor

@@ -2,22 +2,21 @@
 ! See http://factorcode.org/license.txt for BSD license.
 USING: assocs kernel math.intervals math.parser namespaces
 strings random accessors quotations hashtables sequences continuations
-fry calendar combinators combinators.lib destructors alarms
+fry calendar combinators combinators.short-circuit destructors alarms
 io.servers.connection
 db db.tuples db.types
 http http.server http.server.dispatchers http.server.filters
 html.elements
-furnace furnace.cache combinators.short-circuit ;
+furnace furnace.cache furnace.scopes ;
 IN: furnace.sessions
 
-TUPLE: session < server-state namespace user-agent client changed? ;
+TUPLE: session < scope user-agent client ;
 
 : <session> ( id -- session )
     session new-server-state ;
 
 session "SESSIONS"
 {
-    { "namespace" "NAMESPACE" FACTOR-BLOB +not-null+ }
     { "user-agent" "USER_AGENT" TEXT +not-null+ }
     { "client" "CLIENT" TEXT +not-null+ }
 } define-persistent
@@ -39,23 +38,14 @@ TUPLE: sessions < server-state-manager domain verify? ;
     sessions new-server-state-manager
         t >>verify? ;
 
-: (session-changed) ( session -- )
-    t >>changed? drop ;
-
 : session-changed ( -- )
-    session get (session-changed) ;
+    session get scope-changed ;
 
-: sget ( key -- value )
+: sget ( key -- value ) session get scope-get ;
-    session get namespace>> at ;
 
-: sset ( value key -- )
+: sset ( value key -- ) session get scope-set ;
-    session get
-    [ namespace>> set-at ] [ (session-changed) ] bi ;
 
-: schange ( key quot -- )
+: schange ( key quot -- ) session get scope-change ; inline
-    session get
-    [ namespace>> swap change-at ] keep
-    (session-changed) ; inline
 
 : init-session ( session -- )
     session [ sessions get init-session* ] with-variable ;
@@ -70,8 +60,7 @@ TUPLE: sessions < server-state-manager domain verify? ;
     } 0|| ;
 
 : empty-session ( -- session )
-    f <session>
+    session empty-scope
-        H{ } clone >>namespace
         remote-host >>client
         user-agent >>user-agent
         dup touch-session ;
@@ -79,18 +68,8 @@ TUPLE: sessions < server-state-manager domain verify? ;
 : begin-session ( -- session )
     empty-session [ init-session ] [ insert-tuple ] [ ] tri ;
 
-! Destructor
-TUPLE: session-saver session ;
-
-C: <session-saver> session-saver
-
-M: session-saver dispose
-    session>> dup changed?>> [
-        [ touch-session ] [ update-tuple ] bi
-    ] [ drop ] if ;
-
 : save-session-after ( session -- )
-    <session-saver> &dispose drop ;
+    sessions get save-scope-after ;
 
 : existing-session ( path session -- response )
     [ session set ] [ save-session-after ] bi

extra/furnace/syndication/syndication.factor

@@ -1,6 +1,6 @@
 ! Copyright (C) 2008 Slava Pestov.
 ! See http://factorcode.org/license.txt for BSD license.
-USING: accessors kernel sequences fry sequences.lib
+USING: accessors kernel sequences fry
 combinators syndication
 http.server.responses http.server.redirection
 furnace furnace.actions ;

extra/html/parser/parser.factor

@@ -122,7 +122,7 @@ SYMBOL: tagstack
 : parse-attributes ( -- hashtable )
     [ (parse-attributes) ] { } make >hashtable ;
 
-: (parse-tag)
+: (parse-tag) ( string -- string' hashtable )
     [
         read-token >lower
         parse-attributes

extra/html/parser/utils/utils.factor

@@ -4,7 +4,7 @@ namespaces prettyprint quotations sequences splitting
 state-parser strings sequences.lib ;
 IN: html.parser.utils
 
-: string-parse-end?
+: string-parse-end? ( -- ? )
     get-next not ;
 
 : take-string* ( match -- string )

extra/http/http-tests.factor

@@ -275,7 +275,7 @@ test-db [
 
 USING: html.components html.elements html.forms
 xml xml.utilities validators
-furnace furnace.flash ;
+furnace furnace.conversations ;
 
 SYMBOL: a
 
@@ -287,7 +287,7 @@ SYMBOL: a
                 [ [ <html> "a" <field> render </html> ] "text/html" <content> ] >>display
                 [ { { "a" [ v-integer ] } } validate-params ] >>validate
                 [ "a" value a set-global URL" " <redirect> ] >>submit
-            <flash-scopes>
+            <conversations>
             <sessions>
             >>default
             add-quit-action

extra/http/http.factor

@@ -25,7 +25,7 @@ IN: http
     [ CHAR: \r assert= read1 CHAR: \n assert= ] when* ;
 
 : (read-header) ( -- alist )
-    [ read-crlf dup f like ] [ parse-header-line ] [ drop ] unfold ;
+    [ read-crlf dup f like ] [ parse-header-line ] [ drop ] produce ;
 
 : process-header ( alist -- assoc )
     f swap [ [ swap or dup ] dip swap ] assoc-map nip
@@ -114,10 +114,13 @@ TUPLE: cookie name value version comment path domain expires max-age http-only s
         ]
     } case ;
 
+: check-cookie-value ( string -- string )
+    [ "Cookie value must not be f" throw ] unless* ;
+
 : (unparse-cookie) ( cookie -- strings )
     [
         dup name>> check-cookie-string >lower
-        over value>> unparse-cookie-value
+        over value>> check-cookie-value unparse-cookie-value
         "$path" over path>> unparse-cookie-value
         "$domain" over domain>> unparse-cookie-value
         drop
@@ -129,7 +132,7 @@ TUPLE: cookie name value version comment path domain expires max-age http-only s
 : unparse-set-cookie ( cookie -- string )
     [
         dup name>> check-cookie-string >lower
-        over value>> unparse-cookie-value
+        over value>> check-cookie-value unparse-cookie-value
         "path" over path>> unparse-cookie-value
         "domain" over domain>> unparse-cookie-value
         "expires" over expires>> unparse-cookie-value

extra/http/parsers/parsers.factor

@@ -1,4 +1,4 @@
-USING: combinators.short-circuit math math.order math.parser kernel combinators.lib
+USING: combinators.short-circuit math math.order math.parser kernel
 sequences sequences.deep peg peg.parsers assocs arrays
 hashtables strings unicode.case namespaces ascii ;
 IN: http.parsers

extra/http/server/cgi/cgi.factor

@@ -14,10 +14,10 @@ IN: http.server.cgi
 
         [ "PATH_TRANSLATED" set ] [ "SCRIPT_FILENAME" set ] bi
 
-        request get url>> path>> "SCRIPT_NAME" set
+        url get path>> "SCRIPT_NAME" set
 
-        request get url>> host>> "SERVER_NAME" set
+        url get host>> "SERVER_NAME" set
-        request get url>> port>> number>string "SERVER_PORT" set
+        url get port>> number>string "SERVER_PORT" set
         "" "PATH_INFO" set
         "" "REMOTE_HOST" set
         "" "REMOTE_ADDR" set
@@ -26,7 +26,7 @@ IN: http.server.cgi
         "" "REMOTE_IDENT" set
 
         request get method>> "REQUEST_METHOD" set
-        request get url>> query>> assoc>query "QUERY_STRING" set
+        url get query>> assoc>query "QUERY_STRING" set
         request get "cookie" header "HTTP_COOKIE" set 
 
         request get "user-agent" header "HTTP_USER_AGENT" set

extra/http/server/dispatchers/dispatchers.factor

@@ -1,7 +1,7 @@
 ! Copyright (C) 2008 Slava Pestov.
 ! See http://factorcode.org/license.txt for BSD license.
 USING: kernel namespaces sequences assocs accessors splitting
-unicode.case http http.server http.server.responses ;
+unicode.case urls http http.server http.server.responses ;
 IN: http.server.dispatchers
 
 TUPLE: dispatcher default responders ;
@@ -35,7 +35,7 @@ TUPLE: vhost-dispatcher default responders ;
     >lower "www." ?head drop "." ?tail drop ;
 
 : find-vhost ( dispatcher -- responder )
-    request get url>> host>> canonical-host over responders>> at*
+    url get host>> canonical-host over responders>> at*
     [ nip ] [ drop default>> ] if ;
 
 M: vhost-dispatcher call-responder* ( path dispatcher -- response )

extra/http/server/redirection/redirection-tests.factor

@@ -1,6 +1,6 @@
 IN: http.server.redirection.tests
 USING: http http.server.redirection urls accessors
-namespaces tools.test present ;
+namespaces tools.test present kernel ;
 
 \ relative-to-request must-infer
 
@@ -11,6 +11,7 @@ namespaces tools.test present ;
             "www.apple.com" >>host
             "/xxx/bar" >>path
             { { "a" "b" } } >>query
+        dup url set
         >>url
     request set
 

extra/http/server/redirection/redirection.factor

@@ -9,7 +9,7 @@ GENERIC: relative-to-request ( url -- url' )
 M: string relative-to-request ;
 
 M: url relative-to-request
-    request get url>>
+    url get
         clone
         f >>query
     swap derive-url ensure-port ;

extra/http/server/server.factor

@@ -81,8 +81,7 @@ GENERIC: write-full-response ( request response -- )
 
 : ensure-domain ( cookie -- cookie )
     [
-        request get url>>
+        url get host>> dup "localhost" =
-        host>> dup "localhost" =
         [ drop ] [ or ] if
     ] change-domain ;
 
@@ -189,7 +188,7 @@ LOG: httpd-header NOTICE
     "/" split harvest ;
 
 : init-request ( request -- )
-    request set
+    [ request set ] [ url>> url set ] bi
     V{ } clone responder-nesting set ;
 
 : dispatch-request ( request -- response )
@@ -224,7 +223,7 @@ LOG: httpd-benchmark DEBUG
 
 : ?benchmark ( quot -- )
     benchmark? get [
-        [ benchmark ] [ first ] bi request get url>> rot 3array
+        [ benchmark ] [ first ] bi url get rot 3array
         httpd-benchmark
     ] [ call ] if ; inline
 
@@ -235,7 +234,7 @@ M: http-server handle-client*
     [
         64 1024 * limit-input
         ?refresh-all
-        read-request
+        [ read-request ] ?benchmark
         [ do-request ] ?benchmark
         [ do-response ] ?benchmark
     ] with-destructors ;

extra/http/server/static/static.factor

@@ -82,12 +82,12 @@ TUPLE: file-responder root hook special allow-listings ;
     "index.html" append-path dup exists? [ drop f ] unless ;
 
 : serve-directory ( filename -- response )
-    request get url>> path>> "/" tail? [
+    url get path>> "/" tail? [
         dup
         find-index [ serve-file ] [ list-directory ] ?if
     ] [
         drop
-        request get url>> clone [ "/" append ] change-path <permanent-redirect>
+        url get clone [ "/" append ] change-path <permanent-redirect>
     ] if ;
 
 : serve-object ( filename -- response )

extra/io/unix/backend/backend.factor

@@ -159,9 +159,9 @@ M: unix io-multiplex ( ms/f -- )
 ! pipe to non-blocking, and read from it instead of the real
 ! stdin. Very crufty, but it will suffice until we get native
 ! threading support at the language level.
-TUPLE: stdin control size data ;
+TUPLE: stdin control size data disposed ;
 
-M: stdin dispose
+M: stdin dispose*
     [
         [ control>> &dispose drop ]
         [ size>> &dispose drop ]
@@ -194,10 +194,10 @@ M: stdin refill
 : data-read-fd ( -- fd ) "stdin_read" f dlsym *uint ;
 
 : <stdin> ( -- stdin )
-    control-write-fd <fd> <output-port>
+    stdin new
-    size-read-fd <fd> init-fd <input-port>
+        control-write-fd <fd> <output-port> >>control
-    data-read-fd <fd>
+        size-read-fd <fd> init-fd <input-port> >>size
-    stdin boa ;
+        data-read-fd <fd> >>data ;
 
 M: unix (init-stdio) ( -- )
     <stdin> <input-port>

extra/io/windows/nt/files/files.factor

@@ -4,7 +4,7 @@ io.windows.nt.backend windows windows.kernel32
 kernel libc math threads system
 alien.c-types alien.arrays alien.strings sequences combinators
 combinators.short-circuit ascii splitting alien strings
-assocs namespaces io.files.private accessors ;
+assocs namespaces io.files.private accessors tr ;
 IN: io.windows.nt.files
 
 M: winnt cwd
@@ -40,9 +40,11 @@ ERROR: not-absolute-path ;
         unicode-prefix prepend
     ] unless ;
 
+TR: normalize-separators "/" "\\" ;
+
 M: winnt normalize-path ( string -- string' )
     (normalize-path)
-    { { CHAR: / CHAR: \\ } } substitute
+    normalize-separators
     prepend-prefix ;
 
 M: winnt CreateFile-flags ( DWORD -- DWORD )

extra/json/writer/writer.factor

@@ -2,7 +2,7 @@
 ! See http://factorcode.org/license.txt for BSD license.
 USING: kernel io.streams.string io strings splitting sequences math 
        math.parser assocs classes words namespaces prettyprint
-       hashtables mirrors ;
+       hashtables mirrors tr ;
 IN: json.writer
 
 #! Writes the object out to a stream in JSON format
@@ -24,10 +24,7 @@ M: number json-print ( num -- )
 M: sequence json-print ( array -- ) 
   CHAR: [ write1 [ >json ] map "," join write CHAR: ] write1 ;
 
-: jsvar-encode ( string -- string )
+TR: jsvar-encode "-" "_" ;
-  #! Convert the string so that it contains characters usable within
-  #! javascript variable names.
-  { { CHAR: - CHAR: _ } } substitute ;
   
 : tuple>fields ( object -- seq )
   <mirror> [

extra/logging/analysis/analysis.factor

@@ -1,8 +1,9 @@
 ! Copyright (C) 2008 Slava Pestov.
 ! See http://factorcode.org/license.txt for BSD license.
 USING: kernel sequences namespaces words assocs logging sorting
-prettyprint io io.styles strings logging.parser calendar.format
+prettyprint io io.styles io.files io.encodings.utf8
-combinators ;
+strings combinators accessors arrays
+logging.server logging.parser calendar.format ;
 IN: logging.analysis
 
 SYMBOL: word-names
@@ -11,11 +12,11 @@ SYMBOL: word-histogram
 SYMBOL: message-histogram
 
 : analyze-entry ( entry -- )
-    dup second ERROR eq? [ dup errors get push ] when
+    dup level>> { ERROR CRITICAL } memq? [ dup errors get push ] when
-    dup second CRITICAL eq? [ dup errors get push ] when
+    1 over word-name>> word-histogram get at+
-    1 over third word-histogram get at+
+    dup word-name>> word-names get member? [
-    dup third word-names get member? [
+        1 over [ level>> ] [ word-name>> ] [ message>> ] tri 3array
-        1 over rest message-histogram get at+
+        message-histogram get at+
     ] when
     drop ;
 
@@ -45,10 +46,10 @@ SYMBOL: message-histogram
 : log-entry. ( entry -- )
     "====== " write
     {
-        [ first (timestamp>string) bl ]
+        [ date>> (timestamp>string) bl ]
-        [ second pprint bl ]
+        [ level>> pprint bl ]
-        [ third write nl ]
+        [ word-name>> write nl ]
-        [ fourth "\n" join print ]
+        [ message>> "\n" join print ]
     } cleave ;
 
 : errors. ( errors -- )
@@ -58,7 +59,7 @@ SYMBOL: message-histogram
     "==== INTERESTING MESSAGES:" print nl
     "Total: " write dup values sum . nl
     [
-        dup second write ": " write third "\n" join write
+        dup level>> write ": " write message>> "\n" join write
     ] histogram.
     nl
     "==== WORDS:" print nl
@@ -69,3 +70,6 @@ SYMBOL: message-histogram
 
 : analyze-log ( lines word-names -- )
     >r parse-log r> analyze-entries analysis. ;
+
+: analyze-log-file ( service word-names -- )
+    >r parse-log-file r> analyze-entries analysis. ;

extra/logging/insomniac/insomniac.factor

@@ -2,7 +2,7 @@
 ! See http://factorcode.org/license.txt for BSD license.
 USING: logging.analysis logging.server logging smtp kernel
 io.files io.streams.string namespaces alarms assocs
-io.encodings.utf8 accessors calendar qualified ;
+io.encodings.utf8 accessors calendar sequences qualified ;
 QUALIFIED: io.sockets
 IN: logging.insomniac
 
@@ -10,11 +10,7 @@ SYMBOL: insomniac-sender
 SYMBOL: insomniac-recipients
 
 : ?analyze-log ( service word-names -- string/f )
-    >r log-path 1 log# dup exists? [
+    [ analyze-log-file ] with-string-writer ;
-        utf8 file-lines r> [ analyze-log ] with-string-writer
-    ] [
-        r> 2drop f
-    ] if ;
 
 : email-subject ( service -- string )
     [
@@ -22,14 +18,14 @@ SYMBOL: insomniac-recipients
     ] "" make ;
 
 : (email-log-report) ( service word-names -- )
-    dupd ?analyze-log dup [
+    dupd ?analyze-log dup empty? [ 2drop ] [
         <email>
             swap >>body
             insomniac-recipients get >>to
             insomniac-sender get >>from
             swap email-subject >>subject
         send-email
-    ] [ 2drop ] if ;
+    ] if ;
 
 \ (email-log-report) NOTICE add-error-logging
 

extra/logging/parser/parser.factor

@@ -1,12 +1,15 @@
 ! Copyright (C) 2008 Slava Pestov.
 ! See http://factorcode.org/license.txt for BSD license.
-USING: accessors parser-combinators memoize kernel sequences
+USING: accessors peg peg.parsers memoize kernel sequences
-logging arrays words strings vectors io io.files
+logging arrays words strings vectors io io.files io.encodings.utf8
 namespaces combinators combinators.lib logging.server
 calendar calendar.format ;
 IN: logging.parser
 
-: string-of ( quot -- parser ) satisfy <!*> [ >string ] <@ ;
+TUPLE: log-entry date level word-name message ;
+
+: string-of ( quot -- parser )
+    satisfy repeat0 [ >string ] action ; inline
 
 SYMBOL: multiline
 
@@ -14,13 +17,13 @@ SYMBOL: multiline
     [ "]" member? not ] string-of [
         dup multiline-header =
         [ drop multiline ] [ rfc3339>timestamp ] if
-    ] <@
+    ] action
     "[" "]" surrounded-by ;
 
 : 'log-level' ( -- parser )
     log-levels [
-        [ name>> token ] keep [ nip ] curry <@
+        [ name>> token ] keep [ nip ] curry action
-    ] map <or-parser> ;
+    ] map choice ;
 
 : 'word-name' ( -- parser )
     [ " :" member? not ] string-of ;
@@ -28,36 +31,42 @@ SYMBOL: multiline
 SYMBOL: malformed
 
 : 'malformed-line' ( -- parser )
-    [ drop t ] string-of [ malformed swap 2array ] <@ ;
+    [ drop t ] string-of
+    [ log-entry new swap >>message malformed >>level ] action ;
 
 : 'log-message' ( -- parser )
-    [ drop t ] string-of [ 1vector ] <@ ;
+    [ drop t ] string-of
+    [ 1vector ] action ;
 
-MEMO: 'log-line' ( -- parser )
+: 'log-line' ( -- parser )
-    'date' " " token <&
+    [
-    'log-level' " " token <& <&>
+        'date' ,
-    'word-name' ": " token <& <:&>
+        " " token hide ,
-    'log-message' <:&>
+        'log-level' ,
-    'malformed-line' <|> ;
+        " " token hide ,
+        'word-name' ,
+        ": " token hide ,
+        'log-message' ,
+    ] seq* [ first4 log-entry boa ] action
+    'malformed-line' 2choice ;
 
-: parse-log-line ( string -- entry )
+PEG: parse-log-line ( string -- entry ) 'log-line' ;
-    'log-line' parse-1 ;
 
 : malformed? ( line -- ? )
-    first malformed eq? ;
+    level>> malformed eq? ;
 
 : multiline? ( line -- ? )
-    first multiline eq? ;
+    level>> multiline eq? ;
 
 : malformed-line ( line -- )
     "Warning: malformed log line:" print
-    second print ;
+    message>> print ;
 
 : add-multiline ( line -- )
     building get empty? [
         "Warning: log begins with multiline entry" print drop
     ] [
-        fourth first building get peek fourth push
+        message>> first building get peek message>> push
     ] if ;
 
 : parse-log ( lines -- entries )
@@ -70,3 +79,7 @@ MEMO: 'log-line' ( -- parser )
             } cond
         ] each
     ] { } make ;
+
+: parse-log-file ( service -- entries )
+    log-path 1 log# dup exists?
+    [ utf8 file-lines parse-log ] [ drop f ] if ;

extra/math/blas/cblas/cblas.factor

@@ -4,6 +4,7 @@ IN: math.blas.cblas
 << "cblas" {
     { [ os macosx? ] [ "libblas.dylib" "cdecl" add-library ] }
     { [ os windows? ] [ "blas.dll" "cdecl" add-library ] }
+    { [ os openbsd? ] [ "libcblas.so" "cdecl" add-library ] }
     [ "libblas.so" "cdecl" add-library ]
 } cond >>
 

extra/math/combinatorics/combinatorics.factor

@@ -16,7 +16,7 @@ IN: math.combinatorics
 !     http://msdn2.microsoft.com/en-us/library/aa302371.aspx
 
 : factoradic ( n -- factoradic )
-    0 [ over 0 > ] [ 1+ [ /mod ] keep swap ] [ ] unfold reverse 2nip ;
+    0 [ over 0 > ] [ 1+ [ /mod ] keep swap ] [ ] produce reverse 2nip ;
 
 : (>permutation) ( seq n -- seq )
     [ [ dupd >= [ 1+ ] when ] curry map ] keep prefix ;

extra/multi-methods/multi-methods.factor

@@ -80,7 +80,7 @@ SYMBOL: total
 : topological-sort ( seq quot -- newseq )
     >r >vector [ dup empty? not ] r>
     [ dupd maximal-element >r over delete-nth r> ] curry
-    [ ] unfold nip ; inline
+    [ ] produce nip ; inline
 
 : classes< ( seq1 seq2 -- lt/eq/gt )
     [

extra/peg/ebnf/ebnf-tests.factor

@@ -7,11 +7,11 @@ USING: kernel tools.test peg peg.ebnf words math math.parser
 IN: peg.ebnf.tests
 
 { T{ ebnf-non-terminal f "abc" } } [
-  "abc" 'non-terminal' parse ast>> 
+  "abc" 'non-terminal' parse 
 ] unit-test
 
 { T{ ebnf-terminal f "55" } } [
-  "'55'" 'terminal' parse ast>> 
+  "'55'" 'terminal' parse 
 ] unit-test
 
 {
@@ -22,7 +22,7 @@ IN: peg.ebnf.tests
      }
   } 
 } [
-  "digit = '1' | '2'" 'rule' parse ast>>
+  "digit = '1' | '2'" 'rule' parse
 ] unit-test
 
 {
@@ -33,7 +33,7 @@ IN: peg.ebnf.tests
      }
   }   
 } [
-  "digit = '1' '2'" 'rule' parse ast>>
+  "digit = '1' '2'" 'rule' parse
 ] unit-test
 
 {
@@ -46,7 +46,7 @@ IN: peg.ebnf.tests
      }
   } 
 } [
-  "one two | three" 'choice' parse ast>>
+  "one two | three" 'choice' parse
 ] unit-test
 
 {
@@ -61,7 +61,7 @@ IN: peg.ebnf.tests
      }
   } 
 } [
-  "one {two | three}" 'choice' parse ast>>
+  "one {two | three}" 'choice' parse
 ] unit-test
 
 {
@@ -81,7 +81,7 @@ IN: peg.ebnf.tests
      }
   } 
 } [
-  "one ((two | three) four)*" 'choice' parse ast>>
+  "one ((two | three) four)*" 'choice' parse
 ] unit-test
 
 {
@@ -93,166 +93,166 @@ IN: peg.ebnf.tests
      }
   } 
 } [
-  "one ( two )? three" 'choice' parse ast>>
+  "one ( two )? three" 'choice' parse
 ] unit-test
 
 { "foo" } [
-  "\"foo\"" 'identifier' parse ast>>
+  "\"foo\"" 'identifier' parse
 ] unit-test
 
 { "foo" } [
-  "'foo'" 'identifier' parse ast>>
+  "'foo'" 'identifier' parse
 ] unit-test
 
 { "foo" } [
-  "foo" 'non-terminal' parse ast>> ebnf-non-terminal-symbol
+  "foo" 'non-terminal' parse ebnf-non-terminal-symbol
 ] unit-test
 
 { "foo" } [
-  "foo]" 'non-terminal' parse ast>> ebnf-non-terminal-symbol
+  "foo]" 'non-terminal' parse ebnf-non-terminal-symbol
 ] unit-test
 
 { V{ "a" "b" } } [
-  "ab" [EBNF foo='a' 'b' EBNF] call ast>> 
+  "ab" [EBNF foo='a' 'b' EBNF] 
 ] unit-test
 
 { V{ 1 "b" } } [
-  "ab" [EBNF foo=('a')[[ drop 1 ]] 'b' EBNF] call ast>> 
+  "ab" [EBNF foo=('a')[[ drop 1 ]] 'b' EBNF]
 ] unit-test
 
 { V{ 1 2 } } [
-  "ab" [EBNF foo=('a') [[ drop 1 ]] ('b') [[ drop 2 ]] EBNF] call ast>> 
+  "ab" [EBNF foo=('a') [[ drop 1 ]] ('b') [[ drop 2 ]] EBNF]
 ] unit-test
 
 { CHAR: A } [
-  "A" [EBNF foo=[A-Z] EBNF] call ast>> 
+  "A" [EBNF foo=[A-Z] EBNF]
 ] unit-test
 
 { CHAR: Z } [
-  "Z" [EBNF foo=[A-Z] EBNF] call ast>> 
+  "Z" [EBNF foo=[A-Z] EBNF]
 ] unit-test
 
-{ f } [
+[
-  "0" [EBNF foo=[A-Z] EBNF] call  
+  "0" [EBNF foo=[A-Z] EBNF]  
-] unit-test
+] must-fail
 
 { CHAR: 0 } [
-  "0" [EBNF foo=[^A-Z] EBNF] call ast>> 
+  "0" [EBNF foo=[^A-Z] EBNF]
 ] unit-test
 
-{ f } [
+[
-  "A" [EBNF foo=[^A-Z] EBNF] call  
+  "A" [EBNF foo=[^A-Z] EBNF]  
-] unit-test
+] must-fail
 
-{ f } [
+[
-  "Z" [EBNF foo=[^A-Z] EBNF] call  
+  "Z" [EBNF foo=[^A-Z] EBNF]  
-] unit-test
+] must-fail
 
 { V{ "1" "+" "foo" } } [
-  "1+1" [EBNF foo='1' '+' '1' [[ drop "foo" ]] EBNF] call ast>>
+  "1+1" [EBNF foo='1' '+' '1' [[ drop "foo" ]] EBNF]
 ] unit-test
 
 { "foo" } [
-  "1+1" [EBNF foo='1' '+' '1' => [[ drop "foo" ]] EBNF] call ast>>
+  "1+1" [EBNF foo='1' '+' '1' => [[ drop "foo" ]] EBNF]
 ] unit-test
 
 { "foo" } [
-  "1+1" [EBNF foo='1' '+' '1' => [[ drop "foo" ]] | '1' '-' '1' => [[ drop "bar" ]] EBNF] call ast>>
+  "1+1" [EBNF foo='1' '+' '1' => [[ drop "foo" ]] | '1' '-' '1' => [[ drop "bar" ]] EBNF]
 ] unit-test
 
 { "bar" } [
-  "1-1" [EBNF foo='1' '+' '1' => [[ drop "foo" ]] | '1' '-' '1' => [[ drop "bar" ]] EBNF] call ast>>
+  "1-1" [EBNF foo='1' '+' '1' => [[ drop "foo" ]] | '1' '-' '1' => [[ drop "bar" ]] EBNF]
 ] unit-test
 
 { 6 } [
-  "4+2" [EBNF num=[0-9] => [[ digit> ]] foo=num:x '+' num:y => [[ x y + ]] EBNF] call ast>>
+  "4+2" [EBNF num=[0-9] => [[ digit> ]] foo=num:x '+' num:y => [[ x y + ]] EBNF]
 ] unit-test
 
 { 6 } [
-  "4+2" [EBNF foo=[0-9]:x '+' [0-9]:y => [[ x digit> y digit> + ]] EBNF] call ast>>
+  "4+2" [EBNF foo=[0-9]:x '+' [0-9]:y => [[ x digit> y digit> + ]] EBNF]
 ] unit-test
 
 { 10 } [
-  { 1 2 3 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF] call ast>>
+  { 1 2 3 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF]
 ] unit-test
 
-{ f } [
+[
-  { "a" 2 3 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF] call 
+  { "a" 2 3 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF] 
-] unit-test
+] must-fail
 
 { 3 } [
-  { 1 2 "a" 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF] call ast>>
+  { 1 2 "a" 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF]
 ] unit-test
 
-{ f } [
+[
-  "ab" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF] call 
+  "ab" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF] 
-] unit-test
+] must-fail
 
 { V{ "a" " " "b" } } [
-  "a b" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF] call ast>>
+  "a b" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF]
 ] unit-test
 
 { V{ "a" "\t" "b" } } [
-  "a\tb" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF] call ast>> 
+  "a\tb" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF]
 ] unit-test
 
 { V{ "a" "\n" "b" } } [
-  "a\nb" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF] call ast>>
+  "a\nb" [EBNF -=" " | "\t" | "\n" foo="a" - "b" EBNF]
 ] unit-test
 
 { V{ "a" f "b" } } [
-  "ab" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF] call ast>>
+  "ab" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF]
 ] unit-test
 
 { V{ "a" " " "b" } } [
-  "a b" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF] call ast>>
+  "a b" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF]
 ] unit-test
 
 
 { V{ "a" "\t" "b" } } [
-  "a\tb" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF] call ast>>
+  "a\tb" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF]
 ] unit-test
 
 { V{ "a" "\n" "b" } } [
-  "a\nb" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF] call ast>>
+  "a\nb" [EBNF -=" " | "\t" | "\n" foo="a" (-)? "b" EBNF]
 ] unit-test
 
 { V{ "a" "b" } } [
-  "ab" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF] call ast>>
+  "ab" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF]
 ] unit-test
 
 { V{ "a" "b" } } [
-  "a\tb" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF] call ast>>
+  "a\tb" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF]
 ] unit-test
 
 { V{ "a" "b" } } [
-  "a\nb" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF] call ast>>
+  "a\nb" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF]
 ] unit-test
 
-{ f } [
+[
-  "axb" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF] call 
+  "axb" [EBNF -=(" " | "\t" | "\n")? => [[ drop ignore ]] foo="a" - "b" EBNF] 
-] unit-test
+] must-fail
 
 { V{ V{ 49 } "+" V{ 49 } } } [ 
   #! Test direct left recursion. 
   #! Using packrat, so first part of expr fails, causing 2nd choice to be used  
-  "1+1" [EBNF num=([0-9])+ expr=expr "+" num | num EBNF] call ast>>
+  "1+1" [EBNF num=([0-9])+ expr=expr "+" num | num EBNF]
 ] unit-test
 
 { V{ V{ V{ 49 } "+" V{ 49 } } "+" V{ 49 } } } [ 
   #! Test direct left recursion. 
   #! Using packrat, so first part of expr fails, causing 2nd choice to be used  
-  "1+1+1" [EBNF num=([0-9])+ expr=expr "+" num | num EBNF] call ast>>
+  "1+1+1" [EBNF num=([0-9])+ expr=expr "+" num | num EBNF]
 ] unit-test
 
 { V{ V{ V{ 49 } "+" V{ 49 } } "+" V{ 49 } } } [ 
   #! Test indirect left recursion. 
   #! Using packrat, so first part of expr fails, causing 2nd choice to be used  
-  "1+1+1" [EBNF num=([0-9])+ x=expr expr=x "+" num | num EBNF] call ast>>
+  "1+1+1" [EBNF num=([0-9])+ x=expr expr=x "+" num | num EBNF]
 ] unit-test
 
 { t } [
-  "abcd='9' | ('8'):x => [[ x ]]" 'ebnf' parse parse-result-remaining empty?
+  "abcd='9' | ('8'):x => [[ x ]]" 'ebnf' (parse) remaining>> empty?
 ] unit-test
 
 EBNF: primary 
@@ -281,133 +281,133 @@ main = Primary
 ;EBNF 
 
 { "this" } [
-  "this" primary ast>>
+  "this" primary
 ] unit-test
 
 { V{ "this" "." "x" } } [
-  "this.x" primary ast>>
+  "this.x" primary
 ] unit-test
 
 { V{ V{ "this" "." "x" } "." "y" } } [
-  "this.x.y" primary ast>>
+  "this.x.y" primary
 ] unit-test
 
 { V{ V{ "this" "." "x" } "." "m" "(" ")" } } [
-  "this.x.m()" primary ast>>
+  "this.x.m()" primary
 ] unit-test
 
 { V{ V{ V{ "x" "[" "i" "]" } "[" "j" "]" } "." "y" } } [
-  "x[i][j].y" primary ast>>
+  "x[i][j].y" primary
 ] unit-test
 
 'ebnf' compile must-infer
 
 { V{ V{ "a" "b" } "c" } } [
-  "abc" [EBNF a="a" "b" foo=(a "c") EBNF] call ast>>
+  "abc" [EBNF a="a" "b" foo=(a "c") EBNF]
 ] unit-test
 
 { V{ V{ "a" "b" } "c" } } [
-  "abc" [EBNF a="a" "b" foo={a "c"} EBNF] call ast>>
+  "abc" [EBNF a="a" "b" foo={a "c"} EBNF]
 ] unit-test
 
 { V{ V{ "a" "b" } "c" } } [
-  "abc" [EBNF a="a" "b" foo=a "c" EBNF] call ast>>
+  "abc" [EBNF a="a" "b" foo=a "c" EBNF]
 ] unit-test
 
-{ f } [
+[
-  "a bc" [EBNF a="a" "b" foo=(a "c") EBNF] call 
+  "a bc" [EBNF a="a" "b" foo=(a "c") EBNF] 
-] unit-test
+] must-fail
 
-{ f } [
+[
-  "a bc" [EBNF a="a" "b" foo=a "c" EBNF] call 
+  "a bc" [EBNF a="a" "b" foo=a "c" EBNF] 
-] unit-test
+] must-fail
 
-{ f } [
+[
-  "a bc" [EBNF a="a" "b" foo={a "c"} EBNF] call
+  "a bc" [EBNF a="a" "b" foo={a "c"} EBNF]
-] unit-test
+] must-fail
 
-{ f } [
+[
-  "ab c" [EBNF a="a" "b" foo=a "c" EBNF] call 
+  "ab c" [EBNF a="a" "b" foo=a "c" EBNF] 
-] unit-test
+] must-fail
 
 { V{ V{ "a" "b" } "c" } } [
-  "ab c" [EBNF a="a" "b" foo={a "c"} EBNF] call ast>>
+  "ab c" [EBNF a="a" "b" foo={a "c"} EBNF]
 ] unit-test
 
-{ f } [
+[
-  "ab c" [EBNF a="a" "b" foo=(a "c") EBNF] call 
+  "ab c" [EBNF a="a" "b" foo=(a "c") EBNF] 
-] unit-test
+] must-fail
 
-{ f } [
+[
-  "a b c" [EBNF a="a" "b" foo=a "c" EBNF] call 
+  "a b c" [EBNF a="a" "b" foo=a "c" EBNF] 
-] unit-test
+] must-fail
 
-{ f } [
+[
-  "a b c" [EBNF a="a" "b" foo=(a "c") EBNF] call 
+  "a b c" [EBNF a="a" "b" foo=(a "c") EBNF] 
-] unit-test
+] must-fail
 
-{ f } [
+[
-  "a b c" [EBNF a="a" "b" foo={a "c"} EBNF] call 
+  "a b c" [EBNF a="a" "b" foo={a "c"} EBNF] 
-] unit-test
+] must-fail
 
 { V{ V{ V{ "a" "b" } "c" } V{ V{ "a" "b" } "c" } } } [
-  "ab cab c" [EBNF a="a" "b" foo={a "c"}* EBNF] call ast>>
+  "ab cab c" [EBNF a="a" "b" foo={a "c"}* EBNF]
 ] unit-test
 
 { V{ } } [
-  "ab cab c" [EBNF a="a" "b" foo=(a "c")* EBNF] call ast>>
+  "ab cab c" [EBNF a="a" "b" foo=(a "c")* EBNF]
 ] unit-test
 
 { V{ V{ V{ "a" "b" } "c" } V{ V{ "a" "b" } "c" } } } [
-  "ab c ab c" [EBNF a="a" "b" foo={a "c"}* EBNF] call ast>>
+  "ab c ab c" [EBNF a="a" "b" foo={a "c"}* EBNF]
 ] unit-test
 
 { V{ } } [
-  "ab c ab c" [EBNF a="a" "b" foo=(a "c")* EBNF] call ast>>
+  "ab c ab c" [EBNF a="a" "b" foo=(a "c")* EBNF]
 ] unit-test
 
 { V{ "a" "a" "a" } } [
-  "aaa" [EBNF a=('a')* b=!('b') a:x => [[ x ]] EBNF] call ast>>
+  "aaa" [EBNF a=('a')* b=!('b') a:x => [[ x ]] EBNF]
 ] unit-test
 
 { t } [
-  "aaa" [EBNF a=('a')* b=!('b') a:x => [[ x ]] EBNF] call ast>>
+  "aaa" [EBNF a=('a')* b=!('b') a:x => [[ x ]] EBNF]
-  "aaa" [EBNF a=('a')* b=!('b') (a):x => [[ x ]] EBNF] call ast>> =
+  "aaa" [EBNF a=('a')* b=!('b') (a):x => [[ x ]] EBNF] =
 ] unit-test
 
 { V{ "a" "a" "a" } } [
-  "aaa" [EBNF a=('a')* b=a:x => [[ x ]] EBNF] call ast>>
+  "aaa" [EBNF a=('a')* b=a:x => [[ x ]] EBNF]
 ] unit-test
 
 { t } [
-  "aaa" [EBNF a=('a')* b=a:x => [[ x ]] EBNF] call ast>>
+  "aaa" [EBNF a=('a')* b=a:x => [[ x ]] EBNF]
-  "aaa" [EBNF a=('a')* b=(a):x => [[ x ]] EBNF] call ast>> =
+  "aaa" [EBNF a=('a')* b=(a):x => [[ x ]] EBNF] =
 ] unit-test
 
 { t } [
-  "number=(digit)+:n 'a'" 'ebnf' parse remaining>> length zero?
+  "number=(digit)+:n 'a'" 'ebnf' (parse) remaining>> length zero?
 ] unit-test
 
 { t } [
-  "number=(digit)+ 'a'" 'ebnf' parse remaining>> length zero?
+  "number=(digit)+ 'a'" 'ebnf' (parse) remaining>> length zero?
 ] unit-test
 
 { t } [
-  "number=digit+ 'a'" 'ebnf' parse remaining>> length zero?
+  "number=digit+ 'a'" 'ebnf' (parse) remaining>> length zero?
 ] unit-test
 
 { t } [
-  "number=digit+:n 'a'" 'ebnf' parse remaining>> length zero?
+  "number=digit+:n 'a'" 'ebnf' (parse) remaining>> length zero?
 ] unit-test
 
 { t } [
-  "foo=(name):n !(keyword) => [[ n ]]" 'rule' parse ast>>
+  "foo=(name):n !(keyword) => [[ n ]]" 'rule' parse
-  "foo=name:n !(keyword) => [[ n ]]" 'rule' parse ast>> =
+  "foo=name:n !(keyword) => [[ n ]]" 'rule' parse =
 ] unit-test
 
 { t } [
-  "foo=!(keyword) (name):n => [[ n ]]" 'rule' parse ast>>
+  "foo=!(keyword) (name):n => [[ n ]]" 'rule' parse
-  "foo=!(keyword) name:n => [[ n ]]" 'rule' parse ast>> =
+  "foo=!(keyword) name:n => [[ n ]]" 'rule' parse =
 ] unit-test
 
 <<
@@ -429,38 +429,38 @@ foo=<foreign any-char> 'd'
 ;EBNF
 
 { "a" } [
-  "a" parser1 ast>>
+  "a" parser1
 ] unit-test
 
 { V{ "a" "b" } } [
-  "ab" parser2 ast>>
+  "ab" parser2
 ] unit-test
 
 { V{ "a" "c" } } [
-  "ac" parser3 ast>>
+  "ac" parser3
 ] unit-test
 
 { V{ CHAR: a "d" } } [
-  "ad" parser4 ast>>
+  "ad" parser4
 ] unit-test
 
 { t } [
- "USING: kernel peg.ebnf ; [EBNF foo='a' '\n'  => [[ drop \"\n\" ]] EBNF]" eval drop t
+ "USING: kernel peg.ebnf ; \"a\\n\" [EBNF foo='a' '\n'  => [[ drop \"\n\" ]] EBNF]" eval drop t
 ] unit-test
 
 [
-  "USING: peg.ebnf ; [EBNF foo='a' foo='b' EBNF]" eval drop
+  "USING: peg.ebnf ; \"ab\" [EBNF foo='a' foo='b' EBNF]" eval drop
 ] must-fail
 
 { t } [
   #! Rule lookup occurs in a namespace. This causes an incorrect duplicate rule
   #! if a var in a namespace is set. This unit test is to remind me to fix this.
-  [ "fail" "foo" set "foo='a'" 'ebnf' parse ast>> transform drop t ] with-scope
+  [ "fail" "foo" set "foo='a'" 'ebnf' parse transform drop t ] with-scope
 ] unit-test
 
 #! Tokenizer tests
 { V{ "a" CHAR: b } } [
-  "ab" [EBNF tokenizer=default foo="a" . EBNF] call ast>>
+  "ab" [EBNF tokenizer=default foo="a" . EBNF]
 ] unit-test
 
 TUPLE: ast-number value ;
@@ -488,7 +488,7 @@ Tok                = Spaces (Number | Special )
                 tokenizer = <foreign a-tokenizer Tok>  foo=. 
                 tokenizer=default baz=. 
                 main = bar foo foo baz 
-          EBNF] call ast>>
+          EBNF]
 ] unit-test
 
 { V{ CHAR: 5 "+" CHAR: 2 } } [
@@ -499,7 +499,7 @@ Tok                = Spaces (Number | Special )
           spaces=space* => [[ ignore ]] 
           tokenizer=spaces (number | operator) 
           main= . . . 
-        EBNF] call ast>> 
+        EBNF]
 ] unit-test
 
 { V{ CHAR: 5 "+" CHAR: 2 } } [
@@ -510,9 +510,13 @@ Tok                = Spaces (Number | Special )
           spaces=space* => [[ ignore ]] 
           tokenizer=spaces (number | operator) 
           main= . . . 
-        EBNF] call ast>> 
+        EBNF]
 ] unit-test
 
 { "++" } [
-  "++--" [EBNF tokenizer=("++" | "--") main="++" EBNF] call ast>>
+  "++--" [EBNF tokenizer=("++" | "--") main="++" EBNF]
+] unit-test
+
+{ "\\" } [
+  "\\" [EBNF foo="\\" EBNF]
 ] unit-test

extra/peg/ebnf/ebnf.factor

@@ -99,6 +99,7 @@ PEG: escaper ( string -- ast )
     "\\t" token [ drop "\t" ] action ,
     "\\n" token [ drop "\n" ] action ,
     "\\r" token [ drop "\r" ] action ,
+    "\\\\" token [ drop "\\" ] action ,
   ] choice* any-char-parser 2array choice repeat0 ;
 
 : replace-escapes ( string -- string )
@@ -503,7 +504,7 @@ M: ebnf-non-terminal (transform) ( ast -- parser )
   ] [ ] make box ;
 
 : transform-ebnf ( string -- object )
-  'ebnf' parse parse-result-ast transform ;
+  'ebnf' parse transform ;
 
 : check-parse-result ( result -- result )
   dup [
@@ -517,12 +518,18 @@ M: ebnf-non-terminal (transform) ( ast -- parser )
     "Could not parse EBNF" throw
   ] if ;
 
-: ebnf>quot ( string -- hashtable quot )
+: parse-ebnf ( string -- hashtable )
-  'ebnf' parse check-parse-result 
+  'ebnf' (parse) check-parse-result ast>> transform ;
-  parse-result-ast transform dup dup parser [ main swap at compile ] with-variable
-  [ compiled-parse ] curry [ with-scope ] curry ;
 
-: [EBNF "EBNF]" reset-tokenizer parse-multiline-string ebnf>quot nip parsed reset-tokenizer ; parsing
+: ebnf>quot ( string -- hashtable quot )
+  parse-ebnf dup dup parser [ main swap at compile ] with-variable
+  [ compiled-parse ] curry [ with-scope ast>> ] curry ;
+
+: <EBNF "EBNF>" reset-tokenizer parse-multiline-string parse-ebnf main swap at  
+  parsed reset-tokenizer ; parsing
+
+: [EBNF "EBNF]" reset-tokenizer parse-multiline-string ebnf>quot nip 
+  parsed \ call parsed reset-tokenizer ; parsing
 
 : EBNF: 
   reset-tokenizer CREATE-WORD dup ";EBNF" parse-multiline-string  

extra/peg/expr/expr-tests.factor

@@ -5,21 +5,21 @@ USING: kernel tools.test peg peg.expr multiline sequences ;
 IN: peg.expr.tests
 
 { 5 } [
-  "2+3" eval-expr 
+  "2+3" expr 
 ] unit-test
 
 { 6 } [
-  "2*3" eval-expr 
+  "2*3" expr 
 ] unit-test
 
 { 14 } [
-  "2+3*4" eval-expr 
+  "2+3*4" expr 
 ] unit-test
 
 { 17 } [
-  "2+3*4+3" eval-expr 
+  "2+3*4+3" expr 
 ] unit-test
 
 { 23 } [
-  "2+3*(4+3)" eval-expr 
+  "2+3*(4+3)" expr 
 ] unit-test

extra/peg/expr/expr.factor

@@ -18,7 +18,3 @@ exp      =   exp "+" fac    => [[ first3 nip + ]]
            | exp "-" fac    => [[ first3 nip - ]]
            | fac
 ;EBNF
-
-: eval-expr ( string -- number )
-  expr ast>> ;
-

extra/peg/javascript/ast/ast.factor

@@ -7,7 +7,7 @@ TUPLE: ast-keyword value ;
 TUPLE: ast-name value ;
 TUPLE: ast-number value ;
 TUPLE: ast-string value ;
-TUPLE: ast-regexp value ;
+TUPLE: ast-regexp body flags ;
 TUPLE: ast-cond-expr condition then else ;
 TUPLE: ast-set lhs rhs ;
 TUPLE: ast-get value ;
@@ -38,5 +38,6 @@ TUPLE: ast-continue ;
 TUPLE: ast-throw e ;
 TUPLE: ast-try t e c f ;
 TUPLE: ast-return e ;
+TUPLE: ast-with expr body ;
 TUPLE: ast-case c cs ;
 TUPLE: ast-default cs ;

extra/peg/javascript/javascript.factor

@@ -4,8 +4,4 @@ USING: kernel accessors peg.javascript.tokenizer peg.javascript.parser ;
 IN: peg.javascript
 
 : parse-javascript ( string -- ast )
-  javascript [
+  javascript ;
-    ast>>
-  ] [
-    "Unable to parse JavaScript" throw
-  ] if* ;

extra/peg/javascript/parser/parser-tests.factor

@@ -2,7 +2,7 @@
 ! See http://factorcode.org/license.txt for BSD license.
 !
 USING: kernel tools.test peg peg.javascript.ast peg.javascript.parser 
-       accessors multiline sequences math ;
+       accessors multiline sequences math peg.ebnf ;
 IN: peg.javascript.parser.tests
 
 \ javascript must-infer
@@ -23,14 +23,14 @@ IN: peg.javascript.parser.tests
       }
   }
 } [
-  "123; 'hello'; foo(x);" javascript ast>>
+  "123; 'hello'; foo(x);" javascript
 ] unit-test
 
 { t } [ 
 <"
 var x=5
 var y=10
-"> javascript remaining>> length zero?
+"> main \ javascript rule (parse) remaining>> length zero?
 ] unit-test
 
 
@@ -41,7 +41,7 @@ function foldl(f, initial, seq) {
      initial = f(initial, seq[i]);
    return initial;
 }
-"> javascript remaining>> length zero?
+"> main \ javascript rule (parse) remaining>> length zero?
 ] unit-test
 
 { t } [ 
@@ -52,6 +52,6 @@ ParseState.prototype.from = function(index) {
     r.length = this.length - index;
     return r;
 }
-"> javascript remaining>> length zero?
+"> main \ javascript rule (parse) remaining>> length zero?
 ] unit-test
 

extra/peg/javascript/parser/parser.factor

@@ -26,9 +26,9 @@ End               = !(.)
 Space             = " " | "\t" | "\n" 
 Spaces            = Space* => [[ ignore ]]
 Name               = . ?[ ast-name?   ]?   => [[ value>> ]] 
-Number             = . ?[ ast-number? ]?   => [[ value>> ]]
+Number             = . ?[ ast-number? ]?
-String             = . ?[ ast-string? ]?   => [[ value>> ]]
+String             = . ?[ ast-string? ]?
-RegExp             = . ?[ ast-regexp? ]?   => [[ value>> ]]
+RegExp             = . ?[ ast-regexp? ]?   
 SpacesNoNl         = (!(nl) Space)* => [[ ignore ]]
 
 Expr               =   OrExpr:e "?" Expr:t ":" Expr:f   => [[ e t f ast-cond-expr boa ]]
@@ -40,22 +40,77 @@ Expr               =   OrExpr:e "?" Expr:t ":" Expr:f   => [[ e t f ast-cond-exp
                      | OrExpr:e "%=" Expr:rhs           => [[ e rhs "%" ast-mset boa ]]
                      | OrExpr:e "&&=" Expr:rhs          => [[ e rhs "&&" ast-mset boa ]]
                      | OrExpr:e "||=" Expr:rhs          => [[ e rhs "||" ast-mset boa ]]
+                     | OrExpr:e "^=" Expr:rhs           => [[ e rhs "^" ast-mset boa ]]
+                     | OrExpr:e "&=" Expr:rhs           => [[ e rhs "&" ast-mset boa ]]
+                     | OrExpr:e "|=" Expr:rhs           => [[ e rhs "|" ast-mset boa ]]
+                     | OrExpr:e "<<=" Expr:rhs          => [[ e rhs "<<" ast-mset boa ]]
+                     | OrExpr:e ">>=" Expr:rhs          => [[ e rhs ">>" ast-mset boa ]]
+                     | OrExpr:e ">>>=" Expr:rhs         => [[ e rhs ">>>" ast-mset boa ]]
                      | OrExpr:e                         => [[ e ]]
 
+ExprNoIn           =   OrExprNoIn:e "?" ExprNoIn:t ":" ExprNoIn:f => [[ e t f ast-cond-expr boa ]]
+                     | OrExprNoIn:e "=" ExprNoIn:rhs              => [[ e rhs ast-set boa ]]
+                     | OrExprNoIn:e "+=" ExprNoIn:rhs             => [[ e rhs "+" ast-mset boa ]]
+                     | OrExprNoIn:e "-=" ExprNoIn:rhs             => [[ e rhs "-" ast-mset boa ]]
+                     | OrExprNoIn:e "*=" ExprNoIn:rhs             => [[ e rhs "*" ast-mset boa ]]
+                     | OrExprNoIn:e "/=" ExprNoIn:rhs             => [[ e rhs "/" ast-mset boa ]]
+                     | OrExprNoIn:e "%=" ExprNoIn:rhs             => [[ e rhs "%" ast-mset boa ]]
+                     | OrExprNoIn:e "&&=" ExprNoIn:rhs            => [[ e rhs "&&" ast-mset boa ]]
+                     | OrExprNoIn:e "||=" ExprNoIn:rhs            => [[ e rhs "||" ast-mset boa ]]
+                     | OrExprNoIn:e "^=" ExprNoIn:rhs             => [[ e rhs "^" ast-mset boa ]]
+                     | OrExprNoIn:e "&=" ExprNoIn:rhs             => [[ e rhs "&" ast-mset boa ]]
+                     | OrExprNoIn:e "|=" ExprNoIn:rhs             => [[ e rhs "|" ast-mset boa ]]
+                     | OrExprNoIn:e "<<=" ExprNoIn:rhs            => [[ e rhs "<<" ast-mset boa ]]
+                     | OrExprNoIn:e ">>=" ExprNoIn:rhs            => [[ e rhs ">>" ast-mset boa ]]
+                     | OrExprNoIn:e ">>>=" ExprNoIn:rhs           => [[ e rhs ">>>" ast-mset boa ]]
+                     | OrExprNoIn:e                               => [[ e ]]
+
 OrExpr             =   OrExpr:x "||" AndExpr:y          => [[ x y "||" ast-binop boa ]]
                      | AndExpr
-AndExpr            =   AndExpr:x "&&" EqExpr:y          => [[ x y "&&" ast-binop boa ]]
+OrExprNoIn         =   OrExprNoIn:x "||" AndExprNoIn:y  => [[ x y "||" ast-binop boa ]]
+                     | AndExprNoIn
+AndExpr            =   AndExpr:x "&&" BitOrExpr:y       => [[ x y "&&" ast-binop boa ]]
+                     | BitOrExpr
+AndExprNoIn        =   AndExprNoIn:x "&&" BitOrExprNoIn:y => [[ x y "&&" ast-binop boa ]]
+                     | BitOrExprNoIn
+BitOrExpr          =   BitOrExpr:x "|" BitXORExpr:y     => [[ x y "|" ast-binop boa ]]
+                     | BitXORExpr
+BitOrExprNoIn      =   BitOrExprNoIn:x "|" BitXORExprNoIn:y => [[ x y "|" ast-binop boa ]]
+                     | BitXORExprNoIn
+BitXORExpr         =   BitXORExpr:x "^" BitANDExpr:y    => [[ x y "^" ast-binop boa ]]
+                     | BitANDExpr
+BitXORExprNoIn     =   BitXORExprNoIn:x "^" BitANDExprNoIn:y => [[ x y "^" ast-binop boa ]]
+                     | BitANDExprNoIn
+BitANDExpr         =   BitANDExpr:x "&" EqExpr:y        => [[ x y "&" ast-binop boa ]]
                      | EqExpr
+BitANDExprNoIn     =   BitANDExprNoIn:x "&" EqExprNoIn:y => [[ x y "&" ast-binop boa ]]
+                     | EqExprNoIn
 EqExpr             =   EqExpr:x "==" RelExpr:y          => [[ x y "==" ast-binop boa ]]
                      | EqExpr:x "!=" RelExpr:y          => [[ x y "!=" ast-binop boa ]]
                      | EqExpr:x "===" RelExpr:y         => [[ x y "===" ast-binop boa ]]
                      | EqExpr:x "!==" RelExpr:y         => [[ x y "!==" ast-binop boa ]]
                      | RelExpr
-RelExpr            =   RelExpr:x ">" AddExpr:y          => [[ x y ">" ast-binop boa ]]
+EqExprNoIn         =   EqExprNoIn:x "==" RelExprNoIn:y    => [[ x y "==" ast-binop boa ]]
-                     | RelExpr:x ">=" AddExpr:y         => [[ x y ">=" ast-binop boa ]]
+                     | EqExprNoIn:x "!=" RelExprNoIn:y    => [[ x y "!=" ast-binop boa ]]
-                     | RelExpr:x "<" AddExpr:y          => [[ x y "<" ast-binop boa ]]
+                     | EqExprNoIn:x "===" RelExprNoIn:y   => [[ x y "===" ast-binop boa ]]
-                     | RelExpr:x "<=" AddExpr:y         => [[ x y "<=" ast-binop boa ]]
+                     | EqExprNoIn:x "!==" RelExprNoIn:y   => [[ x y "!==" ast-binop boa ]]
-                     | RelExpr:x "instanceof" AddExpr:y => [[ x y "instanceof" ast-binop boa ]]
+                     | RelExprNoIn
+RelExpr            =   RelExpr:x ">" ShiftExpr:y          => [[ x y ">" ast-binop boa ]]
+                     | RelExpr:x ">=" ShiftExpr:y         => [[ x y ">=" ast-binop boa ]]
+                     | RelExpr:x "<" ShiftExpr:y          => [[ x y "<" ast-binop boa ]]
+                     | RelExpr:x "<=" ShiftExpr:y         => [[ x y "<=" ast-binop boa ]]
+                     | RelExpr:x "instanceof" ShiftExpr:y => [[ x y "instanceof" ast-binop boa ]]
+                     | RelExpr:x "in" ShiftExpr:y         => [[ x y "in" ast-binop boa ]]
+                     | ShiftExpr
+RelExprNoIn        =   RelExprNoIn:x ">" ShiftExpr:y          => [[ x y ">" ast-binop boa ]]
+                     | RelExprNoIn:x ">=" ShiftExpr:y         => [[ x y ">=" ast-binop boa ]]
+                     | RelExprNoIn:x "<" ShiftExpr:y          => [[ x y "<" ast-binop boa ]]
+                     | RelExprNoIn:x "<=" ShiftExpr:y         => [[ x y "<=" ast-binop boa ]]
+                     | RelExprNoIn:x "instanceof" ShiftExpr:y => [[ x y "instanceof" ast-binop boa ]]
+                     | ShiftExpr
+ShiftExpr          =   ShiftExpr:x "<<" AddExpr:y       => [[ x y "<<" ast-binop boa ]]
+                     | ShiftExpr:x ">>>" AddExpr:y      => [[ x y ">>>" ast-binop boa ]]
+                     | ShiftExpr:x ">>" AddExpr:y       => [[ x y ">>" ast-binop boa ]]
                      | AddExpr
 AddExpr            =   AddExpr:x "+" MulExpr:y          => [[ x y "+" ast-binop boa ]]
                      | AddExpr:x "-" MulExpr:y          => [[ x y "-" ast-binop boa ]]
@@ -64,14 +119,14 @@ MulExpr            =   MulExpr:x "*" Unary:y            => [[ x y "*" ast-binop
                      | MulExpr:x "/" Unary:y            => [[ x y "/" ast-binop boa ]]
                      | MulExpr:x "%" Unary:y            => [[ x y "%" ast-binop boa ]]
                      | Unary
-Unary              =   "-" Postfix:p                    => [[ p "-" ast-unop boa ]]
+Unary              =   "-" Unary:p                      => [[ p "-" ast-unop boa ]]
-                     | "+" Postfix:p                    => [[ p ]]
+                     | "+" Unary:p                      => [[ p ]]
-                     | "++" Postfix:p                   => [[ p "++" ast-preop boa ]]
+                     | "++" Unary:p                     => [[ p "++" ast-preop boa ]]
-                     | "--" Postfix:p                   => [[ p "--" ast-preop boa ]]
+                     | "--" Unary:p                     => [[ p "--" ast-preop boa ]]
-                     | "!" Postfix:p                    => [[ p "!" ast-unop boa ]]
+                     | "!" Unary:p                      => [[ p "!" ast-unop boa ]]
-                     | "typeof" Postfix:p               => [[ p "typeof" ast-unop boa ]]
+                     | "typeof" Unary:p                 => [[ p "typeof" ast-unop boa ]]
-                     | "void" Postfix:p                 => [[ p "void" ast-unop boa ]]
+                     | "void" Unary:p                   => [[ p "void" ast-unop boa ]]
-                     | "delete" Postfix:p               => [[ p "delete" ast-unop boa ]]
+                     | "delete" Unary:p                 => [[ p "delete" ast-unop boa ]]
                      | Postfix
 Postfix            =   PrimExpr:p SpacesNoNl "++"       => [[ p "++" ast-postop boa ]]
                      | PrimExpr:p SpacesNoNl "--"       => [[ p "--" ast-postop boa ]]
@@ -85,15 +140,15 @@ PrimExpr           =   PrimExpr:p "[" Expr:i "]"             => [[ i p ast-getp
 PrimExprHd         =   "(" Expr:e ")"                        => [[ e ]]
                      | "this"                                => [[ ast-this boa ]]
                      | Name                                  => [[ ast-get boa ]]
-                     | Number                                => [[ ast-number boa ]]
+                     | Number
-                     | String                                => [[ ast-string boa ]]
+                     | String
-                     | RegExp                                => [[ ast-regexp boa ]]
+                     | RegExp
                      | "function" FuncRest:fr                => [[ fr ]]
                      | "new" PrimExpr:n "(" Args:as ")"      => [[ n as ast-new boa ]]
                      | "new" PrimExpr:n                      => [[ n f  ast-new boa ]]
                      | "[" Args:es "]"                       => [[ es ast-array boa ]]
                      | Json
-JsonBindings        = (JsonBinding ("," JsonBinding => [[ second ]])* => [[ first2 swap prefix ]])?
+JsonBindings       = (JsonBinding ("," JsonBinding => [[ second ]])* => [[ first2 swap prefix ]])?
 Json               = "{" JsonBindings:bs "}"                  => [[ bs ast-json boa ]]
 JsonBinding        = JsonPropName:n ":" Expr:v               => [[ n v ast-binding boa ]]
 JsonPropName       = Name | Number | String | RegExp
@@ -105,15 +160,15 @@ Binding            =   Name:n "=" Expr:v                      => [[ n v ast-var
                      | Name:n                                 => [[ n "undefined" ast-get boa ast-var boa ]]
 Block              = "{" SrcElems:ss "}"                      => [[ ss ]]
 Bindings           = (Binding ("," Binding => [[ second ]])* => [[ first2 swap prefix ]])?
-For1               =   "var" Binding => [[ second ]] 
+For1               =   "var" Bindings => [[ second ]] 
-                     | Expr 
+                     | ExprNoIn 
                      | Spaces => [[ "undefined" ast-get boa ]] 
 For2               =   Expr
                      | Spaces => [[ "true" ast-get boa ]] 
 For3               =   Expr
                      | Spaces => [[ "undefined" ast-get boa ]] 
 ForIn1             =   "var" Name:n => [[ n "undefined" ast-get boa ast-var boa ]]
-                     | Expr
+                     | PrimExprHd
 Switch1            =   "case" Expr:c ":" SrcElems:cs => [[ c cs ast-case boa ]]
                      | "default" ":" SrcElems:cs => [[ cs ast-default boa ]]  
 SwitchBody         = Switch1*
@@ -134,6 +189,7 @@ Stmt               =   Block
                      | "try" Block:t "catch" "(" Name:e ")" Block:c Finally:f => [[ t e c f ast-try boa ]]
                      | "return" Expr:e Sc                            => [[ e ast-return boa ]]
                      | "return" Sc                                   => [[ "undefined" ast-get boa ast-return boa ]]
+                     | "with" "(" Expr:e ")" Stmt:b                  => [[ e b ast-with boa ]]
                      | Expr:e Sc                                     => [[ e ]]
                      | ";"                                           => [[ "undefined" ast-get boa ]]
 SrcElem            =   "function" Name:n FuncRest:f                  => [[ n f ast-var boa ]]

extra/peg/javascript/tokenizer/tokenizer-tests.factor

@@ -19,5 +19,9 @@ IN: peg.javascript.tokenizer.tests
     ";"
   }    
 } [
-  "123; 'hello'; foo(x);" tokenize-javascript ast>>
+  "123; 'hello'; foo(x);" tokenize-javascript
 ] unit-test
+
+{ V{ T{ ast-regexp f "<(w+)[^>]*?)/>" "g" } } } [
+  "/<(\\w+)[^>]*?)\\/>/g" tokenize-javascript
+] unit-test

extra/peg/javascript/tokenizer/tokenizer.factor

@@ -57,13 +57,23 @@ StringChars3       = (EscapeChar | !("'") .)* => [[ >string ]]
 Str                =   '"""' StringChars1:cs '"""' => [[ cs ast-string boa ]]
                      | '"' StringChars2:cs '"' => [[ cs ast-string boa ]]
                      | "'" StringChars3:cs "'" => [[ cs ast-string boa ]]
-RegExpBody         = (!("/" | "\n" | "\r") .)* => [[ >string ]]
+RegExpFlags        = NameRest* => [[ >string ]]
-RegExp             = "/" RegExpBody:r "/" => [[ r ast-regexp boa ]]
+NonTerminator      = !("\n" | "\r") .
-Special            =   "("   | ")"   | "{"   | "}"   | "["   | "]"   | ","   | ";"
+BackslashSequence  = "\\" NonTerminator => [[ second ]]
-                     | "?"   | ":"   | "!==" | "!="  | "===" | "=="  | "="   | ">="
+RegExpFirstChar    =   !("*" | "\\" | "/") NonTerminator
-                     | ">"   | "<="  | "<"   | "++"  | "+="  | "+"   | "--"  | "-="
+                     | BackslashSequence
-                     | "-"   | "*="  | "*"   | "/="  | "/"   | "%="  | "%"   | "&&="
+RegExpChar         =   !("\\" | "/") NonTerminator
-                     | "&&"  | "||=" | "||"  | "."   | "!"
+                     | BackslashSequence
+RegExpChars        = RegExpChar*
+RegExpBody         = RegExpFirstChar RegExpChars => [[ first2 swap prefix >string ]]
+RegExp             = "/" RegExpBody:b "/" RegExpFlags:fl => [[ b fl ast-regexp boa ]]
+Special            =   "("    | ")"   | "{"   | "}"   | "["   | "]"   | ","   | ";"
+                     | "?"    | ":"   | "!==" | "!="  | "===" | "=="  | "="   | ">="
+                     | ">>>=" | ">>>" | ">>=" | ">>"  | ">"   | "<="  | "<<=" | "<<"
+                     | "<"    | "++"  | "+="  | "+"   | "--"  | "-="  | "-"   | "*="
+                     | "*"    | "/="  | "/"   | "%="  | "%"   | "&&=" | "&&"  | "||="
+                     | "||"   | "."   | "!"   | "&="  | "&"   | "|="  | "|"   | "^="
+                     | "^"
 Tok                = Spaces (Name | Keyword | Number | Str | RegExp | Special )
 Toks               = Tok* Spaces 
 ;EBNF

extra/peg/parsers/parsers-tests.factor

@@ -1,54 +1,51 @@
-USING: kernel peg peg.parsers tools.test ;
+USING: kernel peg peg.parsers tools.test accessors ;
 IN: peg.parsers.tests
 
-[ V{ "a" } ]
+{ V{ "a" } }
-[ "a" "a" token "," token list-of parse parse-result-ast ] unit-test
+[ "a" "a" token "," token list-of parse ] unit-test
 
-[ V{ "a" "a" "a" "a" } ]
+{ V{ "a" "a" "a" "a" } }
-[ "a,a,a,a" "a" token "," token list-of parse parse-result-ast ] unit-test
+[ "a,a,a,a" "a" token "," token list-of parse ] unit-test
 
-[ f ]
+[ "a" "a" token "," token list-of-many parse ] must-fail
-[ "a" "a" token "," token list-of-many parse ] unit-test
 
-[ V{ "a" "a" "a" "a" } ]
+{ V{ "a" "a" "a" "a" } }
-[ "a,a,a,a" "a" token "," token list-of-many parse parse-result-ast ] unit-test
+[ "a,a,a,a" "a" token "," token list-of-many parse ] unit-test
 
-[ f ]
+[ "aaa" "a" token 4 exactly-n parse ] must-fail
-[ "aaa" "a" token 4 exactly-n parse ] unit-test
 
-[ V{ "a" "a" "a" "a" } ]
+{ V{ "a" "a" "a" "a" } }
-[ "aaaa" "a" token 4 exactly-n parse parse-result-ast ] unit-test
+[ "aaaa" "a" token 4 exactly-n parse ] unit-test
 
-[ f ]
+[ "aaa" "a" token 4 at-least-n parse ] must-fail
-[ "aaa" "a" token 4 at-least-n parse ] unit-test
 
-[ V{ "a" "a" "a" "a" } ]
+{ V{ "a" "a" "a" "a" } }
-[ "aaaa" "a" token 4 at-least-n parse parse-result-ast ] unit-test
+[ "aaaa" "a" token 4 at-least-n parse ] unit-test
 
-[ V{ "a" "a" "a" "a" "a" } ]
+{ V{ "a" "a" "a" "a" "a" } }
-[ "aaaaa" "a" token 4 at-least-n parse parse-result-ast ] unit-test
+[ "aaaaa" "a" token 4 at-least-n parse ] unit-test
 
-[ V{ "a" "a" "a" "a" } ]
+{ V{ "a" "a" "a" "a" } }
-[ "aaaa" "a" token 4 at-most-n parse parse-result-ast ] unit-test
+[ "aaaa" "a" token 4 at-most-n parse ] unit-test
 
-[ V{ "a" "a" "a" "a" } ]
+{ V{ "a" "a" "a" "a" } }
-[ "aaaaa" "a" token 4 at-most-n parse parse-result-ast ] unit-test
+[ "aaaaa" "a" token 4 at-most-n parse ] unit-test
 
-[ V{ "a" "a" "a" } ]
+{ V{ "a" "a" "a" } }
-[ "aaa" "a" token 3 4 from-m-to-n parse parse-result-ast ] unit-test
+[ "aaa" "a" token 3 4 from-m-to-n parse ] unit-test
 
-[ V{ "a" "a" "a" "a" } ]
+{ V{ "a" "a" "a" "a" } }
-[ "aaaa" "a" token 3 4 from-m-to-n parse parse-result-ast ] unit-test
+[ "aaaa" "a" token 3 4 from-m-to-n parse ] unit-test
 
-[ V{ "a" "a" "a" "a" } ]
+{ V{ "a" "a" "a" "a" } }
-[ "aaaaa" "a" token 3 4 from-m-to-n parse parse-result-ast ] unit-test
+[ "aaaaa" "a" token 3 4 from-m-to-n parse ] unit-test
 
-[ 97 ]
+{ 97 }
-[ "a" any-char parse parse-result-ast ] unit-test
+[ "a" any-char parse ] unit-test
 
-[ V{ } ]
+{ V{ } }
-[ "" epsilon parse parse-result-ast ] unit-test
+[ "" epsilon parse ] unit-test
 
 { "a" } [
-  "a" "a" token just parse parse-result-ast
+  "a" "a" token just parse
 ] unit-test

extra/peg/parsers/parsers.factor

@@ -3,7 +3,7 @@
 USING: kernel sequences strings namespaces math assocs shuffle 
      vectors arrays math.parser 
      unicode.categories sequences.deep peg peg.private 
-     peg.search math.ranges words memoize ;
+     peg.search math.ranges words ;
 IN: peg.parsers
 
 TUPLE: just-parser p1 ;
@@ -19,8 +19,8 @@ TUPLE: just-parser p1 ;
 M: just-parser (compile) ( parser -- quot )
   just-parser-p1 compiled-parser just-pattern curry ;
 
-MEMO: just ( parser -- parser )
+: just ( parser -- parser )
-  just-parser boa init-parser ;
+  just-parser boa wrap-peg ;
 
 : 1token ( ch -- parser ) 1string token ;
 
@@ -45,10 +45,10 @@ MEMO: just ( parser -- parser )
 
 PRIVATE>
 
-MEMO: exactly-n ( parser n -- parser' )
+: exactly-n ( parser n -- parser' )
   swap <repetition> seq ;
 
-MEMO: at-most-n ( parser n -- parser' )
+: at-most-n ( parser n -- parser' )
   dup zero? [
     2drop epsilon
   ] [
@@ -56,15 +56,15 @@ MEMO: at-most-n ( parser n -- parser' )
     -rot 1- at-most-n 2choice
   ] if ;
 
-MEMO: at-least-n ( parser n -- parser' )
+: at-least-n ( parser n -- parser' )
   dupd exactly-n swap repeat0 2seq
   [ flatten-vectors ] action ;
 
-MEMO: from-m-to-n ( parser m n -- parser' )
+: from-m-to-n ( parser m n -- parser' )
   >r [ exactly-n ] 2keep r> swap - at-most-n 2seq
   [ flatten-vectors ] action ;
 
-MEMO: pack ( begin body end -- parser )
+: pack ( begin body end -- parser )
   >r >r hide r> r> hide 3seq [ first ] action ;
 
 : surrounded-by ( parser begin end -- parser' )

extra/peg/peg-tests.factor

@@ -5,99 +5,99 @@ USING: kernel tools.test strings namespaces arrays sequences
        peg peg.private accessors words math accessors ;
 IN: peg.tests
 
-{ f } [
+[
   "endbegin" "begin" token parse
-] unit-test
+] must-fail
 
 { "begin" "end" } [
-  "beginend" "begin" token parse 
+  "beginend" "begin" token (parse) 
   { ast>> remaining>> } get-slots
   >string
 ] unit-test
 
-{ f } [
+[
   "" CHAR: a CHAR: z range parse
-] unit-test
+] must-fail
 
-{ f } [
+[
   "1bcd" CHAR: a CHAR: z range parse
-] unit-test
+] must-fail
 
 { CHAR: a } [
-  "abcd" CHAR: a CHAR: z range parse ast>>
+  "abcd" CHAR: a CHAR: z range parse
 ] unit-test
 
 { CHAR: z } [
-  "zbcd" CHAR: a CHAR: z range parse ast>>
+  "zbcd" CHAR: a CHAR: z range parse
 ] unit-test
 
-{ f } [
+[
   "bad" "a" token "b" token 2array seq parse
-] unit-test
+] must-fail
 
 { V{ "g" "o" } } [
-  "good" "g" token "o" token 2array seq parse ast>>
+  "good" "g" token "o" token 2array seq parse
 ] unit-test
 
 { "a" } [
-  "abcd" "a" token "b" token 2array choice parse ast>>
+  "abcd" "a" token "b" token 2array choice parse
 ] unit-test
 
 { "b" } [
-  "bbcd" "a" token "b" token 2array choice parse ast>>
+  "bbcd" "a" token "b" token 2array choice parse
 ] unit-test
 
-{ f } [
+[
   "cbcd" "a" token "b" token 2array choice parse 
-] unit-test
+] must-fail
 
-{ f } [
+[
   "" "a" token "b" token 2array choice parse 
+] must-fail
+
+{ 0 } [
+  "" "a" token repeat0 parse length
 ] unit-test
 
 { 0 } [
-  "" "a" token repeat0 parse ast>> length
+  "b" "a" token repeat0 parse length
-] unit-test
-
-{ 0 } [
-  "b" "a" token repeat0 parse ast>> length
 ] unit-test
 
 { V{ "a" "a" "a" } } [
-  "aaab" "a" token repeat0 parse ast>> 
+  "aaab" "a" token repeat0 parse 
 ] unit-test
 
-{ f } [
+[
   "" "a" token repeat1 parse 
-] unit-test
+] must-fail
 
-{ f } [
+[
   "b" "a" token repeat1 parse 
-] unit-test
+] must-fail
 
 { V{ "a" "a" "a" } } [
-  "aaab" "a" token repeat1 parse ast>>
+  "aaab" "a" token repeat1 parse
 ] unit-test
 
 { V{ "a" "b" } } [ 
-  "ab" "a" token optional "b" token 2array seq parse ast>> 
+  "ab" "a" token optional "b" token 2array seq parse 
 ] unit-test
 
 { V{ f "b" } } [ 
-  "b" "a" token optional "b" token 2array seq parse ast>> 
+  "b" "a" token optional "b" token 2array seq parse 
 ] unit-test
 
-{ f } [ 
+[ 
   "cb" "a" token optional "b" token 2array seq parse  
-] unit-test
+] must-fail
 
 { V{ CHAR: a CHAR: b } } [
-  "ab" "a" token ensure CHAR: a CHAR: z range dup 3array seq parse ast>>
+  "ab" "a" token ensure CHAR: a CHAR: z range dup 3array seq parse
 ] unit-test
 
-{ f } [
+[
   "bb" "a" token ensure CHAR: a CHAR: z range 2array seq parse 
-] unit-test
+] must-fail
 
 { t } [
   "a+b" 
@@ -117,47 +117,47 @@ IN: peg.tests
   parse [ t ] [ f ] if
 ] unit-test
 
-{ f } [
+[
   "a++b" 
   "a" token "+" token "++" token 2array choice "b" token 3array seq
   parse [ t ] [ f ] if
-] unit-test
+] must-fail
 
 { 1 } [
-  "a" "a" token [ drop 1 ] action parse ast>> 
+  "a" "a" token [ drop 1 ] action parse 
 ] unit-test
 
 { V{ 1 1 } } [
-  "aa" "a" token [ drop 1 ] action dup 2array seq parse ast>> 
+  "aa" "a" token [ drop 1 ] action dup 2array seq parse 
 ] unit-test
 
-{ f } [
+[
   "b" "a" token [ drop 1 ] action parse 
-] unit-test
+] must-fail
 
-{ f } [ 
+[ 
   "b" [ CHAR: a = ] satisfy parse 
-] unit-test
+] must-fail
 
 { CHAR: a } [ 
-  "a" [ CHAR: a = ] satisfy parse ast>>
+  "a" [ CHAR: a = ] satisfy parse
 ] unit-test
 
 { "a" } [
-  "    a" "a" token sp parse ast>>
+  "    a" "a" token sp parse
 ] unit-test
 
 { "a" } [
-  "a" "a" token sp parse ast>>
+  "a" "a" token sp parse
 ] unit-test
 
 { V{ "a" } } [
-  "[a]" "[" token hide "a" token "]" token hide 3array seq parse ast>>
+  "[a]" "[" token hide "a" token "]" token hide 3array seq parse
 ] unit-test
 
-{ f } [
+[
   "a]" "[" token hide "a" token "]" token hide 3array seq parse 
-] unit-test
+] must-fail
 
 
 { V{ "1" "-" "1" } V{ "1" "+" "1" } } [
@@ -165,8 +165,8 @@ IN: peg.tests
     [ "1" token , "-" token , "1" token , ] seq* ,
     [ "1" token , "+" token , "1" token , ] seq* ,
   ] choice* 
-  "1-1" over parse ast>> swap
+  "1-1" over parse swap
-  "1+1" swap parse ast>>
+  "1+1" swap parse
 ] unit-test
 
 : expr ( -- parser ) 
@@ -175,21 +175,22 @@ IN: peg.tests
   [ expr ] delay "+" token "1" token 3seq "1" token 2choice ;
 
 { V{ V{ "1" "+" "1" } "+" "1" } } [
-  "1+1+1" expr parse ast>>   
+  "1+1+1" expr parse   
 ] unit-test
 
 { t } [
   #! Ensure a circular parser doesn't loop infinitely
   [ f , "a" token , ] seq*
-  dup parsers>>
+  dup peg>> parsers>>
   dupd 0 swap set-nth compile word?
 ] unit-test
 
-{ f } [
+[
   "A" [ drop t ] satisfy [ 66 >= ] semantic parse 
-] unit-test
+] must-fail
 
 { CHAR: B } [
-  "B" [ drop t ] satisfy [ 66 >= ] semantic parse ast>>
+  "B" [ drop t ] satisfy [ 66 >= ] semantic parse
 ] unit-test
 
+{ f } [ \ + T{ parser f f f } equal? ] unit-test

extra/peg/peg.factor

@@ -1,59 +1,105 @@
 ! Copyright (C) 2007, 2008 Chris Double.
 ! See http://factorcode.org/license.txt for BSD license.
 USING: kernel sequences strings fry namespaces math assocs shuffle debugger io
-       vectors arrays math.parser math.order
+       vectors arrays math.parser math.order vectors combinators combinators.lib
-       unicode.categories compiler.units parser
+       combinators.short-circuit classes sets unicode.categories compiler.units parser
        words quotations effects memoize accessors locals effects splitting ;
 IN: peg
 
 USE: prettyprint
 
 TUPLE: parse-result remaining ast ;
+TUPLE: parse-error position messages ; 
+TUPLE: parser peg compiled id ;
 
-TUPLE: parser id compiled ;
+M: parser equal?    { [ [ class ] bi@ = ] [ [ id>> ] bi@ = ] } 2&& ;
-
-M: parser equal? [ id>> ] bi@ = ;
-
 M: parser hashcode* id>> hashcode* ;
 
-C: <parser> parser
+C: <parse-result> parse-result
+C: <parse-error>  parse-error
 
+M: parse-error error.
+  "Peg parsing error at character position " write dup position>> number>string write 
+  "." print "Expected " write messages>> [ " or " write ] [ write ] interleave nl ;
+
+SYMBOL: error-stack
+
+: (merge-errors) ( a b -- c )
+  {
+    { [ over position>> not ] [ nip ] } 
+    { [ dup  position>> not ] [ drop ] } 
+    [ 2dup [ position>> ] bi@ <=> {
+        { +lt+ [ nip ] }
+        { +gt+ [ drop ] }
+        { +eq+ [ messages>> over messages>> union [ position>> ] dip <parse-error> ] }
+      } case 
+    ]
+  } cond ;
+
+: merge-errors ( -- )
+  error-stack get dup length 1 >  [
+    dup pop over pop swap (merge-errors) swap push
+  ] [
+    drop
+  ] if ;
+
+: add-error ( remaining message -- )
+  <parse-error> error-stack get push ;
+  
 SYMBOL: ignore 
 
-: <parse-result> ( remaining ast -- parse-result )
+: packrat ( id -- cache )
-  parse-result boa ;
+  #! The packrat cache is a mapping of parser-id->cache.
+  #! For each parser it maps to a cache holding a mapping
+  #! of position->result. The packrat cache therefore keeps
+  #! track of all parses that have occurred at each position
+  #! of the input string and the results obtained from that
+  #! parser.
+  \ packrat get [ drop H{ } clone ] cache ;
 
-SYMBOL: packrat
 SYMBOL: pos
 SYMBOL: input
 SYMBOL: fail
 SYMBOL: lrstack
-SYMBOL: heads
+
+: heads ( -- cache )
+  #! A mapping from position->peg-head.	It maps a
+  #! position in the input string being parsed to 
+  #! the head of the left recursion which is currently
+  #! being grown. It is 'f' at any position where
+  #! left recursion growth is not underway.
+  \ heads get ;
 
 : failed? ( obj -- ? )
   fail = ;
 
-: delegates ( -- cache )
+: peg-cache ( -- cache )
-  \ delegates get-global [ H{ } clone dup \ delegates set-global ] unless* ;
+  #! Holds a hashtable mapping a peg tuple to
+  #! the parser tuple for that peg. The parser tuple
+  #! holds a unique id and the compiled form of that peg.
+  \ peg-cache get-global [
+    H{ } clone dup \ peg-cache set-global
+  ] unless* ;
 
 : reset-pegs ( -- )
-  H{ } clone \ delegates set-global ;
+  H{ } clone \ peg-cache set-global ;
 
 reset-pegs 
 
+#! An entry in the table of memoized parse results
+#! ast = an AST produced from the parse
+#!       or the symbol 'fail'
+#!       or a left-recursion object
+#! pos = the position in the input string of this entry
 TUPLE: memo-entry ans pos ;
-C: <memo-entry> memo-entry
 
-TUPLE: left-recursion seed rule head next ;
+TUPLE: left-recursion seed rule-id head next ; 
-C: <left-recursion> left-recursion
+TUPLE: peg-head rule-id involved-set eval-set ;
- 
-TUPLE: peg-head rule involved-set eval-set ;
-C: <head> peg-head
 
-: rule-parser ( rule -- parser ) 
+: rule-id ( word -- id ) 
   #! A rule is the parser compiled down to a word. It has
-  #! a "peg" property containing the original parser.
+  #! a "peg-id" property containing the id of the original parser.
-  "peg" word-prop ;
+  "peg-id" word-prop ;
 
 : input-slice ( -- slice )
   #! Return a slice of the input from the current parse position
@@ -64,11 +110,6 @@ C: <head> peg-head
   #! input slice is based on.
   dup slice? [ slice-from ] [ drop 0 ] if ;
 
-: input-cache ( parser -- cache )
-  #! From the packrat cache, obtain the cache for the parser 
-  #! that maps the position to the parser result.
-  id>> packrat get [ drop H{ } clone ] cache ;
-
 : process-rule-result ( p result -- result )
   [
     nip [ ast>> ] [ remaining>> ] bi input-from pos set    
@@ -79,16 +120,18 @@ C: <head> peg-head
 : eval-rule ( rule -- ast )
   #! Evaluate a rule, return an ast resulting from it.
   #! Return fail if the rule failed. The rule has
-  #! stack effect ( input -- parse-result )
+  #! stack effect ( -- parse-result )
   pos get swap execute process-rule-result ; inline
 
-: memo ( pos rule -- memo-entry )
+: memo ( pos id -- memo-entry )
   #! Return the result from the memo cache. 
-  rule-parser input-cache at ;
+  packrat at 
+!  "  memo result " write dup . 
+  ;
 
-: set-memo ( memo-entry pos rule -- )
+: set-memo ( memo-entry pos id -- )
   #! Store an entry in the cache
-  rule-parser input-cache set-at ;
+  packrat set-at ;
 
 : update-m ( ast m -- )
   swap >>ans pos get >>pos drop ;
@@ -111,22 +154,22 @@ C: <head> peg-head
   ] if ; inline
  
 : grow-lr ( h p r m -- ast )
-  >r >r [ heads get set-at ] 2keep r> r>
+  >r >r [ heads set-at ] 2keep r> r>
   pick over >r >r (grow-lr) r> r>
-  swap heads get delete-at
+  swap heads delete-at
   dup pos>> pos set ans>>
   ; inline
 
 :: (setup-lr) ( r l s -- )
   s head>> l head>> eq? [
     l head>> s (>>head)
-    l head>> [ s rule>> suffix ] change-involved-set drop
+    l head>> [ s rule-id>> suffix ] change-involved-set drop
     r l s next>> (setup-lr)
   ] unless ;
 
 :: setup-lr ( r l -- )
   l head>> [
-    r V{ } clone V{ } clone <head> l (>>head)
+    r rule-id V{ } clone V{ } clone peg-head boa l (>>head)
   ] unless
   r l lrstack get (setup-lr) ;
 
@@ -134,7 +177,7 @@ C: <head> peg-head
   [let* |
           h [ m ans>> head>> ]
         |
-    h rule>> r eq? [
+    h rule-id>> r rule-id eq? [
       m ans>> seed>> m (>>ans)
       m ans>> failed? [
         fail
@@ -148,15 +191,15 @@ C: <head> peg-head
 
 :: recall ( r p -- memo-entry )
   [let* |
-          m [ p r memo ]
+          m [ p r rule-id memo ]
-          h [ p heads get at ]
+          h [ p heads at ]
         |
     h [
-      m r h involved-set>> h rule>> suffix member? not and [
+      m r rule-id h involved-set>> h rule-id>> suffix member? not and [
-        fail p <memo-entry>
+        fail p memo-entry boa
       ] [
-        r h eval-set>> member? [
+        r rule-id h eval-set>> member? [
-          h [ r swap remove ] change-eval-set drop
+          h [ r rule-id swap remove ] change-eval-set drop
           r eval-rule
           m update-m
           m
@@ -171,8 +214,8 @@ C: <head> peg-head
 
 :: apply-non-memo-rule ( r p -- ast )
   [let* |
-          lr  [ fail r f lrstack get <left-recursion> ]
+          lr  [ fail r rule-id f lrstack get left-recursion boa ]
-          m   [ lr lrstack set lr p <memo-entry> dup p r set-memo ]
+          m   [ lr lrstack set lr p memo-entry boa dup p r rule-id set-memo ]
           ans [ r eval-rule ]
         |
     lrstack get next>> lrstack set
@@ -194,10 +237,15 @@ C: <head> peg-head
     nip
   ] if ; 
 
+USE: prettyprint
+
 : apply-rule ( r p -- ast )
+!   2dup [ rule-id ] dip 2array "apply-rule: " write .
    2dup recall [
+!     "  memoed" print
      nip apply-memo-rule
    ] [
+!     "  not memoed" print
      apply-non-memo-rule
    ] if* ; inline
 
@@ -207,24 +255,28 @@ C: <head> peg-head
     input set
     0 pos set
     f lrstack set
-    H{ } clone heads set
+    V{ } clone error-stack set
-    H{ } clone packrat set
+    H{ } clone \ heads set
+    H{ } clone \ packrat set
   ] H{ } make-assoc swap bind ; inline
 
 
-GENERIC: (compile) ( parser -- quot )
+GENERIC: (compile) ( peg -- quot )
 
-: execute-parser ( word -- result )
+: process-parser-result ( result -- result )
-  pos get apply-rule dup failed? [ 
+  dup failed? [ 
     drop f 
   ] [
     input-slice swap <parse-result>
-  ] if ; inline
+  ] if ;
+    
+: execute-parser ( word -- result )
+  pos get apply-rule process-parser-result ; inline
 
 : parser-body ( parser -- quot )
   #! Return the body of the word that is the compiled version
   #! of the parser.
-  gensym 2dup swap (compile) 0 1 <effect> define-declared swap dupd "peg" set-word-prop
+  gensym 2dup swap peg>> (compile) 0 1 <effect> define-declared swap dupd id>> "peg-id" set-word-prop
   [ execute-parser ] curry ;
 
 : compiled-parser ( parser -- word )
@@ -257,11 +309,14 @@ SYMBOL: delayed
   ] with-compilation-unit ;
 
 : compiled-parse ( state word -- result )
-  swap [ execute ] with-packrat ; inline 
+  swap [ execute [ error-stack get first throw ] unless* ] with-packrat ; inline 
 
-: parse ( input parser -- result )
+: (parse) ( input parser -- result )
   dup word? [ compile ] unless compiled-parse ;
 
+: parse ( input parser -- ast )
+  (parse) ast>> ;
+
 <PRIVATE
 
 SYMBOL: id 
@@ -274,24 +329,25 @@ SYMBOL: id
     1 id set-global 0
   ] if* ;
 
-: init-parser ( parser -- parser )
+: wrap-peg ( peg -- parser )
-  #! Set the delegate for the parser. Equivalent parsers
+  #! Wrap a parser tuple around the peg object.
-  #! get a delegate with the same id.
+  #! Look for an existing parser tuple for that
-  dup clone delegates [
+  #! peg object.
-    drop next-id f <parser> 
+  peg-cache [
-  ] cache over set-delegate ;
+    f next-id parser boa 
+  ] cache ;
 
 TUPLE: token-parser symbol ;
 
 : parse-token ( input string -- result )
   #! Parse the string, returning a parse result
   dup >r ?head-slice [
-    r> <parse-result> 
+    r> <parse-result> f f add-error
   ] [
-    r> 2drop f
+    drop pos get "token '" r> append "'" append 1vector add-error f
   ] if ;
 
-M: token-parser (compile) ( parser -- quot )
+M: token-parser (compile) ( peg -- quot )
   symbol>> '[ input-slice , parse-token ] ;
    
 TUPLE: satisfy-parser quot ;
@@ -308,7 +364,7 @@ TUPLE: satisfy-parser quot ;
   ] if ; inline
 
 
-M: satisfy-parser (compile) ( parser -- quot )
+M: satisfy-parser (compile) ( peg -- quot )
   quot>> '[ input-slice , parse-satisfy ] ;
 
 TUPLE: range-parser min max ;
@@ -324,7 +380,7 @@ TUPLE: range-parser min max ;
     ] if
   ] if ;
 
-M: range-parser (compile) ( parser -- quot )
+M: range-parser (compile) ( peg -- quot )
   [ min>> ] [ max>> ] bi '[ input-slice , , parse-range ] ;
 
 TUPLE: seq-parser parsers ;
@@ -351,18 +407,20 @@ TUPLE: seq-parser parsers ;
     2drop f
   ] if ; inline
 
-M: seq-parser (compile) ( parser -- quot )
+M: seq-parser (compile) ( peg -- quot )
   [
     [ input-slice V{ } clone <parse-result> ] %
-    parsers>> [ compiled-parser 1quotation , \ parse-seq-element , ] each 
+    parsers>> unclip compiled-parser 1quotation , \ parse-seq-element , [ 
+      compiled-parser 1quotation [ merge-errors ] compose , \ parse-seq-element , ] each 
   ] [ ] make ;
 
 TUPLE: choice-parser parsers ;
 
-M: choice-parser (compile) ( parser -- quot )
+M: choice-parser (compile) ( peg -- quot )
   [ 
     f ,
-    parsers>> [ compiled-parser 1quotation , \ unless* , ] each
+    parsers>> [ compiled-parser ] map 
+    unclip 1quotation , \ unless* , [ 1quotation [ merge-errors ] compose , \ unless* , ] each
   ] [ ] make ;
 
 TUPLE: repeat0-parser p1 ;
@@ -376,7 +434,7 @@ TUPLE: repeat0-parser p1 ;
     nip
   ] if* ; inline
 
-M: repeat0-parser (compile) ( parser -- quot )
+M: repeat0-parser (compile) ( peg -- quot )
   p1>> compiled-parser 1quotation '[ 
     input-slice V{ } clone <parse-result> , swap (repeat) 
   ] ; 
@@ -390,7 +448,7 @@ TUPLE: repeat1-parser p1 ;
     f
   ] if* ;
 
-M: repeat1-parser (compile) ( parser -- quot )
+M: repeat1-parser (compile) ( peg -- quot )
   p1>> compiled-parser 1quotation '[ 
     input-slice V{ } clone <parse-result> , swap (repeat) repeat1-empty-check  
   ] ; 
@@ -400,7 +458,7 @@ TUPLE: optional-parser p1 ;
 : check-optional ( result -- result )
   [ input-slice f <parse-result> ] unless* ;
 
-M: optional-parser (compile) ( parser -- quot )
+M: optional-parser (compile) ( peg -- quot )
   p1>> compiled-parser 1quotation '[ @ check-optional ] ;
 
 TUPLE: semantic-parser p1 quot ;
@@ -412,7 +470,7 @@ TUPLE: semantic-parser p1 quot ;
     drop
   ] if ; inline
 
-M: semantic-parser (compile) ( parser -- quot )
+M: semantic-parser (compile) ( peg -- quot )
   [ p1>> compiled-parser 1quotation ] [ quot>> ] bi  
   '[ @ , check-semantic ] ;
 
@@ -421,7 +479,7 @@ TUPLE: ensure-parser p1 ;
 : check-ensure ( old-input result -- result )
   [ ignore <parse-result> ] [ drop f ] if ;
 
-M: ensure-parser (compile) ( parser -- quot )
+M: ensure-parser (compile) ( peg -- quot )
   p1>> compiled-parser 1quotation '[ input-slice @ check-ensure ] ;
 
 TUPLE: ensure-not-parser p1 ;
@@ -429,7 +487,7 @@ TUPLE: ensure-not-parser p1 ;
 : check-ensure-not ( old-input result -- result )
   [ drop f ] [ ignore <parse-result> ] if ;
 
-M: ensure-not-parser (compile) ( parser -- quot )
+M: ensure-not-parser (compile) ( peg -- quot )
   p1>> compiled-parser 1quotation '[ input-slice @ check-ensure-not ] ;
 
 TUPLE: action-parser p1 quot ;
@@ -441,7 +499,7 @@ TUPLE: action-parser p1 quot ;
     drop
   ] if ; inline
 
-M: action-parser (compile) ( parser -- quot )
+M: action-parser (compile) ( peg -- quot )
   [ p1>> compiled-parser 1quotation ] [ quot>> ] bi '[ @ , check-action ] ;
 
 : left-trim-slice ( string -- string )
@@ -453,14 +511,14 @@ M: action-parser (compile) ( parser -- quot )
 
 TUPLE: sp-parser p1 ;
 
-M: sp-parser (compile) ( parser -- quot )
+M: sp-parser (compile) ( peg -- quot )
   p1>> compiled-parser 1quotation '[ 
     input-slice left-trim-slice input-from pos set @ 
   ] ;
 
 TUPLE: delay-parser quot ;
 
-M: delay-parser (compile) ( parser -- quot )
+M: delay-parser (compile) ( peg -- quot )
   #! For efficiency we memoize the quotation.
   #! This way it is run only once and the 
   #! parser constructed once at run time.
@@ -468,29 +526,26 @@ M: delay-parser (compile) ( parser -- quot )
 
 TUPLE: box-parser quot ;
 
-M: box-parser (compile) ( parser -- quot )
+M: box-parser (compile) ( peg -- quot )
   #! Calls the quotation at compile time
   #! to produce the parser to be compiled.
   #! This differs from 'delay' which calls
-  #! it at run time. Due to using the runtime
+  #! it at run time.
-  #! environment at compile time, this parser
+  quot>> call compiled-parser 1quotation ;
-  #! must not be cached, so we clear out the
-  #! delgates cache.
-  f >>compiled quot>> call compiled-parser 1quotation ;
 
 PRIVATE>
 
 : token ( string -- parser )
-  token-parser boa init-parser ;      
+  token-parser boa wrap-peg ;      
 
 : satisfy ( quot -- parser )
-  satisfy-parser boa init-parser ;
+  satisfy-parser boa wrap-peg ;
 
 : range ( min max -- parser )
-  range-parser boa init-parser ;
+  range-parser boa wrap-peg ;
 
 : seq ( seq -- parser )
-  seq-parser boa init-parser ;
+  seq-parser boa wrap-peg ;
 
 : 2seq ( parser1 parser2 -- parser )
   2array seq ;
@@ -505,7 +560,7 @@ PRIVATE>
   { } make seq ; inline 
 
 : choice ( seq -- parser )
-  choice-parser boa init-parser ;
+  choice-parser boa wrap-peg ;
 
 : 2choice ( parser1 parser2 -- parser )
   2array choice ;
@@ -520,38 +575,38 @@ PRIVATE>
   { } make choice ; inline 
 
 : repeat0 ( parser -- parser )
-  repeat0-parser boa init-parser ;
+  repeat0-parser boa wrap-peg ;
 
 : repeat1 ( parser -- parser )
-  repeat1-parser boa init-parser ;
+  repeat1-parser boa wrap-peg ;
 
 : optional ( parser -- parser )
-  optional-parser boa init-parser ;
+  optional-parser boa wrap-peg ;
 
 : semantic ( parser quot -- parser )
-  semantic-parser boa init-parser ;
+  semantic-parser boa wrap-peg ;
 
 : ensure ( parser -- parser )
-  ensure-parser boa init-parser ;
+  ensure-parser boa wrap-peg ;
 
 : ensure-not ( parser -- parser )
-  ensure-not-parser boa init-parser ;
+  ensure-not-parser boa wrap-peg ;
 
 : action ( parser quot -- parser )
-  action-parser boa init-parser ;
+  action-parser boa wrap-peg ;
 
 : sp ( parser -- parser )
-  sp-parser boa init-parser ;
+  sp-parser boa wrap-peg ;
 
 : hide ( parser -- parser )
   [ drop ignore ] action ;
 
 : delay ( quot -- parser )
-  delay-parser boa init-parser ;
+  delay-parser boa wrap-peg ;
 
 : box ( quot -- parser )
   #! because a box has its quotation run at compile time
-  #! it must always have a new parser delgate created, 
+  #! it must always have a new parser wrapper created, 
   #! not a cached one. This is because the same box,
   #! compiled twice can have a different compiled word
   #! due to running at compile time.
@@ -561,7 +616,7 @@ PRIVATE>
   #! parse. The action adds an indirection with a parser type
   #! that gets memoized and fixes this. Need to rethink how
   #! to fix boxes so this isn't needed...
-  box-parser boa next-id f <parser> over set-delegate [ ] action ;
+  box-parser boa f next-id parser boa [ ] action ;
 
 ERROR: parse-failed input word ;
 

extra/peg/pl0/pl0-tests.factor

@@ -6,39 +6,39 @@ USING: kernel tools.test peg peg.ebnf peg.pl0
 IN: peg.pl0.tests
 
 { t } [
-  "CONST foo = 1;" "block" \ pl0 rule parse remaining>> empty? 
+  "CONST foo = 1;" "block" \ pl0 rule (parse) remaining>> empty? 
 ] unit-test
 
 { t } [
-  "VAR foo;" "block" \ pl0 rule parse remaining>> empty?
+  "VAR foo;" "block" \ pl0 rule (parse) remaining>> empty?
 ] unit-test
 
 { t } [
-  "VAR foo,bar , baz;" "block" \ pl0 rule parse remaining>> empty? 
+  "VAR foo,bar , baz;" "block" \ pl0 rule (parse) remaining>> empty? 
 ] unit-test
 
 { t } [
-  "foo := 5" "statement" \ pl0 rule parse remaining>> empty? 
+  "foo := 5" "statement" \ pl0 rule (parse) remaining>> empty? 
 ] unit-test
 
 { t } [
-  "BEGIN foo := 5 END" "statement" \ pl0 rule parse remaining>> empty? 
+  "BEGIN foo := 5 END" "statement" \ pl0 rule (parse) remaining>> empty? 
 ] unit-test
 
 { t } [
-  "IF 1=1 THEN foo := 5" "statement" \ pl0 rule parse remaining>> empty? 
+  "IF 1=1 THEN foo := 5" "statement" \ pl0 rule (parse) remaining>> empty? 
 ] unit-test
 
 { t } [
-  "WHILE 1=1 DO foo := 5" "statement" \ pl0 rule parse remaining>> empty? 
+  "WHILE 1=1 DO foo := 5" "statement" \ pl0 rule (parse) remaining>> empty? 
 ] unit-test
 
 { t } [
-  "WHILE ODD 1 DO foo := 5" "statement" \ pl0 rule parse remaining>> empty? 
+  "WHILE ODD 1 DO foo := 5" "statement" \ pl0 rule (parse) remaining>> empty? 
 ] unit-test
 
 { t } [
-  "PROCEDURE square; BEGIN squ:=x*x END" "block" \ pl0 rule parse remaining>> empty? 
+  "PROCEDURE square; BEGIN squ:=x*x END" "block" \ pl0 rule (parse) remaining>> empty? 
 ] unit-test
 
 { t } [
@@ -58,7 +58,7 @@ BEGIN
       x := x + 1;
    END
 END.
-"> pl0 remaining>> empty?
+"> main \ pl0 rule (parse) remaining>> empty?
 ] unit-test
 
 { f } [
@@ -124,5 +124,5 @@ BEGIN
   y := 36;
   CALL gcd;
 END.
-  "> pl0 remaining>> empty?
+  "> main \ pl0 rule (parse) remaining>> empty?
 ] unit-test

extra/peg/search/search.factor

@@ -1,7 +1,7 @@
 ! Copyright (C) 2006 Chris Double.
 ! See http://factorcode.org/license.txt for BSD license.
 USING: kernel math io io.streams.string sequences strings
-combinators peg memoize arrays ;
+combinators peg memoize arrays continuations ;
 IN: peg.search
 
 : tree-write ( object -- )
@@ -16,15 +16,12 @@ MEMO: any-char-parser ( -- parser )
   [ drop t ] satisfy ;
 
 : search ( string parser -- seq )
-  any-char-parser [ drop f ] action 2array choice repeat0 parse dup [
+  any-char-parser [ drop f ] action 2array choice repeat0 
-    parse-result-ast sift
+  [ parse sift ] [ 3drop { } ] recover ;
-  ] [
-    drop { }
-  ] if ;
 
 
 : (replace) ( string parser -- seq )
-  any-char-parser 2array choice repeat0 parse parse-result-ast sift ;
+  any-char-parser 2array choice repeat0 parse sift ;
 
 : replace ( string parser -- result )
  [  (replace) [ tree-write ] each ] with-string-writer ;

extra/project-euler/002/002.factor

@@ -40,7 +40,7 @@ PRIVATE>
 ! -------------------
 
 : fib-upto* ( n -- seq )
-    0 1 [ pick over >= ] [ tuck + dup ] [ ] unfold 3nip
+    0 1 [ pick over >= ] [ tuck + dup ] [ ] produce 3nip
     but-last-slice { 0 1 } prepend ;
 
 : euler002a ( -- answer )

extra/project-euler/019/019.factor

@@ -53,7 +53,7 @@ IN: project-euler.019
 : first-days ( end-date start-date -- days )
     [ 2dup after=? ]
     [ dup 1 months time+ swap day-of-week ]
-    [ ] unfold 2nip ;
+    [ ] produce 2nip ;
 
 PRIVATE>
 

extra/project-euler/148/148.factor

@@ -10,7 +10,7 @@ IN: project-euler.148
     dup 1+ * 2/ ; inline
 
 : >base7 ( x -- y )
-    [ dup 0 > ] [ 7 /mod ] [ ] unfold nip ;
+    [ dup 0 > ] [ 7 /mod ] [ ] produce nip ;
 
 : (use-digit) ( prev x index -- next )
     [ [ 1+ * ] [ sum-1toN 7 sum-1toN ] bi ] dip ^ * + ;

extra/project-euler/common/common.factor

@@ -78,7 +78,7 @@ PRIVATE>
     ] if ;
 
 : number>digits ( n -- seq )
-    [ dup zero? not ] [ 10 /mod ] [ ] unfold reverse nip ;
+    [ dup zero? not ] [ 10 /mod ] [ ] produce reverse nip ;
 
 : nth-triangle ( n -- n )
     dup 1+ * 2 / ;

extra/sequences/lib/lib.factor

@@ -84,17 +84,21 @@ MACRO: firstn ( n -- )
 : v, ( -- ) V{ } clone , ;
 : ,v ( -- ) building get dup peek empty? [ dup pop* ] when drop ;
 
-: monotonic-split ( seq quot -- newseq )
+: (monotonic-split) ( seq quot -- newseq )
     [
         >r dup unclip suffix r>
         v, [ pick ,, call [ v, ] unless ] curry 2each ,v
     ] { } make ;
 
+: monotonic-split ( seq quot -- newseq )
+    over empty? [ 2drop { } ] [ (monotonic-split) ] if ;
+
 : delete-random ( seq -- value )
     [ length random ] keep [ nth ] 2keep delete-nth ;
 
+ERROR: element-not-found ;
 : split-around ( seq quot -- before elem after )
-    dupd find over [ "Element not found" throw ] unless
+    dupd find over [ element-not-found ] unless
     >r cut rest r> swap ; inline
 
 : (map-until) ( quot pred -- quot )
@@ -206,9 +210,6 @@ PRIVATE>
 : nths ( seq indices -- seq' )
     swap [ nth ] curry map ;
 
-: replace ( str oldseq newseq -- str' )
-    zip >hashtable substitute ;
-
 : remove-nth ( seq n -- seq' )
     cut-slice rest-slice append ;
 

extra/soundex/author.txt

@@ -0,0 +1 @@
+Slava Pestov

extra/soundex/soundex-tests.factor

@@ -0,0 +1,4 @@
+IN: soundex.tests
+USING: soundex tools.test ;
+
+[ "S162" ] [ "supercalifrag" soundex ] unit-test

extra/soundex/soundex.factor

@@ -0,0 +1,33 @@
+! Copyright (C) 2008 Slava Pestov.
+! See http://factorcode.org/license.txt for BSD license.
+USING: sequences sequences.lib grouping assocs kernel ascii
+unicode.case tr ;
+IN: soundex
+
+TR: soundex-tr
+    ch>upper
+    "AEHIOUWYBFPVCGJKQSXZDTLMNR"
+    "00000000111122222222334556" ;
+
+: remove-duplicates ( seq -- seq' )
+    #! Remove _consecutive_ duplicates (unlike prune which removes
+    #! all duplicates).
+    [ 2 <clumps> [ = not ] assoc-filter values ] [ first ] bi prefix ;
+
+: first>upper ( seq -- seq' ) 1 head >upper ;
+: trim-first ( seq -- seq' ) dup first [ = ] curry left-trim ;
+: remove-zeroes ( seq -- seq' ) CHAR: 0 swap remove ;
+: remove-non-alpha ( seq -- seq' ) [ alpha? ] filter ;
+: pad-4 ( first seq -- seq' ) "000" 3append 4 head ;
+
+: soundex ( string -- soundex )
+    remove-non-alpha [ f ] [
+        [ first>upper ]
+        [
+            soundex-tr
+            trim-first
+            remove-duplicates
+            remove-zeroes
+        ] bi
+        pad-4
+    ] if-empty ;

extra/soundex/summary.txt

@@ -0,0 +1 @@
+Soundex is a phonetic algorithm for indexing names by sound

extra/tools/disassembler/disassembler.factor

@@ -3,7 +3,7 @@
 USING: io.files io words alien kernel math.parser alien.syntax
 io.launcher system assocs arrays sequences namespaces qualified
 system math generator.fixup io.encodings.ascii accessors
-generic ;
+generic tr ;
 IN: tools.disassembler
 
 : in-file ( -- path ) "gdb-in.txt" temp-file ;
@@ -36,8 +36,7 @@ M: method-spec make-disassemble-cmd
     try-process
     out-file ascii file-lines ;
 
-: tabs>spaces ( str -- str' )
+TR: tabs>spaces "\t" "\s" ;
-    { { CHAR: \t CHAR: \s } } substitute ;
 
 : disassemble ( obj -- )
     make-disassemble-cmd run-gdb

extra/tools/vocabs/monitor/monitor.factor

@@ -2,12 +2,16 @@
 ! See http://factorcode.org/license.txt for BSD license.
 USING: threads io.files io.monitors init kernel
 vocabs vocabs.loader tools.vocabs namespaces continuations
-sequences splitting assocs command-line concurrency.messaging io.backend sets ;
+sequences splitting assocs command-line concurrency.messaging
+io.backend sets tr ;
 IN: tools.vocabs.monitor
 
+TR: convert-separators "/\\" ".." ;
+
 : vocab-dir>vocab-name ( path -- vocab )
-    left-trim-separators right-trim-separators
+    left-trim-separators
-    { { CHAR: / CHAR: . } { CHAR: \\ CHAR: . } } substitute ;
+    right-trim-separators
+    convert-separators ;
 
 : path>vocab-name ( path -- vocab )
     dup ".factor" tail? [ parent-directory ] when ;

extra/tr/authors.txt

@@ -0,0 +1 @@
+Slava Pestov

extra/tr/summary.txt

@@ -0,0 +1 @@
+Fast character-to-character translation of ASCII strings

extra/tr/tr-tests.factor

@@ -0,0 +1,8 @@
+IN: tr.tests
+USING: tr tools.test unicode.case ;
+
+TR: tr-test ch>upper "ABC" "XYZ" ;
+
+[ "XXYY" ] [ "aabb" tr-test ] unit-test
+[ "XXYY" ] [ "AABB" tr-test ] unit-test
+[ { 12345 } ] [ { 12345 } tr-test ] unit-test

extra/tr/tr.factor

@@ -0,0 +1,37 @@
+! Copyright (C) 2008 Slava Pestov.
+! See http://factorcode.org/license.txt for BSD license.
+USING: byte-arrays strings sequences sequences.private
+fry kernel words parser lexer assocs math.order ;
+IN: tr
+
+<PRIVATE
+
+: compute-tr ( quot from to -- mapping )
+    zip [ 256 ] 2dip '[ [ @ , at ] keep or ] B{ } map-as ; inline
+
+: tr-hints ( word -- )
+    { { byte-array } { string } } "specializer" set-word-prop ;
+
+: create-tr ( token -- word )
+    create-in dup tr-hints ;
+
+: tr-quot ( mapping -- quot )
+    '[ [ dup 0 255 between? [ , nth-unsafe ] when ] map ] ;
+
+: define-tr ( word mapping -- )
+    tr-quot (( seq -- translated )) define-declared ;
+
+: fast-tr-quot ( mapping -- quot )
+    '[ [ , nth-unsafe ] change-each ] ;
+
+: define-fast-tr ( word mapping -- )
+    fast-tr-quot (( seq -- )) define-declared ;
+
+PRIVATE>
+
+: TR:
+    scan parse-definition
+    unclip-last [ unclip-last ] dip compute-tr
+    [ [ create-tr ] dip define-tr ]
+    [ [ "-fast" append create-tr ] dip define-fast-tr ] 2bi ;
+    parsing

extra/ui/commands/commands.factor

@@ -2,7 +2,7 @@
 ! See http://factorcode.org/license.txt for BSD license.
 USING: accessors arrays definitions kernel sequences strings
 math assocs words generic namespaces assocs quotations splitting
-ui.gestures unicode.case unicode.categories ;
+ui.gestures unicode.case unicode.categories tr ;
 IN: ui.commands
 
 SYMBOL: +nullary+
@@ -50,8 +50,10 @@ GENERIC: command-word ( command -- word )
     swap pick commands set-at
     update-gestures ;
 
+TR: convert-command-name "-" " " ;
+
 : (command-name) ( string -- newstring )
-    { { CHAR: - CHAR: \s } } substitute >title ;
+    convert-command-name >title ;
 
 M: word command-name ( word -- str )
     name>> 

extra/ui/windows/windows.factor

@@ -24,7 +24,7 @@ SINGLETON: windows-ui-backend
     [ EnumClipboardFormats win32-error dup dup 0 > ]
     [ ]
     [ drop ]
-    unfold nip ;
+    produce nip ;
 
 : with-clipboard ( quot -- )
     f OpenClipboard win32-error=0/f

extra/unicode/data/data.factor

@@ -125,7 +125,7 @@ VALUE: properties
 : process-names ( data -- names-hash )
     1 swap (process-data) [
         ascii-lower { { CHAR: \s CHAR: - } } substitute swap
-    ] assoc-map >hashtable ;
+    ] H{ } assoc-map-as ;
 
 : multihex ( hexstring -- string )
     " " split [ hex> ] map sift ;

extra/unix/types/freebsd/freebsd.factor

@@ -4,8 +4,6 @@ IN: unix.types
 
 ! FreeBSD 7 x86.32
 
-! Need to verify on 64-bit
-
 TYPEDEF: ushort          __uint16_t
 TYPEDEF: uint           __uint32_t
 TYPEDEF: int            __int32_t
@@ -21,6 +19,6 @@ TYPEDEF: __int64_t      off_t
 TYPEDEF: __int64_t      blkcnt_t
 TYPEDEF: __uint32_t     blksize_t
 TYPEDEF: __uint32_t     fflags_t
-TYPEDEF: int            ssize_t
+TYPEDEF: long           ssize_t
 TYPEDEF: int            pid_t
-TYPEDEF: int            time_t
+TYPEDEF: int            time_t

extra/unix/types/openbsd/openbsd.factor

@@ -27,6 +27,6 @@ TYPEDEF: __int64_t      off_t
 TYPEDEF: __int64_t      blkcnt_t
 TYPEDEF: __uint32_t     blksize_t
 TYPEDEF: __uint32_t     fflags_t
-TYPEDEF: int            ssize_t
+TYPEDEF: long           ssize_t
 TYPEDEF: int            pid_t
 TYPEDEF: int            time_t

extra/webapps/blogs/blogs.factor

@@ -160,13 +160,13 @@ M: comment entity-url
 
         [
             validate-post
-            logged-in-user get username>> "author" set-value
+            username "author" set-value
         ] >>validate
 
         [
             f <post>
                 dup { "title" "content" } to-object
-                logged-in-user get username>> >>author
+                username >>author
                 now >>date
             [ insert-tuple ] [ entity-url <redirect> ] bi
         ] >>submit
@@ -177,9 +177,9 @@ M: comment entity-url
         "make a new blog post" >>description ;
 
 : authorize-author ( author -- )
-    logged-in-user get username>> =
+    username =
-    can-administer-blogs? have-capability? or
+    { can-administer-blogs? } have-capabilities? or
-    [ login-required ] unless ;
+    [ "edit a blog post" f login-required ] unless ;
 
 : do-post-action ( -- )
     validate-integer-id
@@ -254,13 +254,13 @@ M: comment entity-url
 
         [
             validate-comment
-            logged-in-user get username>> "author" set-value
+            username "author" set-value
         ] >>validate
 
         [
             "parent" value f <comment>
                 "content" value >>content
-                logged-in-user get username>> >>author
+                username >>author
                 now >>date
             [ insert-tuple ] [ entity-url <redirect> ] bi
         ] >>submit

extra/webapps/todo/todo.factor

@@ -32,7 +32,7 @@ todo "TODO"
 : <todo> ( id -- todo )
     todo new
         swap >>id
-        logged-in-user get username>> >>uid ;
+        username >>uid ;
 
 : <view-action> ( -- action )
     <page-action>

extra/webapps/wiki/changes.xml

@@ -4,26 +4,4 @@
 
 	<t:title>Recent Changes</t:title>
 
-	<div class="revisions">
-
-		<table>
-
-			<tr>
-				<th>Article</th>
-				<th>Date</th>
-				<th>By</th>
-			</tr>
-
-			<t:bind-each t:name="changes">
-				<tr>
-					<td><t:a t:href="$wiki/view" t:rest="title"><t:label t:name="title" /></t:a></td>
-					<td><t:a t:href="$wiki/revision" t:rest="id"><t:label t:name="date" /></t:a></td>
-					<td><t:a t:href="$wiki/user-edits" t:rest="author"><t:label t:name="author" /></t:a></td>
-				</tr>
-			</t:bind-each>
-
-		</table>
-
-	</div>
-
 </t:chloe>

extra/webapps/wiki/diff.xml

@@ -13,7 +13,7 @@
 		</tr>
 		<tr>
 			<th class="field-label">New revision:</th>
-			<t:bind t:name="old">
+			<t:bind t:name="new">
 				<td>Created on <t:label t:name="date" /> by <t:a t:href="user-edits" t:rest="author"><t:label t:name="author" /></t:a>.</td>
 			</t:bind>
 		</tr>

extra/webapps/wiki/edit.xml

@@ -4,12 +4,17 @@
 
 	<t:title>Edit: <t:label t:name="title" /></t:title>
 
-	<t:form t:action="$wiki/edit" t:for="title">
+	<t:form t:action="$wiki/submit" t:for="title">
 
 		<p>
 			<t:textarea t:name="content" t:rows="30" t:cols="80" />
 		</p>
 
+		<p>
+			Describe this revision:
+			<t:field t:name="description" t:size="60" />
+		</p>
+
 		<p>
 			<input type="submit" value="Save" />
 		</p>

extra/webapps/wiki/revisions-common.xml

@@ -0,0 +1,33 @@
+<?xml version='1.0' ?>
+
+<t:chloe xmlns:t="http://factorcode.org/chloe/1.0">
+
+	<div class="revisions">
+
+		<table>
+
+			<tr>
+				<th>Article</th>
+				<th>Date</th>
+				<th>By</th>
+				<th>Description</th>
+				<th>Rollback</th>
+			</tr>
+
+			<t:bind-each t:name="revisions">
+				<tr>
+					<td> <t:a t:href="$wiki/view" t:rest="title"><t:label t:name="title" /></t:a> </td>
+					<td> <t:a t:href="$wiki/revision" t:rest="id"><t:label t:name="date" /></t:a> </td>
+					<td> <t:a t:href="$wiki/user-edits" t:rest="author"><t:label t:name="author" /> </t:a></td>
+					<td> <t:label t:name="description" /> </td>
+					<td> <t:button class="link link-button" t:action="$wiki/rollback" t:for="id">Rollback</t:button> </td>
+				</tr>
+			</t:bind-each>
+
+		</table>
+
+	</div>
+
+	<t:call-next-template />
+
+</t:chloe>

extra/webapps/wiki/revisions.xml

@@ -4,24 +4,6 @@
 
 	<t:title>Revisions of <t:label t:name="title" /></t:title>
 
-	<div class="revisions">
-		<table>
-			<tr>
-				<th>Revision</th>
-				<th>By</th>
-				<th>Rollback</th>
-			</tr>
-
-			<t:bind-each t:name="revisions">
-				<tr>
-					<td> <t:a t:href="$wiki/revision" t:rest="id"><t:label t:name="date" /></t:a> </td>
-					<td> <t:a t:href="$wiki/user-edits" t:rest="author"><t:label t:name="author" /></t:a> </td>
-					<td> <t:button t:action="$wiki/rollback" t:for="id" class="link link-button">Rollback</t:button> </td>
-				</tr>
-			</t:bind-each>
-		</table>
-	</div>
-
 	<h2>View Differences</h2>
 
 	<t:form t:action="$wiki/diff" t:method="get">

extra/webapps/wiki/user-edits.xml

@@ -8,14 +8,4 @@
 
 	<t:title>Edits by <t:label t:name="author" /></t:title>
 
-	<ul>
-		<t:bind-each t:name="user-edits">
-			<li>
-				<t:a t:href="$wiki/view" t:rest="title"><t:label t:name="title" /></t:a>
-				on
-				<t:a t:href="$wiki/revision" t:rest="id"><t:label t:name="date" /></t:a>
-			</li>
-		</t:bind-each>
-	</ul>
-
 </t:chloe>

extra/webapps/wiki/view.xml

@@ -8,6 +8,12 @@
 		<t:farkup t:name="content" />
 	</div>
 
-	<p><em>This revision created on <t:label t:name="date" /> by <t:a t:href="$wiki/user-edits" t:rest="author"><t:label t:name="author" /></t:a>.</em></p>
+	<p>
+		<em>This revision created on <t:label t:name="date" /> by <t:a t:href="$wiki/user-edits" t:rest="author"><t:label t:name="author" /></t:a>
+			<t:if t:value="description">
+				(<t:label t:name="description" />)
+			</t:if>
+		</em>
+	</p>
 
 </t:chloe>