More changes wrt encodings

core/bootstrap/image/image.factor

@@ -416,7 +416,8 @@ M: curry '
     "Writing image to " write
     architecture get boot-image-name resource-path
     dup write "..." print flush
-    binary [ (write-image) ] with-file-writer ;
+    ! binary
+    <file-writer> [ (write-image) ] with-stream ;
 
 PRIVATE>
 

core/io/encodings/ascii/ascii.factor

@@ -1,8 +1,10 @@
-USING: io io.encodings strings kernel ;
+! Copyright (C) 2008 Daniel Ehrenberg.
+! See http://factorcode.org/license.txt for BSD license.
+USING: io io.encodings strings kernel math sequences byte-arrays io.encodings ;
 IN: io.encodings.ascii
 
 : encode-check>= ( string max -- byte-array )
-    dupd [ >= ] curry all? [ >byte-array ] [ encoding-error ] if ;
+    dupd [ >= ] curry all? [ >byte-array ] [ encode-error ] if ;
 
 TUPLE: ascii ;
 
@@ -10,4 +12,4 @@ M: ascii encode-string
     drop 127 encode-check>= ;
 
 M: ascii decode-step
-    3drop over push f f ;
+    3drop dup 127 >= [ encode-error ] when over push f f ;

core/io/encodings/ascii/authors.txt

@@ -0,0 +1 @@
+Daniel Ehrenberg

core/io/encodings/binary/binary.factor

@@ -1 +1,3 @@
+! Copyright (C) 2008 Daniel Ehrenberg.
+! See http://factorcode.org/license.txt for BSD license.
 IN: io.encodings.binary SYMBOL: binary

core/io/encodings/encodings.factor

@@ -1,8 +1,9 @@
-! Copyright (C) 2006, 2007 Daniel Ehrenberg.
+! Copyright (C) 2008 Daniel Ehrenberg.
 ! See http://factorcode.org/license.txt for BSD license.
 USING: math kernel sequences sbufs vectors namespaces
-growable strings io classes continuations
+growable strings io classes continuations combinators
-io.styles io.streams.nested io.encodings.binary ;
+io.styles io.streams.plain io.encodings.binary splitting
+io.streams.string io.streams.duplex ;
 IN: io.encodings
 
 ! Decoding
@@ -134,18 +135,20 @@ M: encoded stream-write1
     >r 1string r> stream-write ;
 
 M: encoded stream-write
-    [ encoding-code encode-string ] keep delegate stream-write ;
+    [ encoded-code encode-string ] keep delegate stream-write ;
 
 M: encoded dispose delegate dispose ;
 
-M: encoded stream-nl
+INSTANCE: encoded plain-writer
-    CHAR: \n swap stream-write1 ;
 
-M: encoded stream-format
+! Rebinding duplex streams which have not read anything yet
-    nip stream-write ;
 
-M: encoded make-span-stream
+: reencode ( stream encoding -- newstream )
-    <style-stream> <ignore-close-stream> ;
+    over encoded? [ >r delegate r> ] when <encoded> ;
 
-M: encoded make-block-stream
+: redecode ( stream encoding -- newstream )
-    nip <ignore-close-stream> ;
+    over decoded? [ >r delegate r> ] when <decoded> ;
+
+: <encoded-duplex> ( duplex-stream encoding -- duplex-stream )
+    swap { duplex-stream-in duplex-stream-out } get-slots
+    pick reencode >r swap redecode r> <duplex-stream> ;

core/io/encodings/latin1/latin1.factor

@@ -1,4 +1,6 @@
-USING: io io.encodings strings kernel io.encodings.ascii ;
+! Copyright (C) 2008 Daniel Ehrenberg.
+! See http://factorcode.org/license.txt for BSD license.
+USING: io io.encodings strings kernel io.encodings.ascii sequences ;
 IN: io.encodings.latin1
 
 TUPLE: latin1 ;

core/io/encodings/utf16/utf16.factor

@@ -1,4 +1,4 @@
-! Copyright (C) 2006, 2007 Daniel Ehrenberg.
+! Copyright (C) 2006, 2008 Daniel Ehrenberg.
 ! See http://factorcode.org/license.txt for BSD license.
 USING: math kernel sequences sbufs vectors namespaces io.binary
 io.encodings combinators splitting io byte-arrays ;
@@ -116,19 +116,16 @@ SYMBOL: ignore
     } cond ;
 
 TUPLE: utf16le ;
-INSTANCE: utf16le encoding-stream 
 
 M: utf16le encode-string drop encode-utf16le ;
 M: utf16le decode-step drop decode-utf16le-step ;
 
 TUPLE: utf16be ;
-INSTANCE: utf16be encoding-stream 
 
 M: utf16be encode-string drop encode-utf16be ;
 M: utf16be decode-step drop decode-utf16be-step ;
 
 TUPLE: utf16 encoding ;
-INSTANCE: utf16 encoding-stream
 M: utf16 underlying-stream delegate dup delegate [ ] [ ] ?if ; ! necessary? 
 M: utf16 set-underlying-stream delegate set-delegate ; ! necessary? 
 

core/io/encodings/utf8/utf8.factor

@@ -1,7 +1,7 @@
-! Copyright (C) 2006, 2007 Daniel Ehrenberg.
+! Copyright (C) 2006, 2008 Daniel Ehrenberg.
 ! See http://factorcode.org/license.txt for BSD license.
 USING: math kernel sequences sbufs vectors growable io continuations
-namespaces io.encodings combinators strings io.streams.c ;
+namespaces io.encodings combinators strings ;
 IN: io.encodings.utf8
 
 ! Decoding UTF-8
@@ -78,7 +78,6 @@ SYMBOL: quad3
 ! Interface for streams
 
 TUPLE: utf8 ;
-INSTANCE: utf8 encoding-stream 
 
 M: utf8 encode-string drop encode-utf8 ;
 M: utf8 decode-step drop decode-utf8-step ;

core/io/files/authors.txt

@@ -1 +1,2 @@
 Slava Pestov
+Daniel Ehrenberg

core/io/files/files.factor

@@ -1,9 +1,10 @@
-! Copyright (C) 2004, 2008 Slava Pestov.
+! Copyright (C) 2004, 2008 Slava Pestov, Daniel Ehrenberg.
 ! See http://factorcode.org/license.txt for BSD license.
 IN: io.files
 USING: io.backend io.files.private io hashtables kernel math
 memory namespaces sequences strings assocs arrays definitions
-system combinators splitting sbufs continuations io.encodings ;
+system combinators splitting sbufs continuations io.encodings
+io.encodings.binary ;
 
 HOOK: cd io-backend ( path -- )
 
@@ -16,13 +17,13 @@ HOOK: file-writer* io-backend ( path -- stream )
 HOOK: file-appender* io-backend ( path -- stream )
 
 : <file-reader> ( path encoding -- stream )
-    swap file-reader* swap <decoding> ;
+    swap file-reader* swap <decoded> ;
 
 : <file-writer> ( path encoding -- stream )
-    swap file-writer* swap <encoding> ;
+    swap file-writer* swap <encoded> ;
 
 : <file-appender> ( path encoding -- stream )
-    swap file-appender* swap <encoding> ;
+    swap file-appender* swap <encoded> ;
 
 HOOK: delete-file io-backend ( path -- )
 

core/io/streams/c/c.factor

@@ -1,8 +1,9 @@
 ! Copyright (C) 2004, 2008 Slava Pestov.
 ! See http://factorcode.org/license.txt for BSD license.
-USING: kernel kernel.private namespaces io
+USING: kernel kernel.private namespaces io io.encodings
 strings sequences math generic threads.private classes
-io.backend io.streams.duplex io.files continuations ;
+io.backend io.streams.duplex io.files continuations
+io.encodings.utf8 ;
 IN: io.streams.c
 
 TUPLE: c-writer handle ;
@@ -49,9 +50,7 @@ M: c-reader dispose
     c-reader-handle fclose ;
 
 : <duplex-c-stream> ( in out -- stream )
-    >r <c-reader> <line-reader> r>
+    >r <c-reader> r> <c-writer> <duplex-stream> ;
-    <c-writer> <plain-writer>
-    <duplex-stream> ;
 
 M: object init-io ;
 
@@ -60,8 +59,9 @@ M: object init-io ;
 : stderr-handle 38 getenv ;
 
 M: object init-stdio
-    stdin-handle stdout-handle <duplex-c-stream> stdio set-global
+    stdin-handle stdout-handle <duplex-c-stream>
-    stderr-handle <c-writer> <plain-writer> stderr set-global ;
+    utf8 <encoded-duplex> stdio set-global
+    stderr-handle <c-writer> utf8 <encoded> stderr set-global ;
 
 M: object io-multiplex (sleep) ;
 

core/io/streams/plain/plain.factor

@@ -1,7 +1,18 @@
 ! Copyright (C) 2005, 2007 Slava Pestov.
 ! See http://factorcode.org/license.txt for BSD license.
+USING: kernel io io.streams.nested ;
 IN: io.streams.plain
-USING: io.encodings.latin1 io.encodings ;
 
-: <plain-writer> ( stream -- new-stream )
+MIXIN: plain-writer
-    latin1 <encoded> ; 
+
+M: plain-writer stream-nl
+    CHAR: \n swap stream-write1 ;
+
+M: plain-writer stream-format
+    nip stream-write ;
+
+M: plain-writer make-span-stream
+    <style-stream> <ignore-close-stream> ;
+
+M: plain-writer make-block-stream
+    nip <ignore-close-stream> ;

core/io/streams/string/string.factor

@@ -2,8 +2,7 @@
 ! See http://factorcode.org/license.txt for BSD license.
 IN: io.streams.string
 USING: io kernel math namespaces sequences sbufs strings
-generic splitting io.streams.plain io.streams.lines growable
+generic splitting growable continuations io.streams.plain ;
-continuations ;
 
 M: growable dispose drop ;
 
@@ -12,31 +11,12 @@ M: growable stream-write push-all ;
 M: growable stream-flush drop ;
 
 : <string-writer> ( -- stream )
-    512 <sbuf> <plain-writer> ;
+    512 <sbuf> ;
 
 : with-string-writer ( quot -- str )
     <string-writer> swap [ stdio get ] compose with-stream*
     >string ; inline
 
-: format-column ( seq ? -- seq )
-    [
-        [ 0 [ length max ] reduce ] keep
-        swap [ CHAR: \s pad-right ] curry map
-    ] unless ;
-
-: map-last ( seq quot -- seq )
-    swap dup length <reversed>
-    [ zero? rot [ call ] keep swap ] 2map nip ; inline
-
-: format-table ( table -- seq )
-    flip [ format-column ] map-last
-    flip [ " " join ] map ;
-
-M: plain-writer stream-write-table
-    [ drop format-table [ print ] each ] with-stream* ;
-
-M: plain-writer make-cell-stream 2drop <string-writer> ;
-
 M: growable stream-read1 dup empty? [ drop f ] [ pop ] if ;
 
 : harden-as ( seq growble-exemplar -- newseq )
@@ -69,7 +49,28 @@ M: growable stream-read-partial
     stream-read ;
 
 : <string-reader> ( str -- stream )
-    >sbuf dup reverse-here <line-reader> ;
+    >sbuf dup reverse-here ;
 
 : with-string-reader ( str quot -- )
     >r <string-reader> r> with-stream ; inline
+
+INSTANCE: growable plain-writer
+
+: format-column ( seq ? -- seq )
+    [
+        [ 0 [ length max ] reduce ] keep
+        swap [ CHAR: \s pad-right ] curry map
+    ] unless ;
+
+: map-last ( seq quot -- seq )
+    swap dup length <reversed>
+    [ zero? rot [ call ] keep swap ] 2map nip ; inline
+
+: format-table ( table -- seq )
+    flip [ format-column ] map-last
+    flip [ " " join ] map ;
+
+M: plain-writer stream-write-table
+    [ drop format-table [ print ] each ] with-stream* ;
+
+M: plain-writer make-cell-stream 2drop <string-writer> ;

extra/io/buffers/buffers-docs.factor

@@ -30,7 +30,7 @@ $nl
 ABOUT: "buffers"
 
 HELP: buffer
-{ $class-description "The class of I/O buffers, which resemble FIFO queues, but are optimize for holding bytes, are have underlying storage allocated at a fixed address. Buffers must be de-allocated manually."
+{ $class-description "The class of I/O buffers, which resemble FIFO queues, but are optimized for holding bytes, are have underlying storage allocated at a fixed address. Buffers must be de-allocated manually."
 $nl
 "Buffers have two internal pointers:"
 { $list

extra/io/launcher/launcher-docs.factor

@@ -148,8 +148,9 @@ HELP: process-stream
 HELP: <process-stream>
 { $values
   { "desc" "a launch descriptor" }
+  { "encoding" "an encoding descriptor" }
   { "stream" "a bidirectional stream" } }
-{ $description "Launches a process and redirects its input and output via a pair of pipes which may be read and written as a stream." }
+{ $description "Launches a process and redirects its input and output via a pair of pipes which may be read and written as a stream of the given encoding." }
 { $notes "Closing the stream will block until the process exits." } ;
 
 HELP: with-process-stream

extra/io/launcher/launcher.factor

@@ -117,12 +117,13 @@ M: process get-lapse process-lapse ;
 M: process timed-out kill-process ;
 
 HOOK: process-stream* io-backend ( desc -- stream process )
+! Process streams are always latin1 for now; will be updated
 
 TUPLE: process-stream process ;
 
-: <process-stream> ( desc -- stream )
+: <process-stream> ( desc encoding -- stream )
-    >descriptor
+    swap >descriptor
-    [ process-stream* ] keep
+    [ process-stream* >r swap <encoded-duplex> r> ] keep
     +timeout+ swap at [ over set-timeout ] when*
     { set-delegate set-process-stream-process }
     process-stream construct ;

extra/io/nonblocking/nonblocking.factor

@@ -45,10 +45,10 @@ GENERIC: close-handle ( handle -- )
 : <writer> ( handle -- stream )
     output-port <buffered-port> ;
 
-: handle>duplex-stream ( in-handle out-handle -- stream )
+: handle>duplex-stream ( in-handle out-handle encoding -- stream )
-    <writer>
+    [ swap <writer> swap <encoded> ] keep
-    [ >r <reader> r> <duplex-stream> ] [ ] [ dispose ]
+    [ -rot >r <reader> swap <decoded> r> <duplex-stream> ]
-    cleanup ;
+    [ ] [ dispose ] cleanup ;
 
 : pending-error ( port -- )
     dup port-error f rot set-port-error [ throw ] when* ;

extra/io/sockets/authors.txt

@@ -1 +1,2 @@
 Slava Pestov
+Daniel Ehrenberg

extra/io/sockets/sockets-docs.factor

@@ -92,11 +92,11 @@ HELP: inet6
 } ;
 
 HELP: <client>
-{ $values { "addrspec" "an address specifier" } { "stream" "a bidirectional stream" } }
+{ $values { "addrspec" "an address specifier" } { "encoding" "an encding descriptor" } { "stream" "a bidirectional stream" } }
-{ $description "Opens a network connection and outputs a bidirectional stream." }
+{ $description "Opens a network connection and outputs a bidirectional stream using the given encoding." }
 { $errors "Throws an error if the connection cannot be established." }
 { $examples
-    { $code "\"www.apple.com\" \"http\" <inet> <client>" }
+    { $code "\"www.apple.com\" \"http\" <inet> utf8 <client>" }
 } ;
 
 HELP: <server>

extra/io/sockets/sockets.factor

@@ -1,8 +1,8 @@
-! Copyright (C) 2007 Slava Pestov.
+! Copyright (C) 2007, 2008 Slava Pestov, Daniel Ehrenberg.
 ! See http://factorcode.org/license.txt for BSD license.
 IN: io.sockets
 USING: generic kernel io.backend namespaces continuations
-sequences arrays ;
+sequences arrays io.encodings ;
 
 TUPLE: local path ;
 
@@ -28,11 +28,12 @@ TUPLE: client-stream addr ;
 
 HOOK: (client) io-backend ( addrspec -- stream )
 
-GENERIC: <client> ( addrspec -- stream )
+GENERIC: client* ( addrspec -- stream )
+M: array client* [ (client) ] attempt-all ;
+M: object client* (client) ;
 
-M: array <client> [ (client) ] attempt-all ;
+: <client> ( addrspec encoding -- stream )
-
+    >r client* r> <encoded-duplex> ;
-M: object <client> (client) ;
 
 HOOK: <server> io-backend ( addrspec -- server )
 
@@ -48,7 +49,7 @@ HOOK: resolve-host io-backend ( host serv passive? -- seq )
 
 HOOK: host-name io-backend ( -- string )
 
-M: inet <client>
+M: inet client*
     dup inet-host swap inet-port f resolve-host
     dup empty? [ "Host name lookup failed" throw ] when
-    <client> ;
+    client* ;

extra/io/unix/backend/backend.factor

@@ -181,8 +181,8 @@ M: unix-io io-multiplex ( ms -- )
     mx get-global wait-for-events ;
 
 M: unix-io init-stdio ( -- )
-    0 1 handle>duplex-stream io:stdio set-global
+    0 1 utf8 handle>duplex-stream io:stdio set-global
-    2 <writer> io:stderr set-global ;
+    2 <writer> utf8 <encoded> io:stderr set-global ;
 
 ! mx io-task for embedding an fd-based mx inside another mx
 TUPLE: mx-port mx ;

extra/io/unix/launcher/launcher.factor

@@ -4,7 +4,7 @@ USING: io io.backend io.launcher io.unix.backend io.unix.files
 io.nonblocking sequences kernel namespaces math system
  alien.c-types debugger continuations arrays assocs 
 combinators unix.process parser-combinators memoize 
-promises strings threads unix ;
+promises strings threads unix io.encodings.latin1 ;
 IN: io.unix.launcher
 
 ! Search unix first
@@ -99,7 +99,7 @@ M: unix-io kill-process* ( pid -- )
 
 M: unix-io process-stream*
     [
-        spawn-process-stream >r handle>duplex-stream r>
+        spawn-process-stream >r latin1 handle>duplex-stream r>
     ] with-descriptor ;
 
 : find-process ( handle -- process )

extra/io/unix/sockets/sockets.factor

@@ -1,4 +1,4 @@
-! Copyright (C) 2004, 2008 Slava Pestov, Ivan Tikhonov.
+! Copyright (C) 2004, 2008 Slava Pestov, Ivan Tikhonov. 
 ! See http://factorcode.org/license.txt for BSD license.
 
 ! We need to fiddle with the exact search order here, since
@@ -92,7 +92,7 @@ USE: io.sockets
     dup rot make-sockaddr/size bind
     zero? [ dup close (io-error) ] unless ;
 
-M: unix-io <server> ( addrspec -- stream )
+M: unix-io <server> ( addrspec -- server )
     [
         SOCK_STREAM server-fd
         dup 10 listen zero? [ dup close (io-error) ] unless

extra/io/unix/unix-tests.factor

@@ -1,6 +1,6 @@
 USING: io.files io.sockets io kernel threads namespaces
 tools.test continuations strings byte-arrays sequences
-prettyprint system io.encodings.binary ;
+prettyprint system io.encodings.binary io.encodings.ascii ;
 IN: temporary
 
 ! Unix domain stream sockets
@@ -24,7 +24,7 @@ yield
 
 [ { "Hello world" "FOO" } ] [
     [
-        "unix-domain-socket-test" resource-path <local> <client>
+        "unix-domain-socket-test" resource-path <local> ascii <client>
         [
             readln ,
             "XYZ" print flush

extra/irc/irc.factor

@@ -1,7 +1,8 @@
 ! Copyright (C) 2007 Doug Coleman, Slava Pestov.
 ! See http://factorcode.org/license.txt for BSD license.
 USING: arrays calendar io io.sockets kernel match namespaces
-sequences splitting strings continuations threads ascii ;
+sequences splitting strings continuations threads ascii
+io.encodings.utf8 ;
 IN: irc
 
 ! "setup" objects
@@ -97,7 +98,7 @@ SYMBOL: irc-client
     " hostname servername :irc.factor" irc-print ;
 
 : connect* ( server port -- )
-    <inet> <client> irc-client get set-irc-client-stream ;
+    <inet> utf8 <client> irc-client get set-irc-client-stream ;
 
 : connect ( server -- ) 6667 connect* ;
 

extra/smtp/smtp.factor

@@ -3,7 +3,7 @@
 ! See http://factorcode.org/license.txt for BSD license.
 USING: namespaces io io.timeouts kernel logging io.sockets
 sequences combinators sequences.lib splitting assocs strings
-math.parser random system calendar ;
+math.parser random system calendar io.encodings.ascii ;
 
 IN: smtp
 
@@ -20,7 +20,7 @@ SYMBOL: esmtp           t esmtp set-global
 : with-smtp-connection ( quot -- )
     smtp-host get smtp-port get
     2dup log-smtp-connection
-    <inet> <client> [
+    <inet> ascii <client> [ ! ASCII until encodings reconsidered
         smtp-domain [ host-name or ] change
         read-timeout get stdio get set-timeout
         call
@@ -180,4 +180,4 @@ TUPLE: email from to subject body ;
 
 : send ( email -- )
   { email-body email-subject email-to email-from } get-slots
-  send-simple-message ;
+  send-simple-message ;

extra/tools/deploy/backend/backend.factor

@@ -6,7 +6,7 @@ continuations math definitions mirrors splitting parser classes
 inspector layouts vocabs.loader prettyprint.config prettyprint
 debugger io.streams.c io.streams.duplex io.files io.backend
 quotations io.launcher words.private tools.deploy.config
-bootstrap.image ;
+bootstrap.image io.encodings.utf8 ;
 IN: tools.deploy.backend
 
 : (copy-lines) ( stream -- )
@@ -20,7 +20,7 @@ IN: tools.deploy.backend
     [
         +arguments+ set
         +stdout+ +stderr+ set
-    ] H{ } make-assoc <process-stream>
+    ] H{ } make-assoc utf8 <process-stream>
     dup duplex-stream-out dispose
     dup copy-lines
     process-stream-process wait-for-process zero? [

extra/webapps/cgi/cgi.factor

@@ -2,7 +2,7 @@
 ! See http://factorcode.org/license.txt for BSD license.
 USING: namespaces kernel assocs io.files combinators
 arrays io.launcher io http.server.responders webapps.file
-sequences strings math.parser unicode.case ;
+sequences strings math.parser unicode.case io.encodings.binary ;
 IN: webapps.cgi
 
 SYMBOL: cgi-root
@@ -50,7 +50,7 @@ SYMBOL: cgi-root
     
 : (do-cgi) ( name -- )
     "200 CGI output follows" response
-    stdio get swap cgi-descriptor <process-stream> [
+    stdio get swap cgi-descriptor binary <process-stream> [
         post? [
             "raw-response" get write flush
         ] when