From 62f9ed5dbd213b731263a9652389e8309e580eb9 Mon Sep 17 00:00:00 2001 From: Daniel Ehrenberg Date: Sat, 16 Feb 2008 22:17:41 -0600 Subject: [PATCH] putting encodings on all file readers/writers --- core/alien/c-types/c-types.factor | 5 +++- core/bootstrap/image/image.factor | 4 +-- core/io/encodings/binary/binary.factor | 3 +-- core/io/encodings/encodings.factor | 26 ++++++++++++------- core/io/encodings/utf16/utf16.factor | 6 ++--- core/io/encodings/utf8/utf8.factor | 4 +-- core/io/files/files-tests.factor | 22 ++++++++-------- core/io/io-tests.factor | 6 ++--- core/io/streams/c/c-tests.factor | 2 +- core/io/streams/c/c.factor | 3 +-- core/io/streams/lines/lines.factor | 8 +++--- core/parser/parser.factor | 4 +-- core/source-files/source-files.factor | 6 ++--- extra/benchmark/fasta/fasta.factor | 4 +-- .../benchmark/knucleotide/knucleotide.factor | 4 +-- extra/benchmark/mandel/mandel.factor | 4 +-- extra/benchmark/raytracer/raytracer.factor | 4 +-- .../reverse-complement.factor | 4 +-- extra/benchmark/sum-file/sum-file.factor | 4 +-- extra/bootstrap/image/upload/upload.factor | 4 +-- extra/bunny/model/model.factor | 4 +-- extra/cpu/8080/emulator/emulator.factor | 6 ++--- extra/cryptlib/cryptlib-tests.factor | 7 ++--- extra/crypto/hmac/hmac.factor | 6 ++--- extra/crypto/md5/md5.factor | 5 ++-- extra/crypto/sha1/sha1.factor | 4 +-- extra/editors/jedit/jedit.factor | 4 +-- extra/furnace/furnace.factor | 4 +-- extra/graphics/bitmap/bitmap.factor | 6 ++--- extra/help/cookbook/cookbook.factor | 4 +-- extra/http/client/client.factor | 4 +-- .../server/templating/templating-tests.factor | 4 +-- .../http/server/templating/templating.factor | 6 ++--- extra/icfp/2006/2006.factor | 4 +-- extra/id3/id3.factor | 10 +++---- extra/io/mmap/mmap-tests.factor | 7 ++--- extra/io/unix/unix-tests.factor | 8 +++--- extra/log-viewer/log-viewer.factor | 4 +-- extra/logging/insomniac/insomniac.factor | 5 ++-- extra/logging/server/server.factor | 4 +-- extra/mad/api/api.factor | 6 ++--- extra/msxml-to-csv/msxml-to-csv.factor | 4 +-- extra/ogg/player/player.factor | 7 ++--- .../porter-stemmer-tests.factor | 4 +-- extra/project-euler/042/042.factor | 4 +-- extra/project-euler/067/067.factor | 5 ++-- extra/project-euler/079/079.factor | 5 ++-- extra/raptor/raptor.factor | 4 +-- extra/rss/rss-tests.factor | 4 +-- extra/tar/tar.factor | 6 ++--- extra/tools/browser/browser.factor | 6 ++--- extra/tools/deploy/macosx/macosx.factor | 4 +-- extra/tools/disassembler/disassembler.factor | 6 ++--- extra/ui/freetype/freetype.factor | 2 +- extra/unicode/breaks/breaks.factor | 4 +-- extra/unicode/data/data.factor | 4 +-- extra/webapps/file/file.factor | 4 +-- extra/webapps/source/source.factor | 4 +-- extra/xml/xml.factor | 5 ++-- extra/xmode/catalog/catalog.factor | 6 ++--- extra/xmode/code2html/code2html.factor | 6 ++--- extra/yahoo/yahoo-tests.factor | 2 +- 62 files changed, 174 insertions(+), 161 deletions(-) diff --git a/core/alien/c-types/c-types.factor b/core/alien/c-types/c-types.factor index 7d01fb2b00..983fea0159 100755 --- a/core/alien/c-types/c-types.factor +++ b/core/alien/c-types/c-types.factor @@ -4,7 +4,7 @@ USING: bit-arrays byte-arrays float-arrays arrays generator.registers assocs kernel kernel.private libc math namespaces parser sequences strings words assocs splitting math.parser cpu.architecture alien alien.accessors quotations -system compiler.units ; +system compiler.units io.encodings.binary ; IN: alien.c-types DEFER: @@ -273,6 +273,9 @@ M: long-long-type box-return ( type -- ) r> add* ] when ; +: malloc-file-contents ( path -- alien ) + binary file-contents >byte-array malloc-byte-array ; + [ [ alien-cell ] [ set-alien-cell ] diff --git a/core/bootstrap/image/image.factor b/core/bootstrap/image/image.factor index 17b56458ce..73085450a8 100755 --- a/core/bootstrap/image/image.factor +++ b/core/bootstrap/image/image.factor @@ -7,7 +7,7 @@ strings sbufs vectors words quotations assocs system layouts splitting growable classes tuples words.private io.binary io.files vocabs vocabs.loader source-files definitions debugger float-arrays quotations.private -sequences.private combinators ; +sequences.private combinators io.encodings.binary ; IN: bootstrap.image : my-arch ( -- arch ) @@ -416,7 +416,7 @@ M: curry ' "Writing image to " write architecture get boot-image-name resource-path dup write "..." print flush - [ (write-image) ] with-file-writer ; + binary [ (write-image) ] with-file-writer ; PRIVATE> diff --git a/core/io/encodings/binary/binary.factor b/core/io/encodings/binary/binary.factor index 17c734b9c8..8a8d09464b 100644 --- a/core/io/encodings/binary/binary.factor +++ b/core/io/encodings/binary/binary.factor @@ -1,2 +1 @@ -IN: io.encodings.binary -SYMBOL: binary +IN: io.encodings.binary SYMBOL: binary diff --git a/core/io/encodings/encodings.factor b/core/io/encodings/encodings.factor index 4ceae70fae..cd9d4b585b 100755 --- a/core/io/encodings/encodings.factor +++ b/core/io/encodings/encodings.factor @@ -1,7 +1,7 @@ ! Copyright (C) 2006, 2007 Daniel Ehrenberg. ! See http://factorcode.org/license.txt for BSD license. USING: math kernel sequences sbufs vectors namespaces -growable strings io classes io.streams.c continuations +growable strings io classes continuations io.styles io.streams.nested io.encodings.binary ; IN: io.encodings @@ -13,12 +13,12 @@ TUPLE: decode-error ; SYMBOL: begin -: decoded ( buf ch -- buf ch state ) +: push-decoded ( buf ch -- buf ch state ) over push 0 begin ; : push-replacement ( buf -- buf ch state ) ! This is the replacement character - HEX: fffd decoded ; + HEX: fffd push-decoded ; : finish-decoding ( buf ch state -- str ) begin eq? [ decode-error ] unless drop "" like ; @@ -59,16 +59,16 @@ TUPLE: decoded code cr ; decoded construct ] if ; -: cr+ t swap set-line-reader-cr ; inline +: cr+ t swap set-decoded-cr ; inline -: cr- f swap set-line-reader-cr ; inline +: cr- f swap set-decoded-cr ; inline : line-ends/eof ( stream str -- str ) f like swap cr- ; inline : line-ends\r ( stream str -- str ) swap cr+ ; inline : line-ends\n ( stream str -- str ) - over line-reader-cr over empty? and + over decoded-cr over empty? and [ drop dup cr- stream-readln ] [ swap cr- ] if ; inline : handle-readln ( stream str ch -- str ) @@ -79,7 +79,7 @@ TUPLE: decoded code cr ; } case ; : fix-read ( stream string -- string ) - over line-reader-cr [ + over decoded-cr [ over cr- "\n" ?head [ swap stream-read1 [ add ] when* @@ -91,13 +91,21 @@ M: decoded stream-read M: decoded stream-read-partial tuck stream-read fix-read ; +: read-until-loop ( stream delim -- ch ) + ! Copied from { c-reader stream-read-until }!!! + over stream-read1 dup [ + dup pick memq? [ 2nip ] [ , read-until-loop ] if + ] [ + 2nip + ] if ; + M: decoded stream-read-until ! Copied from { c-reader stream-read-until }!!! [ swap read-until-loop ] "" make swap over empty? over not and [ 2drop f f ] when ; : fix-read1 ( stream char -- char ) - over line-reader-cr [ + over decoded-cr [ over cr- dup CHAR: \n = [ drop stream-read1 @@ -106,7 +114,7 @@ M: decoded stream-read-until M: decoded stream-read1 1 over stream-read ; -M: line-reader stream-readln ( stream -- str ) +M: decoded stream-readln ( stream -- str ) "\r\n" over stream-read-until handle-readln ; ! Encoding diff --git a/core/io/encodings/utf16/utf16.factor b/core/io/encodings/utf16/utf16.factor index 35b6282e21..1dd317d3c0 100755 --- a/core/io/encodings/utf16/utf16.factor +++ b/core/io/encodings/utf16/utf16.factor @@ -16,7 +16,7 @@ SYMBOL: ignore 8 shift bitor ; : end-multibyte ( buf byte ch -- buf ch state ) - append-nums decoded ; + append-nums push-decoded ; : begin-utf16be ( buf byte -- buf ch state ) dup -3 shift BIN: 11011 number= [ @@ -36,7 +36,7 @@ SYMBOL: ignore { double [ end-multibyte ] } { quad1 [ append-nums quad2 ] } { quad2 [ handle-quad2be ] } - { quad3 [ append-nums HEX: 10000 + decoded ] } + { quad3 [ append-nums HEX: 10000 + push-decoded ] } { ignore [ 2drop push-replacement ] } } case ; @@ -52,7 +52,7 @@ SYMBOL: ignore : handle-quad3le ( buf byte ch -- buf ch state ) swap dup -2 shift BIN: 110111 = [ - BIN: 11 bitand append-nums HEX: 10000 + decoded + BIN: 11 bitand append-nums HEX: 10000 + push-decoded ] [ 2drop push-replacement ] if ; : decode-utf16le-step ( buf byte ch state -- buf ch state ) diff --git a/core/io/encodings/utf8/utf8.factor b/core/io/encodings/utf8/utf8.factor index 6a3a8b8ec7..7f211f92de 100644 --- a/core/io/encodings/utf8/utf8.factor +++ b/core/io/encodings/utf8/utf8.factor @@ -23,7 +23,7 @@ SYMBOL: quad3 : begin-utf8 ( buf byte -- buf ch state ) { - { [ dup -7 shift zero? ] [ decoded ] } + { [ dup -7 shift zero? ] [ push-decoded ] } { [ dup -5 shift BIN: 110 number= ] [ BIN: 11111 bitand double ] } { [ dup -4 shift BIN: 1110 number= ] [ BIN: 1111 bitand triple ] } { [ dup -3 shift BIN: 11110 number= ] [ BIN: 111 bitand quad ] } @@ -31,7 +31,7 @@ SYMBOL: quad3 } cond ; : end-multibyte ( buf byte ch -- buf ch state ) - f append-nums [ decoded ] unless* ; + f append-nums [ push-decoded ] unless* ; : decode-utf8-step ( buf byte ch state -- buf ch state ) { diff --git a/core/io/files/files-tests.factor b/core/io/files/files-tests.factor index d0f9737f19..a92a52a024 100755 --- a/core/io/files/files-tests.factor +++ b/core/io/files/files-tests.factor @@ -1,34 +1,34 @@ IN: temporary -USING: tools.test io.files io threads kernel continuations ; +USING: tools.test io.files io threads kernel continuations io.encodings.ascii ; [ "passwd" ] [ "/etc/passwd" file-name ] unit-test [ "awk" ] [ "/usr/libexec/awk/" file-name ] unit-test [ "awk" ] [ "/usr/libexec/awk///" file-name ] unit-test [ ] [ - "test-foo.txt" resource-path [ + "test-foo.txt" resource-path ascii [ "Hello world." print ] with-file-writer ] unit-test [ ] [ - "test-foo.txt" resource-path [ + "test-foo.txt" resource-path ascii [ "Hello appender." print - ] with-stream + ] with-file-appender ] unit-test [ ] [ - "test-bar.txt" resource-path [ + "test-bar.txt" resource-path ascii [ "Hello appender." print - ] with-stream + ] with-file-appender ] unit-test [ "Hello world.\nHello appender.\n" ] [ - "test-foo.txt" resource-path file-contents + "test-foo.txt" resource-path ascii file-contents ] unit-test [ "Hello appender.\n" ] [ - "test-bar.txt" resource-path file-contents + "test-bar.txt" resource-path ascii file-contents ] unit-test [ ] [ "test-foo.txt" resource-path delete-file ] unit-test @@ -42,7 +42,7 @@ USING: tools.test io.files io threads kernel continuations ; [ ] [ "test-blah" resource-path make-directory ] unit-test [ ] [ - "test-blah/fooz" resource-path dispose + "test-blah/fooz" resource-path ascii dispose ] unit-test [ t ] [ @@ -55,11 +55,11 @@ USING: tools.test io.files io threads kernel continuations ; [ f ] [ "test-blah" resource-path exists? ] unit-test -[ ] [ "test-quux.txt" resource-path [ [ yield "Hi" write ] in-thread ] with-file-writer ] unit-test +[ ] [ "test-quux.txt" resource-path ascii [ [ yield "Hi" write ] in-thread ] with-file-writer ] unit-test [ ] [ "test-quux.txt" resource-path delete-file ] unit-test -[ ] [ "test-quux.txt" resource-path [ [ yield "Hi" write ] in-thread ] with-file-writer ] unit-test +[ ] [ "test-quux.txt" resource-path ascii [ [ yield "Hi" write ] in-thread ] with-file-writer ] unit-test [ ] [ "test-quux.txt" "quux-test.txt" [ resource-path ] 2apply rename-file ] unit-test [ t ] [ "quux-test.txt" resource-path exists? ] unit-test diff --git a/core/io/io-tests.factor b/core/io/io-tests.factor index 00a8078da8..74b6b5034f 100644 --- a/core/io/io-tests.factor +++ b/core/io/io-tests.factor @@ -1,5 +1,5 @@ USING: arrays io io.files kernel math parser strings system -tools.test words namespaces io.encodings.ascii ; +tools.test words namespaces io.encodings.ascii io.encodings.binary ; IN: temporary [ f ] [ @@ -8,7 +8,7 @@ IN: temporary ] unit-test : ( resource -- stream ) - resource-path binary ascii ; + resource-path ascii ; [ "This is a line.\rThis is another line.\r" @@ -53,7 +53,7 @@ IN: temporary ] unit-test [ ] [ - image [ + image binary [ 10 [ 65536 read drop ] times ] with-file-reader ] unit-test diff --git a/core/io/streams/c/c-tests.factor b/core/io/streams/c/c-tests.factor index 5ace929ceb..66ea460126 100755 --- a/core/io/streams/c/c-tests.factor +++ b/core/io/streams/c/c-tests.factor @@ -2,7 +2,7 @@ USING: tools.test io.files io io.streams.c ; IN: temporary [ "hello world" ] [ - "test.txt" resource-path [ + "test.txt" resource-path ascii [ "hello world" write ] with-file-writer diff --git a/core/io/streams/c/c.factor b/core/io/streams/c/c.factor index 9c1a099318..73b8bb32b9 100755 --- a/core/io/streams/c/c.factor +++ b/core/io/streams/c/c.factor @@ -2,8 +2,7 @@ ! See http://factorcode.org/license.txt for BSD license. USING: kernel kernel.private namespaces io strings sequences math generic threads.private classes -io.backend io.streams.lines io.streams.plain io.streams.duplex -io.files continuations ; +io.backend io.streams.duplex io.files continuations ; IN: io.streams.c TUPLE: c-writer handle ; diff --git a/core/io/streams/lines/lines.factor b/core/io/streams/lines/lines.factor index a6a5721ad8..84ff272d69 100755 --- a/core/io/streams/lines/lines.factor +++ b/core/io/streams/lines/lines.factor @@ -1,9 +1,7 @@ ! Copyright (C) 2004, 2007 Slava Pestov. ! See http://factorcode.org/license.txt for BSD license. IN: io.streams.lines -USING: io.encodings.latin1 io.encodings ; +! USING: io.encodings.latin1 io.encodings ; -TUPLE: line-reader cr ; - -: ( stream -- new-stream ) - latin1 ; +! : ( stream -- new-stream ) +! latin1 ; diff --git a/core/parser/parser.factor b/core/parser/parser.factor index fc29445f88..2f7beb56d7 100755 --- a/core/parser/parser.factor +++ b/core/parser/parser.factor @@ -4,7 +4,7 @@ USING: arrays definitions generic assocs kernel math namespaces prettyprint sequences strings vectors words quotations inspector io.styles io combinators sorting splitting math.parser effects continuations debugger -io.files io.streams.string io.streams.lines vocabs +io.files io.streams.string io.streams.lines vocabs io.encodings.utf8 source-files classes hashtables compiler.errors compiler.units ; IN: parser @@ -489,7 +489,7 @@ SYMBOL: interactive-vocabs [ [ [ parsing-file ] keep - [ ?resource-path ] keep + [ ?resource-path utf8 ] keep parse-stream ] with-compiler-errors ] [ diff --git a/core/source-files/source-files.factor b/core/source-files/source-files.factor index c7539ad3eb..d2566982f8 100755 --- a/core/source-files/source-files.factor +++ b/core/source-files/source-files.factor @@ -5,7 +5,7 @@ namespaces prettyprint sequences strings vectors words quotations inspector io.styles io combinators sorting splitting math.parser effects continuations debugger io.files io.crc32 io.streams.string io.streams.lines vocabs -hashtables graphs compiler.units ; +hashtables graphs compiler.units io.encodings.utf8 ; IN: source-files SYMBOL: source-files @@ -17,7 +17,7 @@ uses definitions ; : (source-modified?) ( path modified checksum -- ? ) pick file-modified rot [ 0 or ] 2apply > - [ swap file-lines lines-crc32 = not ] [ 2drop f ] if ; + [ swap utf8 file-lines lines-crc32 = not ] [ 2drop f ] if ; : source-modified? ( path -- ? ) dup source-files get at [ @@ -68,7 +68,7 @@ uses definitions ; : reset-checksums ( -- ) source-files get [ swap ?resource-path dup exists? - [ file-lines swap record-checksum ] [ 2drop ] if + [ utf8 file-lines swap record-checksum ] [ 2drop ] if ] assoc-each ; M: pathname where pathname-string 1 2array ; diff --git a/extra/benchmark/fasta/fasta.factor b/extra/benchmark/fasta/fasta.factor index 75321def2d..a68a84fc59 100644 --- a/extra/benchmark/fasta/fasta.factor +++ b/extra/benchmark/fasta/fasta.factor @@ -1,6 +1,6 @@ ! Based on http://shootout.alioth.debian.org/gp4/benchmark.php?test=fasta&lang=java&id=2 USING: math kernel io io.files locals multiline assocs sequences -sequences.private benchmark.reverse-complement hints +sequences.private benchmark.reverse-complement hints io.encodings.ascii byte-arrays float-arrays ; IN: benchmark.fasta @@ -94,7 +94,7 @@ HINTS: random fixnum ; n [ ] seed [ initial-seed ] | - out [ + out ascii [ n 2 * ALU "Homo sapiens alu" "ONE" write-repeat-fasta initial-seed diff --git a/extra/benchmark/knucleotide/knucleotide.factor b/extra/benchmark/knucleotide/knucleotide.factor index b95e182bd1..e06b81f6de 100644 --- a/extra/benchmark/knucleotide/knucleotide.factor +++ b/extra/benchmark/knucleotide/knucleotide.factor @@ -1,4 +1,4 @@ -USING: kernel io io.files splitting strings +USING: kernel io io.files splitting strings io.encodings.ascii hashtables sequences assocs math namespaces prettyprint math.parser combinators arrays sorting unicode.case ; @@ -57,7 +57,7 @@ IN: benchmark.knucleotide : knucleotide ( -- ) "extra/benchmark/knucleotide/knucleotide-input.txt" resource-path - [ read-input ] with-file-reader + ascii [ read-input ] with-file-reader process-input ; MAIN: knucleotide diff --git a/extra/benchmark/mandel/mandel.factor b/extra/benchmark/mandel/mandel.factor index 230fb2f889..a60fdbf6c7 100755 --- a/extra/benchmark/mandel/mandel.factor +++ b/extra/benchmark/mandel/mandel.factor @@ -1,6 +1,6 @@ IN: benchmark.mandel USING: arrays io kernel math namespaces sequences strings sbufs -math.functions math.parser io.files colors.hsv ; +math.functions math.parser io.files colors.hsv io.encodings.binary ; : max-color 360 ; inline : zoom-fact 0.8 ; inline @@ -66,6 +66,6 @@ SYMBOL: cols : mandel-main ( -- ) "mandel.ppm" resource-path - [ mandel write ] with-file-writer ; + binary [ mandel write ] with-file-writer ; MAIN: mandel-main diff --git a/extra/benchmark/raytracer/raytracer.factor b/extra/benchmark/raytracer/raytracer.factor index 8f2badc95f..9783389e80 100644 --- a/extra/benchmark/raytracer/raytracer.factor +++ b/extra/benchmark/raytracer/raytracer.factor @@ -3,7 +3,7 @@ USING: float-arrays compiler generic io io.files kernel math math.functions math.vectors math.parser namespaces sequences -sequences.private words ; +sequences.private words io.encodings.binary ; IN: benchmark.raytracer ! parameters @@ -171,6 +171,6 @@ DEFER: create ( level c r -- scene ) : raytracer-main "raytracer.pnm" resource-path - [ run write ] with-file-writer ; + binary [ run write ] with-file-writer ; MAIN: raytracer-main diff --git a/extra/benchmark/reverse-complement/reverse-complement.factor b/extra/benchmark/reverse-complement/reverse-complement.factor index cc20b4b349..2ae92223d1 100644 --- a/extra/benchmark/reverse-complement/reverse-complement.factor +++ b/extra/benchmark/reverse-complement/reverse-complement.factor @@ -1,6 +1,6 @@ USING: io io.files io.streams.duplex kernel sequences sequences.private strings vectors words memoize splitting -hints unicode.case ; +hints unicode.case io.encodings.latin1 ; IN: benchmark.reverse-complement MEMO: trans-map ( -- str ) @@ -32,7 +32,7 @@ HINTS: do-line vector string ; readln [ do-line (reverse-complement) ] [ show-seq ] if* ; : reverse-complement ( infile outfile -- ) - >r r> [ + latin1 >r latin1 r> [ 500000 (reverse-complement) ] with-stream ; diff --git a/extra/benchmark/sum-file/sum-file.factor b/extra/benchmark/sum-file/sum-file.factor index e17765d542..3db31f8887 100644 --- a/extra/benchmark/sum-file/sum-file.factor +++ b/extra/benchmark/sum-file/sum-file.factor @@ -1,11 +1,11 @@ -USING: io io.files math math.parser kernel prettyprint ; +USING: io io.files math math.parser kernel prettyprint io.encodings.ascii ; IN: benchmark.sum-file : sum-file-loop ( n -- n' ) readln [ string>number + sum-file-loop ] when* ; : sum-file ( file -- ) - [ 0 sum-file-loop ] with-file-reader . ; + ascii [ 0 sum-file-loop ] with-file-reader . ; : sum-file-main ( -- ) home "sum-file-in.txt" path+ sum-file ; diff --git a/extra/bootstrap/image/upload/upload.factor b/extra/bootstrap/image/upload/upload.factor index 552e26ebf5..4b8ddb0c4b 100755 --- a/extra/bootstrap/image/upload/upload.factor +++ b/extra/bootstrap/image/upload/upload.factor @@ -2,14 +2,14 @@ ! See http://factorcode.org/license.txt for BSD license. IN: bootstrap.image.upload USING: http.client crypto.md5 splitting assocs kernel io.files -bootstrap.image sequences io namespaces io.launcher math ; +bootstrap.image sequences io namespaces io.launcher math io.encodings.ascii ; : destination "slava@factorcode.org:www/images/latest/" ; : boot-image-names images [ boot-image-name ] map ; : compute-checksums ( -- ) - "checksums.txt" [ + "checksums.txt" ascii [ boot-image-names [ dup write bl file>md5str print ] each ] with-file-writer ; diff --git a/extra/bunny/model/model.factor b/extra/bunny/model/model.factor index 7c77ed98af..8d2175846c 100644 --- a/extra/bunny/model/model.factor +++ b/extra/bunny/model/model.factor @@ -1,5 +1,5 @@ USING: alien alien.c-types arrays sequences math math.vectors math.matrices - math.parser io io.files kernel opengl opengl.gl opengl.glu + math.parser io io.files kernel opengl opengl.gl opengl.glu io.encodings.ascii opengl.capabilities shuffle http.client vectors splitting tools.time system combinators combinators.cleave float-arrays continuations namespaces sequences.lib ; @@ -35,7 +35,7 @@ IN: bunny.model : read-model ( stream -- model ) "Reading model" print flush [ - [ parse-model ] with-file-reader + ascii [ parse-model ] with-file-reader [ normals ] 2keep 3array ] time ; diff --git a/extra/cpu/8080/emulator/emulator.factor b/extra/cpu/8080/emulator/emulator.factor index 9499ff8dff..9189a41411 100755 --- a/extra/cpu/8080/emulator/emulator.factor +++ b/extra/cpu/8080/emulator/emulator.factor @@ -4,7 +4,7 @@ USING: kernel math sequences words arrays io io.files namespaces math.parser kernel.private assocs quotations parser parser-combinators tools.time - sequences.private compiler.units ; + sequences.private compiler.units io.encodings.binary ; IN: cpu.8080.emulator TUPLE: cpu b c d e f h l a pc sp halted? last-interrupt cycles ram ; @@ -461,7 +461,7 @@ M: cpu reset ( cpu -- ) : load-rom ( filename cpu -- ) #! Load the contents of the file into ROM. #! (address 0x0000-0x1FFF). - cpu-ram swap [ + cpu-ram swap binary [ 0 swap (load-rom) ] with-file-reader ; @@ -477,7 +477,7 @@ SYMBOL: rom-root #! file path shoul dbe relative to the '/roms' resource path. rom-dir [ cpu-ram [ - swap first2 rom-dir swap path+ [ + swap first2 rom-dir swap path+ binary [ swap (load-rom) ] with-file-reader ] curry each diff --git a/extra/cryptlib/cryptlib-tests.factor b/extra/cryptlib/cryptlib-tests.factor index c404114716..aeac468ba3 100644 --- a/extra/cryptlib/cryptlib-tests.factor +++ b/extra/cryptlib/cryptlib-tests.factor @@ -1,5 +1,6 @@ USING: cryptlib.libcl cryptlib prettyprint kernel alien sequences libc math -tools.test io io.files continuations alien.c-types splitting generic.math ; +tools.test io io.files continuations alien.c-types splitting generic.math +io.encodings.binary ; "=========================================================" print "Envelope/de-envelop test..." print @@ -152,7 +153,7 @@ tools.test io io.files continuations alien.c-types splitting generic.math ; ! envelope CRYPT_FORMAT_CRYPTLIB [ "extra/cryptlib/test/large_data.txt" resource-path - file-contents set-pop-buffer + binary file-contents set-pop-buffer envelope-handle CRYPT_ATTRIBUTE_BUFFERSIZE get-pop-buffer alien>char-string length 10000 + set-attribute envelope-handle CRYPT_ENVINFO_DATASIZE @@ -192,7 +193,7 @@ tools.test io io.files continuations alien.c-types splitting generic.math ; CRYPT_FORMAT_CRYPTLIB [ envelope-handle CRYPT_ENVINFO_PASSWORD "password" set-attribute-string "extra/cryptlib/test/large_data.txt" resource-path - file-contents set-pop-buffer + binary file-contents set-pop-buffer envelope-handle CRYPT_ATTRIBUTE_BUFFERSIZE get-pop-buffer alien>char-string length 10000 + set-attribute envelope-handle CRYPT_ENVINFO_DATASIZE diff --git a/extra/crypto/hmac/hmac.factor b/extra/crypto/hmac/hmac.factor index 7c358a8c09..56d39e71dc 100644 --- a/extra/crypto/hmac/hmac.factor +++ b/extra/crypto/hmac/hmac.factor @@ -1,6 +1,6 @@ USING: arrays combinators crypto.common crypto.md5 crypto.sha1 crypto.md5.private io io.binary io.files io.streams.string -kernel math math.vectors memoize sequences ; +kernel math math.vectors memoize sequences io.encodings.binary ; IN: crypto.hmac : sha1-hmac ( Ko Ki -- hmac ) @@ -32,7 +32,7 @@ MEMO: opad ( -- seq ) 64 HEX: 5c ; [ init-hmac sha1-hmac ] with-stream ; : file>sha1-hmac ( K path -- hmac ) - stream>sha1-hmac ; + binary stream>sha1-hmac ; : string>sha1-hmac ( K string -- hmac ) stream>sha1-hmac ; @@ -42,7 +42,7 @@ MEMO: opad ( -- seq ) 64 HEX: 5c ; [ init-hmac md5-hmac ] with-stream ; : file>md5-hmac ( K path -- hmac ) - stream>md5-hmac ; + binary stream>md5-hmac ; : string>md5-hmac ( K string -- hmac ) stream>md5-hmac ; diff --git a/extra/crypto/md5/md5.factor b/extra/crypto/md5/md5.factor index fe215e32db..9ca3561574 100644 --- a/extra/crypto/md5/md5.factor +++ b/extra/crypto/md5/md5.factor @@ -2,7 +2,8 @@ USING: kernel io io.binary io.files io.streams.string math math.functions math.parser namespaces splitting strings -sequences crypto.common byte-arrays locals sequences.private ; +sequences crypto.common byte-arrays locals sequences.private +io.encodings.binary ; IN: crypto.md5 : string>md5 ( string -- byte-array ) stream>md5 ; : string>md5str ( string -- md5-string ) string>md5 hex-string ; -: file>md5 ( path -- byte-array ) stream>md5 ; +: file>md5 ( path -- byte-array ) binary stream>md5 ; : file>md5str ( path -- md5-string ) file>md5 hex-string ; diff --git a/extra/crypto/sha1/sha1.factor b/extra/crypto/sha1/sha1.factor index f6dfbcd031..8f8b3e0cdd 100644 --- a/extra/crypto/sha1/sha1.factor +++ b/extra/crypto/sha1/sha1.factor @@ -1,4 +1,4 @@ -USING: arrays combinators crypto.common kernel io io.binary +USING: arrays combinators crypto.common kernel io io.encodings.binary io.files io.streams.string math.vectors strings sequences namespaces math parser sequences vectors hashtables ; @@ -123,7 +123,7 @@ SYMBOL: K : string>sha1 ( string -- sha1 ) stream>sha1 ; : string>sha1str ( string -- str ) string>sha1 hex-string ; : string>sha1-bignum ( string -- n ) string>sha1 be> ; -: file>sha1 ( file -- sha1 ) stream>sha1 ; +: file>sha1 ( file -- sha1 ) binary stream>sha1 ; : string>sha1-interleave ( string -- seq ) [ zero? ] left-trim diff --git a/extra/editors/jedit/jedit.factor b/extra/editors/jedit/jedit.factor index fd5b6c1b06..ed579dde42 100644 --- a/extra/editors/jedit/jedit.factor +++ b/extra/editors/jedit/jedit.factor @@ -3,11 +3,11 @@ USING: arrays definitions io kernel math namespaces parser prettyprint sequences strings words editors io.files io.sockets io.streams.string io.binary -math.parser ; +math.parser io.encodings.ascii ; IN: editors.jedit : jedit-server-info ( -- port auth ) - home "/.jedit/server" path+ [ + home "/.jedit/server" path+ ascii [ readln drop readln string>number readln string>number diff --git a/extra/furnace/furnace.factor b/extra/furnace/furnace.factor index 9b7a8a8aa5..b8e8b6885f 100755 --- a/extra/furnace/furnace.factor +++ b/extra/furnace/furnace.factor @@ -5,7 +5,7 @@ furnace.validator hashtables heaps html.elements http http.server.responders http.server.templating io.files kernel math namespaces quotations sequences splitting words strings vectors webapps.callback continuations tuples classes vocabs -html io ; +html io io.encodings.binary ; IN: furnace : code>quotation ( word/quot -- quot ) @@ -194,7 +194,7 @@ SYMBOL: model ; : send-resource ( name -- ) - template-path get swap path+ resource-path + template-path get swap path+ resource-path binary stdio get stream-copy ; : render-link ( quot name -- ) diff --git a/extra/graphics/bitmap/bitmap.factor b/extra/graphics/bitmap/bitmap.factor index 8e61766de1..ec4d6b79e1 100644 --- a/extra/graphics/bitmap/bitmap.factor +++ b/extra/graphics/bitmap/bitmap.factor @@ -4,7 +4,7 @@ USING: alien arrays byte-arrays combinators graphics.viewer io io.binary io.files kernel libc math math.functions namespaces opengl opengl.gl prettyprint -sequences strings ui ui.gadgets.panes ; +sequences strings ui ui.gadgets.panes io.encodings.binary ; IN: graphics.bitmap ! Currently can only handle 24bit bitmaps. @@ -59,7 +59,7 @@ TUPLE: bitmap magic size reserved offset header-length width dup color-index-length read swap set-bitmap-color-index ; : load-bitmap ( path -- bitmap ) - [ + binary [ T{ bitmap } clone dup parse-file-header dup parse-bitmap-header @@ -69,7 +69,7 @@ TUPLE: bitmap magic size reserved offset header-length width raw-bitmap>string >byte-array over set-bitmap-array ; : save-bitmap ( bitmap path -- ) - [ + binary [ "BM" write dup bitmap-array length 14 + 40 + 4 >le write 0 4 >le write diff --git a/extra/help/cookbook/cookbook.factor b/extra/help/cookbook/cookbook.factor index 5be69663f8..0b22ea2d1e 100755 --- a/extra/help/cookbook/cookbook.factor +++ b/extra/help/cookbook/cookbook.factor @@ -191,11 +191,11 @@ ARTICLE: "cookbook-io" "Input and output cookbook" } "Print the lines of a file in sorted order:" { $code - "\"lines.txt\" file-lines natural-sort [ print ] each" + "utf8 \"lines.txt\" file-lines natural-sort [ print ] each" } "Read 1024 bytes from a file:" { $code - "\"data.bin\" [ 1024 read ] with-file-reader" + "\"data.bin\" binary [ 1024 read ] with-file-reader" } "Convert a file of 4-byte cells from little to big endian or vice versa, by directly mapping it into memory:" { $code diff --git a/extra/http/client/client.factor b/extra/http/client/client.factor index 679d603708..251015a30d 100755 --- a/extra/http/client/client.factor +++ b/extra/http/client/client.factor @@ -2,7 +2,7 @@ ! See http://factorcode.org/license.txt for BSD license. USING: assocs http kernel math math.parser namespaces sequences io io.sockets io.streams.string io.files io.timeouts strings -splitting continuations assocs.lib ; +splitting continuations assocs.lib io.encodings.binary ; IN: http.client : parse-host ( url -- host port ) @@ -71,7 +71,7 @@ DEFER: http-get-stream : download-to ( url file -- ) #! Downloads the contents of a URL to a file. >r http-get-stream check-response - r> stream-copy ; + r> binary stream-copy ; : download ( url -- ) dup download-name download-to ; diff --git a/extra/http/server/templating/templating-tests.factor b/extra/http/server/templating/templating-tests.factor index d889cd848a..2c15120552 100644 --- a/extra/http/server/templating/templating-tests.factor +++ b/extra/http/server/templating/templating-tests.factor @@ -1,5 +1,5 @@ USING: io io.files io.streams.string http.server.templating kernel tools.test - sequences ; + sequences io.encodings.utf8 ; IN: temporary : test-template ( path -- ? ) @@ -8,7 +8,7 @@ IN: temporary ".fhtml" append resource-path [ run-template-file ] with-string-writer ] keep - ".html" append resource-path file-contents = ; + ".html" append resource-path utf8 file-contents = ; [ t ] [ "example" test-template ] unit-test [ t ] [ "bug" test-template ] unit-test diff --git a/extra/http/server/templating/templating.factor b/extra/http/server/templating/templating.factor index f364b86524..d15bec4fb2 100755 --- a/extra/http/server/templating/templating.factor +++ b/extra/http/server/templating/templating.factor @@ -4,7 +4,7 @@ USING: continuations sequences kernel parser namespaces io io.files io.streams.lines io.streams.string html html.elements source-files debugger combinators math quotations generic -strings splitting ; +strings splitting io.encodings.utf8 ; IN: http.server.templating @@ -82,7 +82,7 @@ DEFER: <% delimiter templating-vocab use+ dup source-file file set ! so that reload works properly [ - ?resource-path file-contents + ?resource-path utf8 file-contents [ eval-template ] [ html-error. drop ] recover ] keep ] with-file-vocabs @@ -93,4 +93,4 @@ DEFER: <% delimiter swap path+ run-template-file ; : template-convert ( infile outfile -- ) - [ run-template-file ] with-file-writer ; + utf8 [ run-template-file ] with-file-writer ; diff --git a/extra/icfp/2006/2006.factor b/extra/icfp/2006/2006.factor index ae0e058490..1740e8a523 100755 --- a/extra/icfp/2006/2006.factor +++ b/extra/icfp/2006/2006.factor @@ -2,7 +2,7 @@ ! See http://factorcode.org/license.txt for BSD license. USING: kernel math sequences kernel.private namespaces arrays io io.files splitting io.binary math.functions vectors quotations -combinators ; +combinators io.encodings.binary ; IN: icfp.2006 SYMBOL: regs @@ -134,7 +134,7 @@ SYMBOL: open-arrays [ run-op exec-loop ] unless ; : load-platters ( path -- ) - file-contents 4 group [ be> ] map + binary file-contents 4 group [ be> ] map 0 arrays get set-nth ; : init ( path -- ) diff --git a/extra/id3/id3.factor b/extra/id3/id3.factor index 4f633f5be1..02e9515451 100644 --- a/extra/id3/id3.factor +++ b/extra/id3/id3.factor @@ -3,7 +3,7 @@ USING: arrays combinators io io.binary io.files io.paths io.encodings.utf16 kernel math math.parser namespaces sequences -splitting strings assocs unicode.categories ; +splitting strings assocs unicode.categories io.encodings.binary ; IN: id3 @@ -107,20 +107,20 @@ C: extended-header read-header read-frames ; : supported-version? ( version -- ? ) - [ 3 4 ] member? ; + { 3 4 } member? ; : read-id3v2 ( -- tag/f ) read1 dup supported-version? [ (read-id3v2) ] [ drop f ] if ; : id3v2? ( -- ? ) - 3 read "ID3" = ; + 3 read "ID3" sequence= ; : read-tag ( stream -- tag/f ) id3v2? [ read-id3v2 ] [ f ] if ; : id3v2 ( filename -- tag/f ) - [ read-tag ] with-file-reader ; + binary [ read-tag ] with-file-reader ; : file? ( path -- ? ) stat 3drop not ; @@ -135,7 +135,7 @@ C: extended-header [ mp3? ] subset ; : id3? ( file -- ? ) - [ id3v2? ] with-file-reader ; + binary [ id3v2? ] with-file-reader ; : id3s ( files -- id3s ) [ id3? ] subset ; diff --git a/extra/io/mmap/mmap-tests.factor b/extra/io/mmap/mmap-tests.factor index 25caae036d..388deac0db 100644 --- a/extra/io/mmap/mmap-tests.factor +++ b/extra/io/mmap/mmap-tests.factor @@ -1,9 +1,10 @@ -USING: io io.mmap io.files kernel tools.test continuations sequences ; +USING: io io.mmap io.files kernel tools.test continuations +sequences io.encodings.ascii ; IN: temporary [ "mmap-test-file.txt" resource-path delete-file ] ignore-errors -[ ] [ "mmap-test-file.txt" resource-path [ "12345" write ] with-file-writer ] unit-test +[ ] [ "mmap-test-file.txt" resource-path ascii [ "12345" write ] with-file-writer ] unit-test [ ] [ "mmap-test-file.txt" resource-path dup file-length [ CHAR: 2 0 pick set-nth drop ] with-mapped-file ] unit-test [ 5 ] [ "mmap-test-file.txt" resource-path dup file-length [ length ] with-mapped-file ] unit-test -[ "22345" ] [ "mmap-test-file.txt" resource-path file-contents ] unit-test +[ "22345" ] [ "mmap-test-file.txt" resource-path ascii file-contents ] unit-test [ "mmap-test-file.txt" resource-path delete-file ] ignore-errors diff --git a/extra/io/unix/unix-tests.factor b/extra/io/unix/unix-tests.factor index e1c3108952..85d450dac9 100755 --- a/extra/io/unix/unix-tests.factor +++ b/extra/io/unix/unix-tests.factor @@ -1,6 +1,6 @@ USING: io.files io.sockets io kernel threads namespaces tools.test continuations strings byte-arrays sequences -prettyprint system ; +prettyprint system io.encodings.binary ; IN: temporary ! Unix domain stream sockets @@ -131,15 +131,15 @@ client-addr ! Invalid parameter tests [ - image [ stdio get accept ] with-file-reader + image binary [ stdio get accept ] with-file-reader ] must-fail [ - image [ stdio get receive ] with-file-reader + image binary [ stdio get receive ] with-file-reader ] must-fail [ - image [ + image binary [ B{ 1 2 } server-addr stdio get send ] with-file-reader diff --git a/extra/log-viewer/log-viewer.factor b/extra/log-viewer/log-viewer.factor index 0f139d184e..7bc63d3e34 100755 --- a/extra/log-viewer/log-viewer.factor +++ b/extra/log-viewer/log-viewer.factor @@ -1,4 +1,4 @@ -USING: kernel io io.files io.monitors ; +USING: kernel io io.files io.monitors io.encodings.utf8 ; IN: log-viewer : read-lines ( stream -- ) @@ -9,6 +9,6 @@ IN: log-viewer dup next-change 2drop over read-lines tail-file-loop ; : tail-file ( file -- ) - dup dup read-lines + dup utf8 dup read-lines swap parent-directory f tail-file-loop ; diff --git a/extra/logging/insomniac/insomniac.factor b/extra/logging/insomniac/insomniac.factor index bb143879bf..0294085eda 100755 --- a/extra/logging/insomniac/insomniac.factor +++ b/extra/logging/insomniac/insomniac.factor @@ -1,7 +1,8 @@ ! Copyright (C) 2008 Slava Pestov. ! See http://factorcode.org/license.txt for BSD license. USING: logging.analysis logging.server logging smtp io.sockets -kernel io.files io.streams.string namespaces raptor.cron assocs ; +kernel io.files io.streams.string namespaces raptor.cron assocs +io.encodings.utf8 ; IN: logging.insomniac SYMBOL: insomniac-smtp-host @@ -11,7 +12,7 @@ SYMBOL: insomniac-recipients : ?analyze-log ( service word-names -- string/f ) >r log-path 1 log# dup exists? [ - file-lines r> [ analyze-log ] with-string-writer + utf8 file-lines r> [ analyze-log ] with-string-writer ] [ r> 2drop f ] if ; diff --git a/extra/logging/server/server.factor b/extra/logging/server/server.factor index 05029df1d0..e2a57fbc54 100755 --- a/extra/logging/server/server.factor +++ b/extra/logging/server/server.factor @@ -3,7 +3,7 @@ USING: namespaces kernel io calendar sequences io.files io.sockets continuations prettyprint assocs math.parser words debugger math combinators concurrency arrays init -math.ranges strings ; +math.ranges strings io.encodings.utf8 ; IN: logging.server : log-root ( -- string ) @@ -20,7 +20,7 @@ SYMBOL: log-files : open-log-stream ( service -- stream ) log-path dup make-directories - 1 log# ; + 1 log# utf8 ; : log-stream ( service -- stream ) log-files get [ open-log-stream ] cache ; diff --git a/extra/mad/api/api.factor b/extra/mad/api/api.factor index d803fa64e0..fdc2903d46 100644 --- a/extra/mad/api/api.factor +++ b/extra/mad/api/api.factor @@ -1,7 +1,8 @@ ! Copyright (C) 2007 Adam Wendt. ! See http://factorcode.org/license.txt for BSD license. USING: alien alien.c-types byte-arrays io io.binary io.files kernel mad - namespaces prettyprint sbufs sequences tools.interpreter vars ; + namespaces prettyprint sbufs sequences tools.interpreter vars + io.encodings.binary ; IN: mad.api VARS: buffer-start buffer-length output-callback-var ; @@ -80,9 +81,6 @@ VARS: buffer-start buffer-length output-callback-var ; : make-decoder ( -- decoder ) "mad_decoder" malloc-object ; -: malloc-file-contents ( path -- alien ) - file-contents >byte-array malloc-byte-array ; - : mad-run ( -- int ) make-decoder [ mad-init ] keep MAD_DECODER_MODE_SYNC mad_decoder_run ; diff --git a/extra/msxml-to-csv/msxml-to-csv.factor b/extra/msxml-to-csv/msxml-to-csv.factor index 8a9ba9cf98..3004324511 100644 --- a/extra/msxml-to-csv/msxml-to-csv.factor +++ b/extra/msxml-to-csv/msxml-to-csv.factor @@ -1,4 +1,4 @@ -USING: io io.files sequences xml xml.utilities ; +USING: io io.files sequences xml xml.utilities io.encodings.utf8 ; IN: msxml-to-csv : print-csv ( table -- ) [ "," join print ] each ; @@ -13,6 +13,6 @@ IN: msxml-to-csv ] map ; : msxml>csv ( infile outfile -- ) - [ + utf8 [ file>xml (msxml>csv) print-csv ] with-file-writer ; diff --git a/extra/ogg/player/player.factor b/extra/ogg/player/player.factor index 518030ee4d..94a45c90e3 100644 --- a/extra/ogg/player/player.factor +++ b/extra/ogg/player/player.factor @@ -14,7 +14,8 @@ USING: kernel alien ogg ogg.vorbis ogg.theora io byte-arrays sequences libc shuffle alien.c-types system openal math namespaces threads shuffle opengl arrays ui.gadgets.worlds combinators math.parser ui.gadgets ui.render opengl.gl ui - continuations io.files hints combinators.lib sequences.lib ; + continuations io.files hints combinators.lib sequences.lib + io.encodings.binary ; IN: ogg.player @@ -612,7 +613,7 @@ M: theora-gadget draw-gadget* ( gadget -- ) play-ogg ; : play-vorbis-file ( filename -- ) - play-vorbis-stream ; + binary play-vorbis-stream ; : play-theora-stream ( stream -- ) @@ -620,5 +621,5 @@ M: theora-gadget draw-gadget* ( gadget -- ) play-ogg ; : play-theora-file ( filename -- ) - play-theora-stream ; + binary play-theora-stream ; diff --git a/extra/porter-stemmer/porter-stemmer-tests.factor b/extra/porter-stemmer/porter-stemmer-tests.factor index d3e031fdc6..1e46fcd871 100644 --- a/extra/porter-stemmer/porter-stemmer-tests.factor +++ b/extra/porter-stemmer/porter-stemmer-tests.factor @@ -1,6 +1,6 @@ IN: temporary USING: arrays io kernel porter-stemmer sequences tools.test -io.files ; +io.files io.encodings.utf8 ; [ 0 ] [ "xa" consonant-seq ] unit-test [ 0 ] [ "xxaa" consonant-seq ] unit-test @@ -56,7 +56,7 @@ io.files ; [ "hell" ] [ "hell" step5 "" like ] unit-test [ "mate" ] [ "mate" step5 "" like ] unit-test -: resource-lines resource-path file-lines ; +: resource-lines resource-path utf8 file-lines ; [ { } ] [ "extra/porter-stemmer/test/voc.txt" resource-lines diff --git a/extra/project-euler/042/042.factor b/extra/project-euler/042/042.factor index c66be27df7..a87722debc 100644 --- a/extra/project-euler/042/042.factor +++ b/extra/project-euler/042/042.factor @@ -1,7 +1,7 @@ ! Copyright (c) 2008 Aaron Schaefer. ! See http://factorcode.org/license.txt for BSD license. USING: ascii io.files kernel math math.functions namespaces - project-euler.common sequences sequences.lib splitting ; + project-euler.common sequences sequences.lib splitting io.encodings.ascii ; IN: project-euler.042 ! http://projecteuler.net/index.php?section=problems&id=42 @@ -31,7 +31,7 @@ IN: project-euler.042 : source-042 ( -- seq ) "extra/project-euler/042/words.txt" resource-path - file-contents [ quotable? ] subset "," split ; + ascii file-contents [ quotable? ] subset "," split ; : (triangle-upto) ( limit n -- ) 2dup nth-triangle > [ diff --git a/extra/project-euler/067/067.factor b/extra/project-euler/067/067.factor index f206f59472..436ccde776 100644 --- a/extra/project-euler/067/067.factor +++ b/extra/project-euler/067/067.factor @@ -1,6 +1,7 @@ ! Copyright (c) 2007 Samuel Tardieu, Aaron Schaefer. ! See http://factorcode.org/license.txt for BSD license. -USING: io.files math.parser namespaces project-euler.common sequences splitting ; +USING: io.files math.parser namespaces project-euler.common +io.encodings.ascii sequences splitting ; IN: project-euler.067 ! http://projecteuler.net/index.php?section=problems&id=67 @@ -38,7 +39,7 @@ IN: project-euler.067 : source-067 ( -- seq ) "extra/project-euler/067/triangle.txt" resource-path - file-lines [ " " split [ string>number ] map ] map ; + ascii file-lines [ " " split [ string>number ] map ] map ; PRIVATE> diff --git a/extra/project-euler/079/079.factor b/extra/project-euler/079/079.factor index f068db77ec..30c46de0a0 100644 --- a/extra/project-euler/079/079.factor +++ b/extra/project-euler/079/079.factor @@ -1,6 +1,7 @@ ! Copyright (c) 2008 Aaron Schaefer. ! See http://factorcode.org/license.txt for BSD license. -USING: assocs hashtables io.files kernel math math.parser namespaces sequences ; +USING: assocs hashtables io.files kernel math math.parser namespaces +io.encodings.ascii sequences ; IN: project-euler.079 ! http://projecteuler.net/index.php?section=problems&id=79 @@ -26,7 +27,7 @@ IN: project-euler.079 edges ( seq -- seq ) [ diff --git a/extra/raptor/raptor.factor b/extra/raptor/raptor.factor index d776739d89..36da6d9434 100644 --- a/extra/raptor/raptor.factor +++ b/extra/raptor/raptor.factor @@ -42,10 +42,10 @@ SYMBOL: networking-hook ! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! USING: io io.files io.streams.lines io.streams.plain io.streams.duplex - listener ; + listener io.encodings.utf8 ; : tty-listener ( tty -- ) - [ ] [ ] bi + [ utf8 ] [ utf8 ] bi [ listener ] with-stream ; ! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! diff --git a/extra/rss/rss-tests.factor b/extra/rss/rss-tests.factor index 68a40704b3..1d493d3c14 100644 --- a/extra/rss/rss-tests.factor +++ b/extra/rss/rss-tests.factor @@ -1,9 +1,9 @@ -USING: rss io kernel io.files tools.test ; +USING: rss io kernel io.files tools.test io.encodings.utf8 ; : load-news-file ( filename -- feed ) #! Load an news syndication file and process it, returning #! it as an feed tuple. - read-feed ; + utf8 read-feed ; [ T{ feed diff --git a/extra/tar/tar.factor b/extra/tar/tar.factor index d92b4bd48b..ed083386ed 100755 --- a/extra/tar/tar.factor +++ b/extra/tar/tar.factor @@ -1,7 +1,7 @@ USING: combinators io io.files io.streams.duplex io.streams.string kernel math math.parser continuations namespaces pack prettyprint sequences strings system -hexdump tools.interpreter ; +hexdump tools.interpreter io.encodings.binary ; IN: tar : zero-checksum 256 ; @@ -94,7 +94,7 @@ TUPLE: unimplemented-typeflag header ; ! Normal file : typeflag-0 - tar-header-name tar-path+ + tar-header-name tar-path+ binary [ read-data-blocks ] keep dispose ; ! Hard link @@ -236,7 +236,7 @@ TUPLE: unimplemented-typeflag header ; ] when* ; : parse-tar ( path -- obj ) - [ + binary [ "tar-test" resource-path base-dir set global [ nl nl nl "Starting to parse .tar..." print flush ] bind global [ "Expanding to: " write base-dir get . flush ] bind diff --git a/extra/tools/browser/browser.factor b/extra/tools/browser/browser.factor index 7c28983519..63f9a087f5 100755 --- a/extra/tools/browser/browser.factor +++ b/extra/tools/browser/browser.factor @@ -5,12 +5,12 @@ words vocabs vocabs.loader definitions parser continuations inspector debugger io io.styles io.streams.lines hashtables sorting prettyprint source-files arrays combinators strings system math.parser help.markup help.topics help.syntax -help.stylesheet memoize ; +help.stylesheet memoize io.encodings.utf8 ; IN: tools.browser MEMO: (vocab-file-contents) ( path -- lines ) ?resource-path dup exists? - [ file-lines ] [ drop f ] if ; + [ utf8 file-lines ] [ drop f ] if ; : vocab-file-contents ( vocab name -- seq ) vocab-path+ dup [ (vocab-file-contents) ] when ; @@ -18,7 +18,7 @@ MEMO: (vocab-file-contents) ( path -- lines ) : set-vocab-file-contents ( seq vocab name -- ) dupd vocab-path+ [ ?resource-path - [ [ print ] each ] with-file-writer + utf8 [ [ print ] each ] with-file-writer ] [ "The " swap vocab-name " vocabulary was not loaded from the file system" diff --git a/extra/tools/deploy/macosx/macosx.factor b/extra/tools/deploy/macosx/macosx.factor index eb1a4af4a7..e4a0d539d1 100755 --- a/extra/tools/deploy/macosx/macosx.factor +++ b/extra/tools/deploy/macosx/macosx.factor @@ -46,8 +46,8 @@ IN: tools.deploy.macosx ] H{ } make-assoc print-plist ; : create-app-plist ( vocab bundle-name -- ) - dup "Contents/Info.plist" path+ - [ print-app-plist ] with-stream ; + dup "Contents/Info.plist" path+ + utf8 [ print-app-plist ] with-file-writer ; : create-app-dir ( vocab bundle-name -- vm ) dup "Frameworks" copy-bundle-dir diff --git a/extra/tools/disassembler/disassembler.factor b/extra/tools/disassembler/disassembler.factor index 833a6e7548..57fe7b43e8 100755 --- a/extra/tools/disassembler/disassembler.factor +++ b/extra/tools/disassembler/disassembler.factor @@ -2,7 +2,7 @@ ! See http://factorcode.org/license.txt for BSD license. USING: io.files io words alien kernel math.parser alien.syntax io.launcher system assocs arrays sequences namespaces qualified -system math windows.kernel32 generator.fixup ; +system math windows.kernel32 generator.fixup io.encodings.ascii ; IN: tools.disassembler : in-file "gdb-in.txt" resource-path ; @@ -15,7 +15,7 @@ M: word make-disassemble-cmd word-xt code-format - 2array make-disassemble-cmd ; M: pair make-disassemble-cmd - in-file [ + in-file ascii [ "attach " write current-process-handle number>string print "disassemble " write @@ -28,7 +28,7 @@ M: pair make-disassemble-cmd out-file +stdout+ set [ "gdb" , "-x" , in-file , "-batch" , ] { } make +arguments+ set ] { } make-assoc run-process drop - out-file file-lines ; + out-file ascii file-lines ; : tabs>spaces ( str -- str' ) { { CHAR: \t CHAR: \s } } substitute ; diff --git a/extra/ui/freetype/freetype.factor b/extra/ui/freetype/freetype.factor index 2dade0f58e..8078ec4a33 100755 --- a/extra/ui/freetype/freetype.factor +++ b/extra/ui/freetype/freetype.factor @@ -73,7 +73,7 @@ M: freetype-renderer free-fonts ( world -- ) : open-face ( font style -- face ) ttf-name ttf-path - dup file-contents >byte-array malloc-byte-array + dup malloc-file-contents swap file-length (open-face) ; diff --git a/extra/unicode/breaks/breaks.factor b/extra/unicode/breaks/breaks.factor index 1014d3ad7e..dfc7bf2264 100644 --- a/extra/unicode/breaks/breaks.factor +++ b/extra/unicode/breaks/breaks.factor @@ -1,7 +1,7 @@ USING: unicode.categories kernel math combinators splitting sequences math.parser io.files io assocs arrays namespaces combinators.lib assocs.lib math.ranges unicode.normalize -unicode.syntax unicode.data compiler.units alien.syntax ; +unicode.syntax unicode.data compiler.units alien.syntax io.encodings.ascii ; IN: unicode.breaks C-ENUM: Any L V T Extend Control CR LF graphemes ; @@ -30,7 +30,7 @@ CATEGORY: grapheme-control Zl Zp Cc Cf ; concat >set ; : other-extend-lines ( -- lines ) - "extra/unicode/PropList.txt" resource-path file-lines ; + "extra/unicode/PropList.txt" resource-path ascii file-lines ; VALUE: other-extend diff --git a/extra/unicode/data/data.factor b/extra/unicode/data/data.factor index c3998a6132..11be803893 100755 --- a/extra/unicode/data/data.factor +++ b/extra/unicode/data/data.factor @@ -1,6 +1,6 @@ USING: assocs math kernel sequences io.files hashtables quotations splitting arrays math.parser combinators.lib hash2 -byte-arrays words namespaces words compiler.units parser ; +byte-arrays words namespaces words compiler.units parser io.encodings.ascii ; IN: unicode.data << @@ -21,7 +21,7 @@ IN: unicode.data ! Loading data from UnicodeData.txt : data ( filename -- data ) - file-lines [ ";" split ] map ; + ascii file-lines [ ";" split ] map ; : load-data ( -- data ) "extra/unicode/UnicodeData.txt" resource-path data ; diff --git a/extra/webapps/file/file.factor b/extra/webapps/file/file.factor index 552f5e0977..12627b985f 100755 --- a/extra/webapps/file/file.factor +++ b/extra/webapps/file/file.factor @@ -3,7 +3,7 @@ USING: calendar html io io.files kernel math math.parser http.server.responders http.server.templating namespaces parser sequences strings assocs hashtables debugger http.mime sorting -html.elements logging ; +html.elements logging io.encodings.binary ; IN: webapps.file @@ -37,7 +37,7 @@ SYMBOL: serve-file-hook [ dupd file-response - stdio get stream-copy + binary stdio get stream-copy ] serve-file-hook set-global : serve-static ( filename mime-type -- ) diff --git a/extra/webapps/source/source.factor b/extra/webapps/source/source.factor index 4c0701c687..dc97805b78 100755 --- a/extra/webapps/source/source.factor +++ b/extra/webapps/source/source.factor @@ -1,7 +1,7 @@ ! Copyright (C) 2007 Slava Pestov. ! See http://factorcode.org/license.txt for BSD license. USING: io.files namespaces webapps.file http.server.responders -xmode.code2html kernel html sequences ; +xmode.code2html kernel html sequences io.encodings.utf8 ; IN: webapps.source ! This responder is a potential security problem. Make sure you @@ -15,7 +15,7 @@ IN: webapps.source : source-responder ( path mime-type -- ) drop serving-html - [ dup htmlize-stream ] with-html-stream ; + [ dup utf8 htmlize-stream ] with-html-stream ; global [ ! Serve up our own source code diff --git a/extra/xml/xml.factor b/extra/xml/xml.factor index ec3e24b99d..970ff39cf1 100644 --- a/extra/xml/xml.factor +++ b/extra/xml/xml.factor @@ -3,7 +3,7 @@ USING: io io.streams.string io.files kernel math namespaces prettyprint sequences arrays generic strings vectors xml.char-classes xml.data xml.errors xml.tokenize xml.writer -xml.utilities state-parser assocs ascii ; +xml.utilities state-parser assocs ascii io.encodings.utf8 ; IN: xml ! -- Overall parser with data tree @@ -167,7 +167,8 @@ TUPLE: pull-xml scope ; read-xml ; : file>xml ( filename -- xml ) - read-xml ; + ! Autodetect encoding! + utf8 read-xml ; : xml-reprint ( string -- ) string>xml print-xml ; diff --git a/extra/xmode/catalog/catalog.factor b/extra/xmode/catalog/catalog.factor index d6402603fa..6bff786fff 100755 --- a/extra/xmode/catalog/catalog.factor +++ b/extra/xmode/catalog/catalog.factor @@ -1,6 +1,6 @@ USING: xmode.loader xmode.utilities xmode.rules namespaces strings splitting assocs sequences kernel io.files xml memoize -words globs combinators ; +words globs combinators io.encodings.utf8 ; IN: xmode.catalog TUPLE: mode file file-name-glob first-line-glob ; @@ -25,7 +25,7 @@ TAGS> : load-catalog ( -- modes ) "extra/xmode/modes/catalog" resource-path - read-xml parse-modes-tag ; + file>xml parse-modes-tag ; : modes ( -- assoc ) \ modes get-global [ @@ -38,7 +38,7 @@ TAGS> MEMO: (load-mode) ( name -- rule-sets ) modes at mode-file "extra/xmode/modes/" swap append - resource-path parse-mode ; + resource-path utf8 parse-mode ; SYMBOL: rule-sets diff --git a/extra/xmode/code2html/code2html.factor b/extra/xmode/code2html/code2html.factor index 3db70cf2e9..47e619cc00 100755 --- a/extra/xmode/code2html/code2html.factor +++ b/extra/xmode/code2html/code2html.factor @@ -1,5 +1,5 @@ USING: xmode.tokens xmode.marker xmode.catalog kernel html html.elements io - io.files sequences words ; + io.files sequences words io.encodings.utf8 ; IN: xmode.code2html : htmlize-tokens ( tokens -- ) @@ -20,7 +20,7 @@ IN: xmode.code2html : default-stylesheet ( -- ) ; : htmlize-stream ( path stream -- ) @@ -40,5 +40,5 @@ IN: xmode.code2html ; : htmlize-file ( path -- ) - dup over ".html" append + dup utf8 over ".html" append utf8 [ htmlize-stream ] with-stream ; diff --git a/extra/yahoo/yahoo-tests.factor b/extra/yahoo/yahoo-tests.factor index 22ea687a29..197fa4900b 100644 --- a/extra/yahoo/yahoo-tests.factor +++ b/extra/yahoo/yahoo-tests.factor @@ -6,6 +6,6 @@ USING: tools.test yahoo kernel io.files xml sequences ; "Official Foo Fighters" "http://www.foofighters.com/" "Official site with news, tour dates, discography, store, community, and more." -} ] [ "extra/yahoo/test-results.xml" resource-path read-xml parse-yahoo first ] unit-test +} ] [ "extra/yahoo/test-results.xml" resource-path file>xml parse-yahoo first ] unit-test [ "http://search.yahooapis.com/WebSearchService/V1/webSearch?appid=Factor-search&query=hi&results=1" ] [ "hi" 1 query ] unit-test