io.encodings: use new latin1 instead of 8-bit encoding from file.
parent
093e6411be
commit
0ab7ed6979
|
@ -3,7 +3,7 @@
|
||||||
USING: accessors calendar calendar.format classes combinators
|
USING: accessors calendar calendar.format classes combinators
|
||||||
combinators.short-circuit concurrency.promises continuations
|
combinators.short-circuit concurrency.promises continuations
|
||||||
destructors ftp io io.directories io.encodings
|
destructors ftp io io.directories io.encodings
|
||||||
io.encodings.8-bit io.encodings.binary io.encodings.utf8
|
io.encodings.binary io.encodings.latin1 io.encodings.utf8
|
||||||
io.files io.files.info io.pathnames io.servers io.sockets
|
io.files io.files.info io.pathnames io.servers io.sockets
|
||||||
io.streams.string io.timeouts kernel logging math math.bitwise
|
io.streams.string io.timeouts kernel logging math math.bitwise
|
||||||
math.parser namespaces sequences simple-tokenizer splitting
|
math.parser namespaces sequences simple-tokenizer splitting
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
USING: assocs byte-arrays destructors help.markup help.syntax http
|
USING: assocs byte-arrays destructors help.markup help.syntax http
|
||||||
http.client.post-data.private http.client.private
|
http.client.post-data.private http.client.private
|
||||||
io.encodings.8-bit io.encodings.binary io.pathnames kernel
|
io.encodings.binary io.encodings.latin1 io.pathnames kernel
|
||||||
sequences strings urls urls.encoding ;
|
sequences strings urls urls.encoding ;
|
||||||
IN: http.client
|
IN: http.client
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
USING: destructors http http.server http.server.requests http.client
|
USING: destructors http http.server http.server.requests http.client
|
||||||
http.client.private tools.test multiline fry io.streams.string io.crlf
|
http.client.private tools.test multiline fry io.streams.string io.crlf
|
||||||
io.encodings.utf8 io.encodings.8-bit io.encodings.binary io.encodings.string
|
io.encodings.utf8 io.encodings.latin1 io.encodings.binary io.encodings.string
|
||||||
io.encodings.ascii kernel arrays splitting sequences assocs io.sockets db
|
io.encodings.ascii kernel arrays splitting sequences assocs io.sockets db
|
||||||
db.sqlite make continuations urls hashtables accessors namespaces xml.data
|
db.sqlite make continuations urls hashtables accessors namespaces xml.data
|
||||||
random combinators.short-circuit literals ;
|
random combinators.short-circuit literals ;
|
||||||
|
|
|
@ -11,3 +11,5 @@ io.encodings.8-bit.private strings tools.test ;
|
||||||
|
|
||||||
{ t } [ \ latin1 8-bit-encoding? ] unit-test
|
{ t } [ \ latin1 8-bit-encoding? ] unit-test
|
||||||
{ "bar" } [ "bar" \ latin1 decode ] unit-test
|
{ "bar" } [ "bar" \ latin1 decode ] unit-test
|
||||||
|
|
||||||
|
{ { 0x221a 0x00b1 0x0040 } } [ B{ 0xfb 0xf1 0x40 } cp437 decode >array ] unit-test
|
||||||
|
|
|
@ -1,8 +1,8 @@
|
||||||
! Copyright (C) 2008 Daniel Ehrenberg, Doug Coleman.
|
! Copyright (C) 2008 Daniel Ehrenberg, Doug Coleman.
|
||||||
! See http://factorcode.org/license.txt for BSD license.
|
! See http://factorcode.org/license.txt for BSD license.
|
||||||
USING: accessors assocs biassocs classes.mixin classes.singleton
|
USING: accessors assocs biassocs classes.singleton generic io
|
||||||
fry io io.encodings io.encodings.iana kernel lexer namespaces
|
io.encodings io.encodings.iana kernel lexer parser sequences
|
||||||
parser sequences simple-flat-file ;
|
simple-flat-file words ;
|
||||||
IN: io.encodings.8-bit
|
IN: io.encodings.8-bit
|
||||||
|
|
||||||
<<
|
<<
|
||||||
|
@ -11,41 +11,28 @@ IN: io.encodings.8-bit
|
||||||
: encoding-file ( file-name -- stream )
|
: encoding-file ( file-name -- stream )
|
||||||
"vocab:io/encodings/8-bit/" ".TXT" surround ;
|
"vocab:io/encodings/8-bit/" ".TXT" surround ;
|
||||||
|
|
||||||
SYMBOL: 8-bit-encodings
|
TUPLE: 8-bit { table biassoc read-only } ;
|
||||||
8-bit-encodings [ H{ } clone ] initialize
|
|
||||||
|
|
||||||
TUPLE: 8-bit { biassoc biassoc read-only } ;
|
|
||||||
|
|
||||||
: 8-bit-encode ( char 8-bit -- byte )
|
: 8-bit-encode ( char 8-bit -- byte )
|
||||||
biassoc>> value-at [ encode-error ] unless* ; inline
|
table>> value-at [ encode-error ] unless* ; inline
|
||||||
|
|
||||||
M: 8-bit encode-char
|
M: 8-bit encode-char
|
||||||
swap [ 8-bit-encode ] dip stream-write1 ;
|
swap [ 8-bit-encode ] dip stream-write1 ;
|
||||||
|
|
||||||
M: 8-bit decode-char
|
M: 8-bit decode-char
|
||||||
swap stream-read1
|
swap stream-read1 [
|
||||||
[ swap biassoc>> at [ replacement-char ] unless* ]
|
swap table>> at [ replacement-char ] unless*
|
||||||
[ drop f ] if* ;
|
] [ drop f ] if* ;
|
||||||
|
|
||||||
MIXIN: 8-bit-encoding
|
|
||||||
|
|
||||||
M: 8-bit-encoding <encoder>
|
|
||||||
8-bit-encodings get-global at <encoder> ;
|
|
||||||
|
|
||||||
M: 8-bit-encoding <decoder>
|
|
||||||
8-bit-encodings get-global at <decoder> ;
|
|
||||||
|
|
||||||
: create-encoding ( name -- word )
|
: create-encoding ( name -- word )
|
||||||
create-word-in
|
create-word-in dup define-singleton-class ;
|
||||||
[ define-singleton-class ]
|
|
||||||
[ 8-bit-encoding add-mixin-instance ]
|
|
||||||
[ ] tri ;
|
|
||||||
|
|
||||||
: load-encoding ( name iana-name file-name -- )
|
: load-encoding ( name iana-name file-name -- )
|
||||||
[ create-encoding dup ]
|
[ create-encoding dup ]
|
||||||
[ register-encoding ]
|
[ register-encoding ]
|
||||||
[ encoding-file load-codetable-file 8-bit boa ] tri*
|
[ encoding-file load-codetable-file 8-bit boa ] tri*
|
||||||
swap 8-bit-encodings get-global set-at ;
|
[ [ \ <encoder> create-method ] dip [ nip <encoder> ] curry define ]
|
||||||
|
[ [ \ <decoder> create-method ] dip [ nip <decoder> ] curry define ] 2bi ;
|
||||||
|
|
||||||
PRIVATE>
|
PRIVATE>
|
||||||
|
|
||||||
|
@ -75,7 +62,7 @@ SYNTAX: 8-BIT: scan-token scan-token scan-token load-encoding ;
|
||||||
8-BIT: latin/greek ISO_8859-7:1987 8859-7
|
8-BIT: latin/greek ISO_8859-7:1987 8859-7
|
||||||
8-BIT: latin/hebrew ISO_8859-8:1988 8859-8
|
8-BIT: latin/hebrew ISO_8859-8:1988 8859-8
|
||||||
8-BIT: latin/thai TIS-620 8859-11
|
8-BIT: latin/thai TIS-620 8859-11
|
||||||
8-BIT: latin1 ISO_8859-1:1987 8859-1
|
! 8-BIT: latin1 ISO_8859-1:1987 8859-1
|
||||||
8-BIT: latin2 ISO_8859-2:1987 8859-2
|
8-BIT: latin2 ISO_8859-2:1987 8859-2
|
||||||
8-BIT: latin3 ISO_8859-3:1988 8859-3
|
8-BIT: latin3 ISO_8859-3:1988 8859-3
|
||||||
8-BIT: latin4 ISO_8859-4:1988 8859-4
|
8-BIT: latin4 ISO_8859-4:1988 8859-4
|
||||||
|
|
|
@ -3,7 +3,7 @@
|
||||||
USING: accessors alien alien.c-types alien.data alien.enums
|
USING: accessors alien alien.c-types alien.data alien.enums
|
||||||
alien.strings assocs byte-arrays classes.struct combinators
|
alien.strings assocs byte-arrays classes.struct combinators
|
||||||
combinators.short-circuit destructors fry io io.backend
|
combinators.short-circuit destructors fry io io.backend
|
||||||
io.binary io.buffers io.encodings.8-bit io.encodings.utf8
|
io.binary io.buffers io.encodings.latin1 io.encodings.utf8
|
||||||
io.files io.pathnames io.ports io.sockets io.sockets.secure
|
io.files io.pathnames io.ports io.sockets io.sockets.secure
|
||||||
io.timeouts kernel libc locals math math.functions math.order
|
io.timeouts kernel libc locals math math.functions math.order
|
||||||
math.parser memoize namespaces openssl openssl.libcrypto
|
math.parser memoize namespaces openssl openssl.libcrypto
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
USING: destructors io io.encodings io.encodings.8-bit
|
USING: destructors io io.encodings io.encodings.latin1
|
||||||
io.encodings.ascii io.encodings.binary io.encodings.string
|
io.encodings.ascii io.encodings.binary io.encodings.string
|
||||||
io.encodings.utf8 io.files io.pipes io.streams.byte-array
|
io.encodings.utf8 io.files io.pipes io.streams.byte-array
|
||||||
io.streams.duplex io.streams.limited io.streams.string kernel
|
io.streams.duplex io.streams.limited io.streams.string kernel
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
! Copyright (C) 2009 Daniel Ehrenberg
|
! Copyright (C) 2009 Daniel Ehrenberg
|
||||||
! See http://factorcode.org/license.txt for BSD license.
|
! See http://factorcode.org/license.txt for BSD license.
|
||||||
USING: tools.test quoted-printable io.encodings.string
|
USING: tools.test quoted-printable io.encodings.string
|
||||||
sequences splitting kernel io.encodings.8-bit ;
|
sequences splitting kernel io.encodings.latin1 ;
|
||||||
IN: quoted-printable.tests
|
IN: quoted-printable.tests
|
||||||
|
|
||||||
{ "José was the
|
{ "José was the
|
||||||
|
|
|
@ -2,6 +2,6 @@ USING: io.encodings.string kernel io.encodings.8-bit ;
|
||||||
IN: tools.deploy.test.4
|
IN: tools.deploy.test.4
|
||||||
|
|
||||||
: deploy-test-4 ( -- )
|
: deploy-test-4 ( -- )
|
||||||
"xyzthg" \ latin7 encode drop ;
|
"xyzthg" latin7 encode drop ;
|
||||||
|
|
||||||
MAIN: deploy-test-4
|
MAIN: deploy-test-4
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
USING: alien.strings alien.c-types alien.data tools.test
|
USING: alien.strings alien.c-types alien.data tools.test
|
||||||
kernel libc io.encodings.utf8 io.encodings.utf16 io.encodings.utf16n
|
kernel libc io.encodings.utf8 io.encodings.utf16 io.encodings.utf16n
|
||||||
io.encodings.ascii alien io.encodings.string io.encodings.8-bit ;
|
io.encodings.ascii alien io.encodings.string io.encodings.latin1 ;
|
||||||
|
|
||||||
{ "\u0000ff" }
|
{ "\u0000ff" }
|
||||||
[ "\u0000ff" latin1 string>alien latin1 alien>string ]
|
[ "\u0000ff" latin1 string>alien latin1 alien>string ]
|
||||||
|
|
|
@ -1,10 +1,10 @@
|
||||||
! Copyright (C) 2008, 2011 Slava Pestov.
|
! Copyright (C) 2008, 2011 Slava Pestov.
|
||||||
! See http://factorcode.org/license.txt for BSD license.
|
! See http://factorcode.org/license.txt for BSD license.
|
||||||
USING: accessors alien arrays byte-arrays byte-vectors init io
|
USING: accessors alien arrays byte-arrays byte-vectors init io
|
||||||
io.encodings io.encodings.ascii io.encodings.utf16n
|
io.encodings io.encodings.ascii io.encodings.latin1
|
||||||
io.encodings.utf8 io.streams.memory kernel kernel.private math
|
io.encodings.utf16n io.encodings.utf8 io.streams.memory kernel
|
||||||
namespaces sequences sequences.private strings strings.private
|
kernel.private math namespaces sequences sequences.private
|
||||||
system system.private ;
|
strings strings.private system system.private ;
|
||||||
IN: alien.strings
|
IN: alien.strings
|
||||||
|
|
||||||
GENERIC#: alien>string 1 ( c-ptr encoding -- string/f )
|
GENERIC#: alien>string 1 ( c-ptr encoding -- string/f )
|
||||||
|
@ -31,7 +31,7 @@ M: c-ptr string>alien drop ;
|
||||||
<PRIVATE
|
<PRIVATE
|
||||||
|
|
||||||
: fast-string? ( string encoding -- ? )
|
: fast-string? ( string encoding -- ? )
|
||||||
swap aux>> not [ { ascii utf8 } member-eq? ] [ drop f ] if ; inline
|
swap aux>> not [ { ascii latin1 utf8 } member-eq? ] [ drop f ] if ; inline
|
||||||
|
|
||||||
: string>alien-fast ( string encoding -- byte-array )
|
: string>alien-fast ( string encoding -- byte-array )
|
||||||
{ string object } declare ! aux>> must be f
|
{ string object } declare ! aux>> must be f
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
USING: alien alien.c-types alien.data arrays classes.struct
|
USING: alien alien.c-types alien.data arrays classes.struct
|
||||||
compiler.units continuations destructors fry generic.single io
|
compiler.units continuations destructors fry generic.single io
|
||||||
io.backend io.directories io.encodings io.encodings.8-bit
|
io.backend io.directories io.encodings io.encodings.ascii
|
||||||
io.encodings.ascii io.encodings.binary io.encodings.string
|
io.encodings.binary io.encodings.latin1 io.encodings.string
|
||||||
io.encodings.utf16 io.encodings.utf8 io.files io.files.private
|
io.encodings.utf16 io.encodings.utf8 io.files io.files.private
|
||||||
io.pathnames kernel locals make math sequences
|
io.pathnames kernel locals make math sequences
|
||||||
specialized-arrays system threads tools.test vocabs ;
|
specialized-arrays system threads tools.test vocabs ;
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
! Copyright (C) 2009 Doug Coleman.
|
! Copyright (C) 2009 Doug Coleman.
|
||||||
! See http://factorcode.org/license.txt for BSD license.
|
! See http://factorcode.org/license.txt for BSD license.
|
||||||
USING: combinators combinators.smart csv io.encodings.8-bit
|
USING: combinators combinators.smart csv io.encodings.latin1
|
||||||
kernel math.parser memoize money sequences unicode ;
|
kernel math.parser memoize money sequences unicode ;
|
||||||
IN: geobytes
|
IN: geobytes
|
||||||
|
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
USING: accessors arrays assocs continuations formatting graphviz
|
USING: accessors arrays assocs continuations formatting graphviz
|
||||||
graphviz.notation graphviz.render graphviz.render.private
|
graphviz.notation graphviz.render graphviz.render.private
|
||||||
images.loader.private io.directories io.encodings.8-bit
|
images.loader.private io.directories io.encodings.ascii
|
||||||
io.encodings.ascii io.encodings.utf8 io.files io.launcher kernel
|
io.encodings.latin1 io.encodings.utf8 io.files io.launcher kernel
|
||||||
locals make math math.combinatorics math.parser namespaces
|
locals make math math.combinatorics math.parser namespaces
|
||||||
sequences sequences.extras sets splitting system tools.test ;
|
sequences sequences.extras sets splitting system tools.test ;
|
||||||
IN: graphviz.tests
|
IN: graphviz.tests
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
! Copyright (C) 2012 Alex Vondrak.
|
! Copyright (C) 2012 Alex Vondrak.
|
||||||
! See http://factorcode.org/license.txt for BSD license.
|
! See http://factorcode.org/license.txt for BSD license.
|
||||||
USING: graphviz help.markup help.syntax images.viewer
|
USING: graphviz help.markup help.syntax images.viewer
|
||||||
io.encodings.8-bit io.encodings.utf8 io.launcher kernel
|
io.encodings.latin1 io.encodings.utf8 io.launcher kernel
|
||||||
strings ;
|
strings ;
|
||||||
IN: graphviz.render
|
IN: graphviz.render
|
||||||
|
|
||||||
|
|
|
@ -2,7 +2,7 @@
|
||||||
! See http://factorcode.org/license.txt for BSD license.
|
! See http://factorcode.org/license.txt for BSD license.
|
||||||
USING: calendar combinators compiler.units continuations
|
USING: calendar combinators compiler.units continuations
|
||||||
graphviz.dot images.viewer io.backend io.directories
|
graphviz.dot images.viewer io.backend io.directories
|
||||||
io.encodings.8-bit io.encodings.utf8 io.files
|
io.encodings.latin1 io.encodings.utf8 io.files
|
||||||
io.files.temp io.files.unique io.launcher io.standard-paths
|
io.files.temp io.files.unique io.launcher io.standard-paths
|
||||||
kernel locals make namespaces sequences summary system threads
|
kernel locals make namespaces sequences summary system threads
|
||||||
unicode vocabs webbrowser words ;
|
unicode vocabs webbrowser words ;
|
||||||
|
|
|
@ -3,7 +3,7 @@
|
||||||
USING: accessors alien.c-types alien.data arrays byte-arrays
|
USING: accessors alien.c-types alien.data arrays byte-arrays
|
||||||
combinators compression.run-length fry grouping images
|
combinators compression.run-length fry grouping images
|
||||||
images.loader images.normalization io io.binary
|
images.loader images.normalization io io.binary
|
||||||
io.encodings.8-bit io.encodings.string io.streams.throwing
|
io.encodings.latin1 io.encodings.string io.streams.throwing
|
||||||
kernel math math.bitwise sequences specialized-arrays summary ;
|
kernel math math.bitwise sequences specialized-arrays summary ;
|
||||||
QUALIFIED-WITH: bitstreams b
|
QUALIFIED-WITH: bitstreams b
|
||||||
SPECIALIZED-ARRAYS: uint ushort ;
|
SPECIALIZED-ARRAYS: uint ushort ;
|
||||||
|
|
|
@ -2,10 +2,10 @@
|
||||||
! See http://factorcode.org/license.txt for BSD license.
|
! See http://factorcode.org/license.txt for BSD license.
|
||||||
USING: accessors arrays assocs checksums checksums.crc32
|
USING: accessors arrays assocs checksums checksums.crc32
|
||||||
combinators compression.inflate fry grouping images
|
combinators compression.inflate fry grouping images
|
||||||
images.loader io io.binary io.encodings.8-bit io.encodings.ascii
|
images.loader io io.binary io.encodings.ascii
|
||||||
io.encodings.binary io.encodings.string io.streams.byte-array
|
io.encodings.binary io.encodings.latin1 io.encodings.string
|
||||||
io.streams.throwing kernel locals math math.bitwise
|
io.streams.byte-array io.streams.throwing kernel locals math
|
||||||
math.functions sequences sorting ;
|
math.bitwise math.functions sequences sorting ;
|
||||||
QUALIFIED: bitstreams
|
QUALIFIED: bitstreams
|
||||||
IN: images.png
|
IN: images.png
|
||||||
|
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
! Copyright (C) 2010 Joe Groff.
|
! Copyright (C) 2010 Joe Groff.
|
||||||
! See http://factorcode.org/license.txt for BSD license.
|
! See http://factorcode.org/license.txt for BSD license.
|
||||||
USING: byte-arrays io.encodings.8-bit io.encodings.binary
|
USING: byte-arrays io.encodings.binary io.encodings.detect
|
||||||
io.encodings.detect io.encodings.utf16 io.encodings.utf32
|
io.encodings.latin1 io.encodings.utf16 io.encodings.utf32
|
||||||
io.encodings.utf8 namespaces tools.test ;
|
io.encodings.utf8 namespaces tools.test ;
|
||||||
|
|
||||||
! UTF encodings with BOMs
|
! UTF encodings with BOMs
|
||||||
|
|
|
@ -1,15 +1,14 @@
|
||||||
! Copyright (C) 2010 Joe Groff.
|
! Copyright (C) 2010 Joe Groff.
|
||||||
! See http://factorcode.org/license.txt for BSD license.
|
! See http://factorcode.org/license.txt for BSD license.
|
||||||
USING: accessors byte-arrays combinators continuations fry io
|
USING: byte-arrays combinators continuations fry io io.encodings
|
||||||
io.encodings io.encodings.8-bit io.encodings.ascii
|
io.encodings.ascii io.encodings.binary io.encodings.iana
|
||||||
io.encodings.binary io.encodings.iana io.encodings.string
|
io.encodings.latin1 io.encodings.string io.encodings.utf16
|
||||||
io.encodings.utf16 io.encodings.utf32 io.encodings.utf8
|
io.encodings.utf32 io.encodings.utf8 io.files kernel literals
|
||||||
io.files io.streams.string kernel literals math namespaces
|
math namespaces sequences strings ;
|
||||||
sequences strings ;
|
|
||||||
IN: io.encodings.detect
|
IN: io.encodings.detect
|
||||||
|
|
||||||
SYMBOL: default-8bit-encoding
|
SYMBOL: default-encoding
|
||||||
default-8bit-encoding [ latin1 ] initialize
|
default-encoding [ latin1 ] initialize
|
||||||
|
|
||||||
<PRIVATE
|
<PRIVATE
|
||||||
|
|
||||||
|
@ -41,7 +40,7 @@ PRIVATE>
|
||||||
{ [ 0 over member? ] [ drop binary ] }
|
{ [ 0 over member? ] [ drop binary ] }
|
||||||
{ [ dup empty? ] [ drop utf8 ] }
|
{ [ dup empty? ] [ drop utf8 ] }
|
||||||
{ [ dup valid-utf8? ] [ drop utf8 ] }
|
{ [ dup valid-utf8? ] [ drop utf8 ] }
|
||||||
[ drop default-8bit-encoding get ]
|
[ drop default-encoding get ]
|
||||||
} cond ;
|
} cond ;
|
||||||
|
|
||||||
: detect-stream ( stream -- sample encoding )
|
: detect-stream ( stream -- sample encoding )
|
||||||
|
|
Loading…
Reference in New Issue