io.encodings: use new latin1 instead of 8-bit encoding from file.
parent
093e6411be
commit
0ab7ed6979
|
@ -3,7 +3,7 @@
|
|||
USING: accessors calendar calendar.format classes combinators
|
||||
combinators.short-circuit concurrency.promises continuations
|
||||
destructors ftp io io.directories io.encodings
|
||||
io.encodings.8-bit io.encodings.binary io.encodings.utf8
|
||||
io.encodings.binary io.encodings.latin1 io.encodings.utf8
|
||||
io.files io.files.info io.pathnames io.servers io.sockets
|
||||
io.streams.string io.timeouts kernel logging math math.bitwise
|
||||
math.parser namespaces sequences simple-tokenizer splitting
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
USING: assocs byte-arrays destructors help.markup help.syntax http
|
||||
http.client.post-data.private http.client.private
|
||||
io.encodings.8-bit io.encodings.binary io.pathnames kernel
|
||||
io.encodings.binary io.encodings.latin1 io.pathnames kernel
|
||||
sequences strings urls urls.encoding ;
|
||||
IN: http.client
|
||||
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
USING: destructors http http.server http.server.requests http.client
|
||||
http.client.private tools.test multiline fry io.streams.string io.crlf
|
||||
io.encodings.utf8 io.encodings.8-bit io.encodings.binary io.encodings.string
|
||||
io.encodings.utf8 io.encodings.latin1 io.encodings.binary io.encodings.string
|
||||
io.encodings.ascii kernel arrays splitting sequences assocs io.sockets db
|
||||
db.sqlite make continuations urls hashtables accessors namespaces xml.data
|
||||
random combinators.short-circuit literals ;
|
||||
|
|
|
@ -11,3 +11,5 @@ io.encodings.8-bit.private strings tools.test ;
|
|||
|
||||
{ t } [ \ latin1 8-bit-encoding? ] unit-test
|
||||
{ "bar" } [ "bar" \ latin1 decode ] unit-test
|
||||
|
||||
{ { 0x221a 0x00b1 0x0040 } } [ B{ 0xfb 0xf1 0x40 } cp437 decode >array ] unit-test
|
||||
|
|
|
@ -1,8 +1,8 @@
|
|||
! Copyright (C) 2008 Daniel Ehrenberg, Doug Coleman.
|
||||
! See http://factorcode.org/license.txt for BSD license.
|
||||
USING: accessors assocs biassocs classes.mixin classes.singleton
|
||||
fry io io.encodings io.encodings.iana kernel lexer namespaces
|
||||
parser sequences simple-flat-file ;
|
||||
USING: accessors assocs biassocs classes.singleton generic io
|
||||
io.encodings io.encodings.iana kernel lexer parser sequences
|
||||
simple-flat-file words ;
|
||||
IN: io.encodings.8-bit
|
||||
|
||||
<<
|
||||
|
@ -11,41 +11,28 @@ IN: io.encodings.8-bit
|
|||
: encoding-file ( file-name -- stream )
|
||||
"vocab:io/encodings/8-bit/" ".TXT" surround ;
|
||||
|
||||
SYMBOL: 8-bit-encodings
|
||||
8-bit-encodings [ H{ } clone ] initialize
|
||||
|
||||
TUPLE: 8-bit { biassoc biassoc read-only } ;
|
||||
TUPLE: 8-bit { table biassoc read-only } ;
|
||||
|
||||
: 8-bit-encode ( char 8-bit -- byte )
|
||||
biassoc>> value-at [ encode-error ] unless* ; inline
|
||||
table>> value-at [ encode-error ] unless* ; inline
|
||||
|
||||
M: 8-bit encode-char
|
||||
swap [ 8-bit-encode ] dip stream-write1 ;
|
||||
|
||||
M: 8-bit decode-char
|
||||
swap stream-read1
|
||||
[ swap biassoc>> at [ replacement-char ] unless* ]
|
||||
[ drop f ] if* ;
|
||||
|
||||
MIXIN: 8-bit-encoding
|
||||
|
||||
M: 8-bit-encoding <encoder>
|
||||
8-bit-encodings get-global at <encoder> ;
|
||||
|
||||
M: 8-bit-encoding <decoder>
|
||||
8-bit-encodings get-global at <decoder> ;
|
||||
swap stream-read1 [
|
||||
swap table>> at [ replacement-char ] unless*
|
||||
] [ drop f ] if* ;
|
||||
|
||||
: create-encoding ( name -- word )
|
||||
create-word-in
|
||||
[ define-singleton-class ]
|
||||
[ 8-bit-encoding add-mixin-instance ]
|
||||
[ ] tri ;
|
||||
create-word-in dup define-singleton-class ;
|
||||
|
||||
: load-encoding ( name iana-name file-name -- )
|
||||
[ create-encoding dup ]
|
||||
[ register-encoding ]
|
||||
[ encoding-file load-codetable-file 8-bit boa ] tri*
|
||||
swap 8-bit-encodings get-global set-at ;
|
||||
[ [ \ <encoder> create-method ] dip [ nip <encoder> ] curry define ]
|
||||
[ [ \ <decoder> create-method ] dip [ nip <decoder> ] curry define ] 2bi ;
|
||||
|
||||
PRIVATE>
|
||||
|
||||
|
@ -75,7 +62,7 @@ SYNTAX: 8-BIT: scan-token scan-token scan-token load-encoding ;
|
|||
8-BIT: latin/greek ISO_8859-7:1987 8859-7
|
||||
8-BIT: latin/hebrew ISO_8859-8:1988 8859-8
|
||||
8-BIT: latin/thai TIS-620 8859-11
|
||||
8-BIT: latin1 ISO_8859-1:1987 8859-1
|
||||
! 8-BIT: latin1 ISO_8859-1:1987 8859-1
|
||||
8-BIT: latin2 ISO_8859-2:1987 8859-2
|
||||
8-BIT: latin3 ISO_8859-3:1988 8859-3
|
||||
8-BIT: latin4 ISO_8859-4:1988 8859-4
|
||||
|
|
|
@ -3,7 +3,7 @@
|
|||
USING: accessors alien alien.c-types alien.data alien.enums
|
||||
alien.strings assocs byte-arrays classes.struct combinators
|
||||
combinators.short-circuit destructors fry io io.backend
|
||||
io.binary io.buffers io.encodings.8-bit io.encodings.utf8
|
||||
io.binary io.buffers io.encodings.latin1 io.encodings.utf8
|
||||
io.files io.pathnames io.ports io.sockets io.sockets.secure
|
||||
io.timeouts kernel libc locals math math.functions math.order
|
||||
math.parser memoize namespaces openssl openssl.libcrypto
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
USING: destructors io io.encodings io.encodings.8-bit
|
||||
USING: destructors io io.encodings io.encodings.latin1
|
||||
io.encodings.ascii io.encodings.binary io.encodings.string
|
||||
io.encodings.utf8 io.files io.pipes io.streams.byte-array
|
||||
io.streams.duplex io.streams.limited io.streams.string kernel
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
! Copyright (C) 2009 Daniel Ehrenberg
|
||||
! See http://factorcode.org/license.txt for BSD license.
|
||||
USING: tools.test quoted-printable io.encodings.string
|
||||
sequences splitting kernel io.encodings.8-bit ;
|
||||
sequences splitting kernel io.encodings.latin1 ;
|
||||
IN: quoted-printable.tests
|
||||
|
||||
{ "José was the
|
||||
|
|
|
@ -2,6 +2,6 @@ USING: io.encodings.string kernel io.encodings.8-bit ;
|
|||
IN: tools.deploy.test.4
|
||||
|
||||
: deploy-test-4 ( -- )
|
||||
"xyzthg" \ latin7 encode drop ;
|
||||
"xyzthg" latin7 encode drop ;
|
||||
|
||||
MAIN: deploy-test-4
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
USING: alien.strings alien.c-types alien.data tools.test
|
||||
kernel libc io.encodings.utf8 io.encodings.utf16 io.encodings.utf16n
|
||||
io.encodings.ascii alien io.encodings.string io.encodings.8-bit ;
|
||||
io.encodings.ascii alien io.encodings.string io.encodings.latin1 ;
|
||||
|
||||
{ "\u0000ff" }
|
||||
[ "\u0000ff" latin1 string>alien latin1 alien>string ]
|
||||
|
|
|
@ -1,10 +1,10 @@
|
|||
! Copyright (C) 2008, 2011 Slava Pestov.
|
||||
! See http://factorcode.org/license.txt for BSD license.
|
||||
USING: accessors alien arrays byte-arrays byte-vectors init io
|
||||
io.encodings io.encodings.ascii io.encodings.utf16n
|
||||
io.encodings.utf8 io.streams.memory kernel kernel.private math
|
||||
namespaces sequences sequences.private strings strings.private
|
||||
system system.private ;
|
||||
io.encodings io.encodings.ascii io.encodings.latin1
|
||||
io.encodings.utf16n io.encodings.utf8 io.streams.memory kernel
|
||||
kernel.private math namespaces sequences sequences.private
|
||||
strings strings.private system system.private ;
|
||||
IN: alien.strings
|
||||
|
||||
GENERIC#: alien>string 1 ( c-ptr encoding -- string/f )
|
||||
|
@ -31,7 +31,7 @@ M: c-ptr string>alien drop ;
|
|||
<PRIVATE
|
||||
|
||||
: fast-string? ( string encoding -- ? )
|
||||
swap aux>> not [ { ascii utf8 } member-eq? ] [ drop f ] if ; inline
|
||||
swap aux>> not [ { ascii latin1 utf8 } member-eq? ] [ drop f ] if ; inline
|
||||
|
||||
: string>alien-fast ( string encoding -- byte-array )
|
||||
{ string object } declare ! aux>> must be f
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
USING: alien alien.c-types alien.data arrays classes.struct
|
||||
compiler.units continuations destructors fry generic.single io
|
||||
io.backend io.directories io.encodings io.encodings.8-bit
|
||||
io.encodings.ascii io.encodings.binary io.encodings.string
|
||||
io.backend io.directories io.encodings io.encodings.ascii
|
||||
io.encodings.binary io.encodings.latin1 io.encodings.string
|
||||
io.encodings.utf16 io.encodings.utf8 io.files io.files.private
|
||||
io.pathnames kernel locals make math sequences
|
||||
specialized-arrays system threads tools.test vocabs ;
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
! Copyright (C) 2009 Doug Coleman.
|
||||
! See http://factorcode.org/license.txt for BSD license.
|
||||
USING: combinators combinators.smart csv io.encodings.8-bit
|
||||
USING: combinators combinators.smart csv io.encodings.latin1
|
||||
kernel math.parser memoize money sequences unicode ;
|
||||
IN: geobytes
|
||||
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
USING: accessors arrays assocs continuations formatting graphviz
|
||||
graphviz.notation graphviz.render graphviz.render.private
|
||||
images.loader.private io.directories io.encodings.8-bit
|
||||
io.encodings.ascii io.encodings.utf8 io.files io.launcher kernel
|
||||
images.loader.private io.directories io.encodings.ascii
|
||||
io.encodings.latin1 io.encodings.utf8 io.files io.launcher kernel
|
||||
locals make math math.combinatorics math.parser namespaces
|
||||
sequences sequences.extras sets splitting system tools.test ;
|
||||
IN: graphviz.tests
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
! Copyright (C) 2012 Alex Vondrak.
|
||||
! See http://factorcode.org/license.txt for BSD license.
|
||||
USING: graphviz help.markup help.syntax images.viewer
|
||||
io.encodings.8-bit io.encodings.utf8 io.launcher kernel
|
||||
io.encodings.latin1 io.encodings.utf8 io.launcher kernel
|
||||
strings ;
|
||||
IN: graphviz.render
|
||||
|
||||
|
|
|
@ -2,7 +2,7 @@
|
|||
! See http://factorcode.org/license.txt for BSD license.
|
||||
USING: calendar combinators compiler.units continuations
|
||||
graphviz.dot images.viewer io.backend io.directories
|
||||
io.encodings.8-bit io.encodings.utf8 io.files
|
||||
io.encodings.latin1 io.encodings.utf8 io.files
|
||||
io.files.temp io.files.unique io.launcher io.standard-paths
|
||||
kernel locals make namespaces sequences summary system threads
|
||||
unicode vocabs webbrowser words ;
|
||||
|
|
|
@ -3,7 +3,7 @@
|
|||
USING: accessors alien.c-types alien.data arrays byte-arrays
|
||||
combinators compression.run-length fry grouping images
|
||||
images.loader images.normalization io io.binary
|
||||
io.encodings.8-bit io.encodings.string io.streams.throwing
|
||||
io.encodings.latin1 io.encodings.string io.streams.throwing
|
||||
kernel math math.bitwise sequences specialized-arrays summary ;
|
||||
QUALIFIED-WITH: bitstreams b
|
||||
SPECIALIZED-ARRAYS: uint ushort ;
|
||||
|
|
|
@ -2,10 +2,10 @@
|
|||
! See http://factorcode.org/license.txt for BSD license.
|
||||
USING: accessors arrays assocs checksums checksums.crc32
|
||||
combinators compression.inflate fry grouping images
|
||||
images.loader io io.binary io.encodings.8-bit io.encodings.ascii
|
||||
io.encodings.binary io.encodings.string io.streams.byte-array
|
||||
io.streams.throwing kernel locals math math.bitwise
|
||||
math.functions sequences sorting ;
|
||||
images.loader io io.binary io.encodings.ascii
|
||||
io.encodings.binary io.encodings.latin1 io.encodings.string
|
||||
io.streams.byte-array io.streams.throwing kernel locals math
|
||||
math.bitwise math.functions sequences sorting ;
|
||||
QUALIFIED: bitstreams
|
||||
IN: images.png
|
||||
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
! Copyright (C) 2010 Joe Groff.
|
||||
! See http://factorcode.org/license.txt for BSD license.
|
||||
USING: byte-arrays io.encodings.8-bit io.encodings.binary
|
||||
io.encodings.detect io.encodings.utf16 io.encodings.utf32
|
||||
USING: byte-arrays io.encodings.binary io.encodings.detect
|
||||
io.encodings.latin1 io.encodings.utf16 io.encodings.utf32
|
||||
io.encodings.utf8 namespaces tools.test ;
|
||||
|
||||
! UTF encodings with BOMs
|
||||
|
|
|
@ -1,15 +1,14 @@
|
|||
! Copyright (C) 2010 Joe Groff.
|
||||
! See http://factorcode.org/license.txt for BSD license.
|
||||
USING: accessors byte-arrays combinators continuations fry io
|
||||
io.encodings io.encodings.8-bit io.encodings.ascii
|
||||
io.encodings.binary io.encodings.iana io.encodings.string
|
||||
io.encodings.utf16 io.encodings.utf32 io.encodings.utf8
|
||||
io.files io.streams.string kernel literals math namespaces
|
||||
sequences strings ;
|
||||
USING: byte-arrays combinators continuations fry io io.encodings
|
||||
io.encodings.ascii io.encodings.binary io.encodings.iana
|
||||
io.encodings.latin1 io.encodings.string io.encodings.utf16
|
||||
io.encodings.utf32 io.encodings.utf8 io.files kernel literals
|
||||
math namespaces sequences strings ;
|
||||
IN: io.encodings.detect
|
||||
|
||||
SYMBOL: default-8bit-encoding
|
||||
default-8bit-encoding [ latin1 ] initialize
|
||||
SYMBOL: default-encoding
|
||||
default-encoding [ latin1 ] initialize
|
||||
|
||||
<PRIVATE
|
||||
|
||||
|
@ -41,7 +40,7 @@ PRIVATE>
|
|||
{ [ 0 over member? ] [ drop binary ] }
|
||||
{ [ dup empty? ] [ drop utf8 ] }
|
||||
{ [ dup valid-utf8? ] [ drop utf8 ] }
|
||||
[ drop default-8bit-encoding get ]
|
||||
[ drop default-encoding get ]
|
||||
} cond ;
|
||||
|
||||
: detect-stream ( stream -- sample encoding )
|
||||
|
|
Loading…
Reference in New Issue