Merge branch 'master' of git://factorcode.org/git/factor

db4
Slava Pestov 2009-07-03 21:33:55 -05:00
commit a3c3445643
3 changed files with 40 additions and 1 deletions

View File

@ -7,11 +7,16 @@ ARTICLE: "unicode.breaks" "Word and grapheme breaks"
"The " { $vocab-link "unicode.breaks" "unicode.breaks" } " vocabulary partially implements Unicode Standard Annex #29. This provides for segmentation of a string along grapheme and word boundaries. In Unicode, a grapheme, or a basic unit of display in text, may be more than one code point. For example, in the string \"e\\u000301\" (where U+0301 is a combining acute accent), there is only one grapheme, as the acute accent goes above the e, forming a single grapheme. Word breaks, in general, are more complicated than simply splitting by whitespace, and the Unicode algorithm provides for that." "The " { $vocab-link "unicode.breaks" "unicode.breaks" } " vocabulary partially implements Unicode Standard Annex #29. This provides for segmentation of a string along grapheme and word boundaries. In Unicode, a grapheme, or a basic unit of display in text, may be more than one code point. For example, in the string \"e\\u000301\" (where U+0301 is a combining acute accent), there is only one grapheme, as the acute accent goes above the e, forming a single grapheme. Word breaks, in general, are more complicated than simply splitting by whitespace, and the Unicode algorithm provides for that."
$nl "Operations for graphemes:" $nl "Operations for graphemes:"
{ $subsection first-grapheme } { $subsection first-grapheme }
{ $subsection first-grapheme-from }
{ $subsection last-grapheme } { $subsection last-grapheme }
{ $subsection last-grapheme-from }
{ $subsection >graphemes } { $subsection >graphemes }
{ $subsection string-reverse } { $subsection string-reverse }
"Operations on words:" "Operations on words:"
{ $subsection first-word } { $subsection first-word }
{ $subsection first-word-from }
{ $subsection last-word }
{ $subsection last-word-from }
{ $subsection >words } ; { $subsection >words } ;
HELP: first-grapheme HELP: first-grapheme
@ -22,6 +27,14 @@ HELP: last-grapheme
{ $values { "str" string } { "i" "an index" } } { $values { "str" string } { "i" "an index" } }
{ $description "Finds the index of the start of the last grapheme of the string. This can be used to traverse the graphemes of a string backwards." } ; { $description "Finds the index of the start of the last grapheme of the string. This can be used to traverse the graphemes of a string backwards." } ;
HELP: first-grapheme-from
{ $values { "start" "an index" } { "str" string } { "i" "an index" } }
{ $description "Finds the length of the first grapheme of the string, starting from the given index. This can be used repeatedly to efficiently traverse the graphemes of the string, using slices." } ;
HELP: last-grapheme-from
{ $values { "end" "an index" } { "str" string } { "i" "an index" } }
{ $description "Finds the index of the start of the last grapheme of the string, starting from the given index. This can be used to traverse the graphemes of a string backwards." } ;
HELP: >graphemes HELP: >graphemes
{ $values { "str" string } { "graphemes" "an array of strings" } } { $values { "str" string } { "graphemes" "an array of strings" } }
{ $description "Divides a string into a sequence of individual graphemes." } ; { $description "Divides a string into a sequence of individual graphemes." } ;
@ -32,7 +45,19 @@ HELP: string-reverse
HELP: first-word HELP: first-word
{ $values { "str" string } { "i" "index" } } { $values { "str" string } { "i" "index" } }
{ $description "Finds the length of the first word in the string." } ; { $description "Finds the index of the end of the first word in the string." } ;
HELP: last-word
{ $values { "str" string } { "i" "index" } }
{ $description "Finds the index of the beginning of the last word in the string." } ;
HELP: first-word-from
{ $values { "start" "index" } { "str" string } { "i" "index" } }
{ $description "Finds the index of the end of the first word in the string, starting from the given index." } ;
HELP: last-word-from
{ $values { "end" "index" } { "str" string } { "i" "index" } }
{ $description "Finds the index of the start of the word that the index is contained in." } ;
HELP: >words HELP: >words
{ $values { "str" string } { "words" "an array of strings" } } { $values { "str" string } { "words" "an array of strings" } }

View File

@ -12,6 +12,11 @@ IN: unicode.breaks.tests
[ 3 ] [ 2 "hello" first-grapheme-from ] unit-test [ 3 ] [ 2 "hello" first-grapheme-from ] unit-test
[ 1 ] [ 2 "hello" last-grapheme-from ] unit-test [ 1 ] [ 2 "hello" last-grapheme-from ] unit-test
[ 4 ] [ 2 "what am I saying" first-word-from ] unit-test
[ 0 ] [ 2 "what am I saying" last-word-from ] unit-test
[ 16 ] [ 11 "what am I saying" first-word-from ] unit-test
[ 10 ] [ 11 "what am I saying" last-word-from ] unit-test
: grapheme-break-test ( -- filename ) : grapheme-break-test ( -- filename )
"vocab:unicode/breaks/GraphemeBreakTest.txt" ; "vocab:unicode/breaks/GraphemeBreakTest.txt" ;

View File

@ -247,3 +247,12 @@ PRIVATE>
word-break-next nip word-break-next nip
] ]
} 2|| ; } 2|| ;
: first-word-from ( start str -- i )
over tail-slice first-word + ;
: last-word ( str -- i )
[ length ] keep '[ _ word-break-at? ] find-last drop 0 or ;
: last-word-from ( end str -- i )
swap head-slice last-word ;