modern: wrap whitespace

extra/modern/lexer/lexer.factor

@@ -1,8 +1,20 @@
 ! Copyright (C) 2019 Doug Coleman.
 ! See http://factorcode.org/license.txt for BSD license.
-USING: constructors ;
+USING: accessors ascii constructors kernel prettyprint.custom
+sequences ;
 IN: modern.lexer
 
+ERROR: ws-expected string ;
+
+TUPLE: ws string ;
+CONSTRUCTOR: <ws> ws ( string -- ws )
+    dup string>> [ blank? not ] any? [ ws-expected ] when ;
+
+! Weird experiment
+M: ws pprint*
+    drop ;
+    ! string>> dup "\"" "\"" pprint-string ;
+
 TUPLE: lexed tokens ;
 
 TUPLE: bracket < lexed tag payload ;

extra/modern/modern.factor

@@ -134,7 +134,7 @@ MACRO:: read-matched ( ch -- quot: ( string n tag -- string n' slice' ) )
     2over ?nth-of char: \[ = [
         [ 1 + ] dip 1 modify-to 2over ?nth-of read-double-matched-bracket
     ] [
-        [ slice-til-eol drop ] dip swap 2array
+        [ slice-til-eol drop <ws> ] dip swap 2array
     ] if ;
 
 : terminator? ( slice -- ? )
@@ -152,7 +152,7 @@ MACRO:: read-matched ( ch -- quot: ( string n tag -- string n' slice' ) )
     dup [ char: \: = ] count-tail
     '[
         _ [
-            slice-til-not-whitespace drop ! XXX: whitespace here
+            slice-til-not-whitespace drop <ws> ! XXX: whitespace here
             [ dup [ f unexpected-eof ] unless ] dip
             [ lex-factor ] dip swap 2array
         ] replicate
@@ -335,7 +335,7 @@ MACRO:: read-matched ( ch -- quot: ( string n tag -- string n' slice' ) )
         dup [ char: \\ = ] count-tail
         '[
             _ [
-                slice-til-not-whitespace drop
+                slice-til-not-whitespace drop <ws>
                 [ slice-til-whitespace drop ] dip
                 swap 2array
             ] replicate