From 4ef0a41d05bb36feb1fa1d5cb19392f33731b209 Mon Sep 17 00:00:00 2001
From: Doug Coleman <doug.coleman@gmail.com>
Date: Thu, 17 Oct 2019 00:48:52 -0500
Subject: [PATCH] modern: If we find whitespace, there was no delimiter, so
 it's just a token.

---
 extra/modern/modern.factor | 30 ++++++++++++------------------
 1 file changed, 12 insertions(+), 18 deletions(-)

diff --git a/extra/modern/modern.factor b/extra/modern/modern.factor
index cb9cdd29a4..1fb8894397 100644
--- a/extra/modern/modern.factor
+++ b/extra/modern/modern.factor
@@ -358,18 +358,10 @@ DEFER: lex-factor-top*
         over "\\" tail? [ drop (read-backslash) ] [ lex-factor-top* ] if
     ] if ;
 
-! If the slice is 0 width, we stopped on whitespace.
-! Advance the index and read again!
-
-: read-token-or-whitespace-top ( string n slice -- string n' slice/f )
+! If the slice is 0 width, we stopped on whitespace before any token.
+! Return it to the main loop as a ws form.
+: read-token-or-whitespace ( string n slice -- string n' slice/f )
     dup length 0 = [
-        ! [ 1 + ] 2dip drop lex-factor-top
-        merge-slice-til-not-whitespace <ws>
-    ] when ;
-
-: read-token-or-whitespace-nested ( string n slice -- string n' slice/f )
-    dup length 0 = [
-        ! [ 1 + ] 2dip drop lex-factor-nested
         merge-slice-til-not-whitespace <ws>
     ] when ;
 
@@ -418,9 +410,11 @@ DEFER: lex-factor-top*
             [ slice-til-whitespace drop ] dip span-slices
             dup section-open? [ rewind-slice f ] when
         ] }
-        { char: \s [ read-token-or-whitespace-nested ] }
-        { char: \r [ read-token-or-whitespace-nested ] }
-        { char: \n [ read-token-or-whitespace-nested ] }
+
+        ! Two cases: zero width slice if we found whitespace token, otherwise text token
+        { char: \s [ read-token-or-whitespace ] }
+        { char: \r [ read-token-or-whitespace ] }
+        { char: \n [ read-token-or-whitespace ] }
         [ lex-factor-fallthrough ]
     } case ;
 
@@ -445,10 +439,10 @@ DEFER: lex-factor-top*
             dup section-open? [ read-acute ] when
         ] }
 
-        ! Two cases: zero width slice if we found whitespace, otherwise token
-        { char: \s [ read-token-or-whitespace-top ] }
-        { char: \r [ read-token-or-whitespace-top ] }
-        { char: \n [ read-token-or-whitespace-top ] }
+        ! Two cases: zero width slice if we found whitespace token, otherwise text token
+        { char: \s [ read-token-or-whitespace ] }
+        { char: \r [ read-token-or-whitespace ] }
+        { char: \n [ read-token-or-whitespace ] }
         [ lex-factor-fallthrough ]
     } case ;