modern: working on compound tokens

extra/modern/modern-tests.factor

@@ -139,3 +139,28 @@ IN: modern.tests
 {
     { { "\"" "google.com" "\"" } }
 } [ [[ "google.com" ]] string>literals >strings ] unit-test
+
+! Concatenated syntax
+{
+    {
+        { "(" { "a" "b" } ")" }
+        { "[" { "a" "b" "+" } "]" }
+        { "(" { "c" } ")" }
+    }
+} [ "( a b )[ a b + ]( c )" string>literals >strings ] unit-test
+
+{
+    {
+        { "(" { "a" "b" } ")" }
+        { "[" { "a" "b" "+" } "]" }
+        { "(" { "c" } ")" }
+    }
+} [ "( a b ) [ a b + ] ( c )" string>literals >strings ] unit-test
+
+
+{
+    {
+        { "\"" "abc" "\"" }
+        { "[" { "0" } "]" }
+    }
+} [ "\"abc\"[ 0 ]" string>literals >strings ] unit-test

extra/modern/modern.factor

@@ -267,6 +267,9 @@ ERROR: mismatched-terminator n string slice ;
         { char: \[ [ read-bracket ] }
         { char: \{ [ read-brace ] }
         { char: \( [ read-paren ] }
+        { char: \] [  ] }
+        { char: \} [  ] }
+        { char: \) [  ] }
         { char: \s [ read-token-or-whitespace ] }
         { char: \r [ read-token-or-whitespace ] }
         { char: \n [ read-token-or-whitespace ] }
@@ -275,9 +278,11 @@ ERROR: mismatched-terminator n string slice ;
 
 : lex-factor ( n/f string -- n'/f string literal )
     over [
-        skip-whitespace "\"\\!:[{(<>\s\r\n" slice-til-either
+        ! skip-whitespace
+        "\"\\!:[{(]})<>\s\r\n" slice-til-either
+
         ! \foo foo\bar \foo{
-        dup char: \\ = [
+        dup char: \\ =  [
             drop
             ! foo\ so far, could be foo\bar{
             ! remove the \ and continue til delimiter/eof

extra/modern/out/out.factor

@@ -1,11 +1,15 @@
 ! Copyright (C) 2017 Doug Coleman.
 ! See http://factorcode.org/license.txt for BSD license.
-USING: accessors arrays assocs constructors continuations fry io
+USING: accessors arrays assocs combinators.short-circuit
-io.encodings.utf8 io.files io.streams.string kernel modern
+constructors continuations fry io io.encodings.utf8 io.files
-modern.paths modern.slices multiline namespaces prettyprint
+io.streams.string kernel modern modern.paths modern.slices
-sequences sequences.extras splitting strings ;
+multiline namespaces prettyprint sequences sequences.extras
+splitting strings ;
 IN: modern.out
 
+: token? ( obj -- ? )
+    { [ slice? ] [ seq>> string? ] } 1&& ;
+
 TUPLE: renamed slice string ;
 CONSTRUCTOR: <renamed> renamed ( slice string -- obj ) ;