From b974133285990ffb7ffc9e427c3503bc08b42281 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Tue, 3 Jun 2008 11:01:04 +1200 Subject: [PATCH 01/47] Re-add jni library to unmaintained --- unmaintained/jni/jni-internals.factor | 357 ++++++++++++++++++++++++++ unmaintained/jni/jni.factor | 22 ++ unmaintained/jni/load.factor | 4 + 3 files changed, 383 insertions(+) create mode 100644 unmaintained/jni/jni-internals.factor create mode 100644 unmaintained/jni/jni.factor create mode 100644 unmaintained/jni/load.factor diff --git a/unmaintained/jni/jni-internals.factor b/unmaintained/jni/jni-internals.factor new file mode 100644 index 0000000000..49bc57b108 --- /dev/null +++ b/unmaintained/jni/jni-internals.factor @@ -0,0 +1,357 @@ +! Copyright (C) 2006 Chris Double. +! See http://factorcode.org/license.txt for BSD license. +IN: jni-internals +USING: kernel alien arrays sequences ; + +LIBRARY: jvm + +TYPEDEF: int jint +TYPEDEF: uchar jboolean +TYPEDEF: void* JNIEnv + +C-STRUCT: jdk-init-args + { "jint" "version" } + { "void*" "properties" } + { "jint" "check-source" } + { "jint" "native-stack-size" } + { "jint" "java-stack-size" } + { "jint" "min-heap-size" } + { "jint" "max-heap-size" } + { "jint" "verify-mode" } + { "char*" "classpath" } + { "void*" "vprintf" } + { "void*" "exit" } + { "void*" "abort" } + { "jint" "enable-class-gc" } + { "jint" "enable-verbose-gc" } + { "jint" "disable-async-gc" } + { "jint" "verbose" } + { "jboolean" "debugging" } + { "jint" "debug-port" } ; + +C-STRUCT: JNIInvokeInterface + { "void*" "reserved0" } + { "void*" "reserved1" } + { "void*" "reserved2" } + { "void*" "DestroyJavaVM" } + { "void*" "AttachCurrentThread" } + { "void*" "DetachCurrentThread" } + { "void*" "GetEnv" } + { "void*" "AttachCurrentThreadAsDaemon" } ; + +C-STRUCT: JavaVM + { "JNIInvokeInterface*" "functions" } ; + +C-STRUCT: JNINativeInterface + { "void*" "reserved0" } + { "void*" "reserved1" } + { "void*" "reserved2" } + { "void*" "reserved3" } + { "void*" "GetVersion" } + { "void*" "DefineClass" } + { "void*" "FindClass" } + { "void*" "FromReflectedMethod" } + { "void*" "FromReflectedField" } + { "void*" "ToReflectedMethod" } + { "void*" "GetSuperclass" } + { "void*" "IsAssignableFrom" } + { "void*" "ToReflectedField" } + { "void*" "Throw" } + { "void*" "ThrowNew" } + { "void*" "ExceptionOccurred" } + { "void*" "ExceptionDescribe" } + { "void*" "ExceptionClear" } + { "void*" "FatalError" } + { "void*" "PushLocalFrame" } + { "void*" "PopLocalFrame" } + { "void*" "NewGlobalRef" } + { "void*" "DeleteGlobalRef" } + { "void*" "DeleteLocalRef" } + { "void*" "IsSameObject" } + { "void*" "NewLocalRef" } + { "void*" "EnsureLocalCapacity" } + { "void*" "AllocObject" } + { "void*" "NewObject" } + { "void*" "NewObjectV" } + { "void*" "NewObjectA" } + { "void*" "GetObjectClass" } + { "void*" "IsInstanceOf" } + { "void*" "GetMethodID" } + { "void*" "CallObjectMethod" } + { "void*" "CallObjectMethodV" } + { "void*" "CallObjectMethodA" } + { "void*" "CallBooleanMethod" } + { "void*" "CallBooleanMethodV" } + { "void*" "CallBooleanMethodA" } + { "void*" "CallByteMethod" } + { "void*" "CallByteMethodV" } + { "void*" "CallByteMethodA" } + { "void*" "CallCharMethod" } + { "void*" "CallCharMethodV" } + { "void*" "CallCharMethodA" } + { "void*" "CallShortMethod" } + { "void*" "CallShortMethodV" } + { "void*" "CallShortMethodA" } + { "void*" "CallIntMethod" } + { "void*" "CallIntMethodV" } + { "void*" "CallIntMethodA" } + { "void*" "CallLongMethod" } + { "void*" "CallLongMethodV" } + { "void*" "CallLongMethodA" } + { "void*" "CallFloatMethod" } + { "void*" "CallFloatMethodV" } + { "void*" "CallFloatMethodA" } + { "void*" "CallDoubleMethod" } + { "void*" "CallDoubleMethodV" } + { "void*" "CallDoubleMethodA" } + { "void*" "CallVoidMethod" } + { "void*" "CallVoidMethodV" } + { "void*" "CallVoidMethodA" } + { "void*" "CallNonvirtualObjectMethod" } + { "void*" "CallNonvirtualObjectMethodV" } + { "void*" "CallNonvirtualObjectMethodA" } + { "void*" "CallNonvirtualBooleanMethod" } + { "void*" "CallNonvirtualBooleanMethodV" } + { "void*" "CallNonvirtualBooleanMethodA" } + { "void*" "CallNonvirtualByteMethod" } + { "void*" "CallNonvirtualByteMethodV" } + { "void*" "CallNonvirtualByteMethodA" } + { "void*" "CallNonvirtualCharMethod" } + { "void*" "CallNonvirtualCharMethodV" } + { "void*" "CallNonvirtualCharMethodA" } + { "void*" "CallNonvirtualShortMethod" } + { "void*" "CallNonvirtualShortMethodV" } + { "void*" "CallNonvirtualShortMethodA" } + { "void*" "CallNonvirtualIntMethod" } + { "void*" "CallNonvirtualIntMethodV" } + { "void*" "CallNonvirtualIntMethodA" } + { "void*" "CallNonvirtualLongMethod" } + { "void*" "CallNonvirtualLongMethodV" } + { "void*" "CallNonvirtualLongMethodA" } + { "void*" "CallNonvirtualFloatMethod" } + { "void*" "CallNonvirtualFloatMethodV" } + { "void*" "CallNonvirtualFloatMethodA" } + { "void*" "CallNonvirtualDoubleMethod" } + { "void*" "CallNonvirtualDoubleMethodV" } + { "void*" "CallNonvirtualDoubleMethodA" } + { "void*" "CallNonvirtualVoidMethod" } + { "void*" "CallNonvirtualVoidMethodV" } + { "void*" "CallNonvirtualVoidMethodA" } + { "void*" "GetFieldID" } + { "void*" "GetObjectField" } + { "void*" "GetBooleanField" } + { "void*" "GetByteField" } + { "void*" "GetCharField" } + { "void*" "GetShortField" } + { "void*" "GetIntField" } + { "void*" "GetLongField" } + { "void*" "GetFloatField" } + { "void*" "GetDoubleField" } + { "void*" "SetObjectField" } + { "void*" "SetBooleanField" } + { "void*" "SetByteField" } + { "void*" "SetCharField" } + { "void*" "SetShortField" } + { "void*" "SetIntField" } + { "void*" "SetLongField" } + { "void*" "SetFloatField" } + { "void*" "SetDoubleField" } + { "void*" "GetStaticMethodID" } + { "void*" "CallStaticObjectMethod" } + { "void*" "CallStaticObjectMethodV" } + { "void*" "CallStaticObjectMethodA" } + { "void*" "CallStaticBooleanMethod" } + { "void*" "CallStaticBooleanMethodV" } + { "void*" "CallStaticBooleanMethodA" } + { "void*" "CallStaticByteMethod" } + { "void*" "CallStaticByteMethodV" } + { "void*" "CallStaticByteMethodA" } + { "void*" "CallStaticCharMethod" } + { "void*" "CallStaticCharMethodV" } + { "void*" "CallStaticCharMethodA" } + { "void*" "CallStaticShortMethod" } + { "void*" "CallStaticShortMethodV" } + { "void*" "CallStaticShortMethodA" } + { "void*" "CallStaticIntMethod" } + { "void*" "CallStaticIntMethodV" } + { "void*" "CallStaticIntMethodA" } + { "void*" "CallStaticLongMethod" } + { "void*" "CallStaticLongMethodV" } + { "void*" "CallStaticLongMethodA" } + { "void*" "CallStaticFloatMethod" } + { "void*" "CallStaticFloatMethodV" } + { "void*" "CallStaticFloatMethodA" } + { "void*" "CallStaticDoubleMethod" } + { "void*" "CallStaticDoubleMethodV" } + { "void*" "CallStaticDoubleMethodA" } + { "void*" "CallStaticVoidMethod" } + { "void*" "CallStaticVoidMethodV" } + { "void*" "CallStaticVoidMethodA" } + { "void*" "GetStaticFieldID" } + { "void*" "GetStaticObjectField" } + { "void*" "GetStaticBooleanField" } + { "void*" "GetStaticByteField" } + { "void*" "GetStaticCharField" } + { "void*" "GetStaticShortField" } + { "void*" "GetStaticIntField" } + { "void*" "GetStaticLongField" } + { "void*" "GetStaticFloatField" } + { "void*" "GetStaticDoubleField" } + { "void*" "SetStaticObjectField" } + { "void*" "SetStaticBooleanField" } + { "void*" "SetStaticByteField" } + { "void*" "SetStaticCharField" } + { "void*" "SetStaticShortField" } + { "void*" "SetStaticIntField" } + { "void*" "SetStaticLongField" } + { "void*" "SetStaticFloatField" } + { "void*" "SetStaticDoubleField" } + { "void*" "NewString" } + { "void*" "GetStringLength" } + { "void*" "GetStringChars" } + { "void*" "ReleaseStringChars" } + { "void*" "NewStringUTF" } + { "void*" "GetStringUTFLength" } + { "void*" "GetStringUTFChars" } + { "void*" "ReleaseStringUTFChars" } + { "void*" "GetArrayLength" } + { "void*" "NewObjectArray" } + { "void*" "GetObjectArrayElement" } + { "void*" "SetObjectArrayElement" } + { "void*" "NewBooleanArray" } + { "void*" "NewByteArray" } + { "void*" "NewCharArray" } + { "void*" "NewShortArray" } + { "void*" "NewIntArray" } + { "void*" "NewLongArray" } + { "void*" "NewFloatArray" } + { "void*" "NewDoubleArray" } + { "void*" "GetBooleanArrayElements" } + { "void*" "GetByteArrayElements" } + { "void*" "GetCharArrayElements" } + { "void*" "GetShortArrayElements" } + { "void*" "GetIntArrayElements" } + { "void*" "GetLongArrayElements" } + { "void*" "GetFloatArrayElements" } + { "void*" "GetDoubleArrayElements" } + { "void*" "ReleaseBooleanArrayElements" } + { "void*" "ReleaseByteArrayElements" } + { "void*" "ReleaseCharArrayElements" } + { "void*" "ReleaseShortArrayElements" } + { "void*" "ReleaseIntArrayElements" } + { "void*" "ReleaseLongArrayElements" } + { "void*" "ReleaseFloatArrayElements" } + { "void*" "ReleaseDoubleArrayElements" } + { "void*" "GetBooleanArrayRegion" } + { "void*" "GetByteArrayRegion" } + { "void*" "GetCharArrayRegion" } + { "void*" "GetShortArrayRegion" } + { "void*" "GetIntArrayRegion" } + { "void*" "GetLongArrayRegion" } + { "void*" "GetFloatArrayRegion" } + { "void*" "GetDoubleArrayRegion" } + { "void*" "SetBooleanArrayRegion" } + { "void*" "SetByteArrayRegion" } + { "void*" "SetCharArrayRegion" } + { "void*" "SetShortArrayRegion" } + { "void*" "SetIntArrayRegion" } + { "void*" "SetLongArrayRegion" } + { "void*" "SetFloatArrayRegion" } + { "void*" "SetDoubleArrayRegion" } + { "void*" "RegisterNatives" } + { "void*" "UnregisterNatives" } + { "void*" "MonitorEnter" } + { "void*" "MonitorExit" } + { "void*" "GetJavaVM" } + { "void*" "GetStringRegion" } + { "void*" "GetStringUTFRegion" } + { "void*" "GetPrimitiveArrayCritical" } + { "void*" "ReleasePrimitiveArrayCritical" } + { "void*" "GetStringCritical" } + { "void*" "ReleaseStringCritical" } + { "void*" "NewWeakGlobalRef" } + { "void*" "DeleteWeakGlobalRef" } + { "void*" "ExceptionCheck" } + { "void*" "NewDirectByteBuffer" } + { "void*" "GetDirectBufferAddress" } + { "void*" "GetDirectBufferCapacity" } ; + +C-STRUCT: JNIEnv + { "JNINativeInterface*" "functions" } ; + +FUNCTION: jint JNI_GetDefaultJavaVMInitArgs ( jdk-init-args* args ) ; +FUNCTION: jint JNI_CreateJavaVM ( void** pvm, void** penv, void* args ) ; + +: ( -- jdk-init-args ) + "jdk-init-args" HEX: 00010004 over set-jdk-init-args-version ; + +: jni1 ( -- init-args int ) + dup JNI_GetDefaultJavaVMInitArgs ; + +: jni2 ( -- vm env int ) + f f [ + jni1 drop JNI_CreateJavaVM + ] 2keep rot dup 0 = [ + >r >r 0 swap void*-nth r> 0 swap void*-nth r> + ] when ; + +: (destroy-java-vm) + "int" { "void*" } "cdecl" alien-indirect ; + +: (attach-current-thread) + "int" { "void*" "void*" "void*" } "cdecl" alien-indirect ; + +: (detach-current-thread) + "int" { "void*" } "cdecl" alien-indirect ; + +: (get-env) + "int" { "void*" "void*" "int" } "cdecl" alien-indirect ; + +: (attach-current-thread-as-daemon) + "int" { "void*" "void*" "void*" } "cdecl" alien-indirect ; + +: destroy-java-vm ( javavm -- int ) + dup JavaVM-functions JNIInvokeInterface-DestroyJavaVM (destroy-java-vm) ; + +: (get-version) + "jint" { "JNIEnv*" } "cdecl" alien-indirect ; + +: get-version ( jnienv -- int ) + dup JNIEnv-functions JNINativeInterface-GetVersion (get-version) ; + +: (find-class) + "void*" { "JNINativeInterface*" "char*" } "cdecl" alien-indirect ; + +: find-class ( name jnienv -- int ) + dup swapd JNIEnv-functions JNINativeInterface-FindClass (find-class) ; + +: (get-static-field-id) + "void*" { "JNINativeInterface*" "void*" "char*" "char*" } "cdecl" alien-indirect ; + +: get-static-field-id ( class name sig jnienv -- int ) + dup >r >r 3array r> swap first3 r> JNIEnv-functions JNINativeInterface-GetStaticFieldID (get-static-field-id) ; + +: (get-static-object-field) + "void*" { "JNINativeInterface*" "void*" "void*" } "cdecl" alien-indirect ; + +: get-static-object-field ( class id jnienv -- int ) + dup >r >r 2array r> swap first2 r> JNIEnv-functions JNINativeInterface-GetStaticObjectField (get-static-object-field) ; + +: (get-method-id) + "void*" { "JNINativeInterface*" "void*" "char*" "char*" } "cdecl" alien-indirect ; + +: get-method-id ( class name sig jnienv -- int ) + dup >r >r 3array r> swap first3 r> JNIEnv-functions JNINativeInterface-GetMethodID (get-method-id) ; + +: (new-string) + "void*" { "JNINativeInterface*" "char*" "int" } "cdecl" alien-indirect ; + +: new-string ( str jnienv -- str ) + dup >r >r dup length 2array r> swap first2 r> JNIEnv-functions JNINativeInterface-NewString (new-string) ; + +: (call1) + "void" { "JNINativeInterface*" "void*" "void*" "int" } "cdecl" alien-indirect ; + +: call1 ( obj method-id jstr jnienv -- ) + dup >r >r 3array r> swap first3 r> JNIEnv-functions JNINativeInterface-CallObjectMethod (call1) ; + diff --git a/unmaintained/jni/jni.factor b/unmaintained/jni/jni.factor new file mode 100644 index 0000000000..86e1670c50 --- /dev/null +++ b/unmaintained/jni/jni.factor @@ -0,0 +1,22 @@ +! Copyright (C) 2006 Chris Double. +! See http://factorcode.org/license.txt for BSD license. +IN: jni +USING: kernel jni-internals namespaces ; + +! High level interface for JNI to be added here... + +: test0 ( -- ) + jni2 drop nip "env" set ; + +: test1 ( -- system ) + "java/lang/System" "env" get find-class ; + +: test2 ( system -- system.out ) + dup "out" "Ljava/io/PrintStream;" "env" get get-static-field-id + "env" get get-static-object-field ; + +: test3 ( int system.out -- ) + "java/io/PrintStream" "env" get find-class ! jstr out class + "println" "(I)V" "env" get get-method-id ! jstr out id + rot "env" get call1 ; + \ No newline at end of file diff --git a/unmaintained/jni/load.factor b/unmaintained/jni/load.factor new file mode 100644 index 0000000000..f5fd45c8d9 --- /dev/null +++ b/unmaintained/jni/load.factor @@ -0,0 +1,4 @@ +! Copyright (C) 2006 Chris Double. +! See http://factorcode.org/license.txt for BSD license. +PROVIDE: libs/jni +{ +files+ { "jni-internals.factor" "jni.factor" } } ; From 1b8943a8e0476d7f68542924b3a3079dc1d2d361 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Mon, 16 Jun 2008 17:39:14 +1200 Subject: [PATCH 02/47] Add failing peg.ebnf tests --- extra/peg/ebnf/ebnf-tests.factor | 33 ++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/extra/peg/ebnf/ebnf-tests.factor b/extra/peg/ebnf/ebnf-tests.factor index faaa63f4bd..425c05f391 100644 --- a/extra/peg/ebnf/ebnf-tests.factor +++ b/extra/peg/ebnf/ebnf-tests.factor @@ -365,3 +365,36 @@ main = Primary "ab c ab c" [EBNF a="a" "b" foo=(a "c")* EBNF] call ast>> ] unit-test +{ V{ "a" "a" "a" } } [ + "aaa" [EBNF a=('a')* b=!('b') a:x => [[ drop x ]] EBNF] call ast>> +] unit-test + +{ t } [ + "aaa" [EBNF a=('a')* b=!('b') a:x => [[ drop x ]] EBNF] call ast>> + "aaa" [EBNF a=('a')* b=!('b') (a):x => [[ drop x ]] EBNF] call ast>> = +] unit-test + +{ V{ "a" "a" "a" } } [ + "aaa" [EBNF a=('a')* b=a:x => [[ drop x ]] EBNF] call ast>> +] unit-test + +{ t } [ + "aaa" [EBNF a=('a')* b=a:x => [[ drop x ]] EBNF] call ast>> + "aaa" [EBNF a=('a')* b=(a):x => [[ drop x ]] EBNF] call ast>> = +] unit-test + +{ t } [ + "number=(digit)+:n 'a'" 'ebnf' parse remaining>> length zero? +] unit-test + +{ t } [ + "number=(digit)+ 'a'" 'ebnf' parse remaining>> length zero? +] unit-test + +{ t } [ + "number=digit+ 'a'" 'ebnf' parse remaining>> length zero? +] unit-test + +{ t } [ + "number=digit+:n 'a'" 'ebnf' parse remaining>> length zero? +] unit-test \ No newline at end of file From f1219c906aab7aa0d84b27694e998f2960c30775 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Tue, 17 Jun 2008 15:25:47 +1200 Subject: [PATCH 03/47] Check stack effect of actions in ebnf. Do implicit drop if needed --- extra/peg/ebnf/ebnf.factor | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/extra/peg/ebnf/ebnf.factor b/extra/peg/ebnf/ebnf.factor index fc10a65024..44765cc60c 100644 --- a/extra/peg/ebnf/ebnf.factor +++ b/extra/peg/ebnf/ebnf.factor @@ -2,8 +2,8 @@ ! See http://factorcode.org/license.txt for BSD license. USING: kernel compiler.units parser words arrays strings math.parser sequences quotations vectors namespaces math assocs continuations peg - peg.parsers unicode.categories multiline combinators.lib - splitting accessors effects sequences.deep peg.search ; + peg.parsers unicode.categories multiline combinators combinators.lib + splitting accessors effects sequences.deep peg.search inference ; IN: peg.ebnf TUPLE: ebnf-non-terminal symbol ; @@ -340,9 +340,16 @@ M: ebnf-var build-locals ( code ast -- ) M: object build-locals ( code ast -- ) drop ; +: check-action-effect ( quot -- quot ) + dup infer { + { [ dup (( a -- b )) effect<= ] [ drop ] } + { [ dup (( -- b )) effect<= ] [ drop [ drop ] prepose ] } + [ "Bad effect: " swap effect>string append throw ] + } cond ; + M: ebnf-action (transform) ( ast -- parser ) [ parser>> (transform) ] [ code>> ] [ parser>> ] tri build-locals - string-lines parse-lines action ; + string-lines parse-lines check-action-effect action ; M: ebnf-semantic (transform) ( ast -- parser ) [ parser>> (transform) ] [ code>> ] [ parser>> ] tri build-locals From dfa4926a84f5fbec165fe398fe7ed78e3666f298 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Tue, 17 Jun 2008 22:47:38 +1200 Subject: [PATCH 04/47] Print ebnf quotation on error. Fix generated local quotation --- extra/peg/ebnf/ebnf.factor | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/extra/peg/ebnf/ebnf.factor b/extra/peg/ebnf/ebnf.factor index 44765cc60c..335607b463 100644 --- a/extra/peg/ebnf/ebnf.factor +++ b/extra/peg/ebnf/ebnf.factor @@ -3,7 +3,8 @@ USING: kernel compiler.units parser words arrays strings math.parser sequences quotations vectors namespaces math assocs continuations peg peg.parsers unicode.categories multiline combinators combinators.lib - splitting accessors effects sequences.deep peg.search inference ; + splitting accessors effects sequences.deep peg.search inference + io.streams.string io prettyprint ; IN: peg.ebnf TUPLE: ebnf-non-terminal symbol ; @@ -324,7 +325,7 @@ M: ebnf-sequence build-locals ( code ast -- code ) ] 2each " | " % % - " ]" % + " nip ]" % ] "" make ] if ; @@ -334,7 +335,7 @@ M: ebnf-var build-locals ( code ast -- ) name>> % " [ dup ] " % " | " % % - " ]" % + " nip ]" % ] "" make ; M: object build-locals ( code ast -- ) @@ -344,7 +345,12 @@ M: object build-locals ( code ast -- ) dup infer { { [ dup (( a -- b )) effect<= ] [ drop ] } { [ dup (( -- b )) effect<= ] [ drop [ drop ] prepose ] } - [ "Bad effect: " swap effect>string append throw ] + [ + [ + "Bad effect: " write effect>string write + " for quotation " write pprint + ] with-string-writer throw + ] } cond ; M: ebnf-action (transform) ( ast -- parser ) From 479fa6a5b5a3ea5ebec18d4eba8ae30579531c60 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Wed, 18 Jun 2008 15:07:23 +1200 Subject: [PATCH 05/47] Add support for calling foreign peg.ebnf rules --- extra/peg/ebnf/ebnf.factor | 43 +++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/extra/peg/ebnf/ebnf.factor b/extra/peg/ebnf/ebnf.factor index 335607b463..4828ace9af 100644 --- a/extra/peg/ebnf/ebnf.factor +++ b/extra/peg/ebnf/ebnf.factor @@ -1,14 +1,19 @@ ! Copyright (C) 2007 Chris Double. ! See http://factorcode.org/license.txt for BSD license. -USING: kernel compiler.units parser words arrays strings math.parser sequences +USING: kernel compiler.units words arrays strings math.parser sequences quotations vectors namespaces math assocs continuations peg peg.parsers unicode.categories multiline combinators combinators.lib splitting accessors effects sequences.deep peg.search inference - io.streams.string io prettyprint ; + io.streams.string io prettyprint parser ; IN: peg.ebnf +: rule ( name word -- parser ) + #! Given an EBNF word produced from EBNF: return the EBNF rule + "ebnf-parser" word-prop at ; + TUPLE: ebnf-non-terminal symbol ; TUPLE: ebnf-terminal symbol ; +TUPLE: ebnf-foreign word rule ; TUPLE: ebnf-any-character ; TUPLE: ebnf-range pattern ; TUPLE: ebnf-ensure group ; @@ -27,6 +32,7 @@ TUPLE: ebnf rules ; C: ebnf-non-terminal C: ebnf-terminal +C: ebnf-foreign C: ebnf-any-character C: ebnf-range C: ebnf-ensure @@ -88,6 +94,8 @@ C: ebnf [ dup CHAR: ? = ] [ dup CHAR: : = ] [ dup CHAR: ~ = ] + [ dup CHAR: < = ] + [ dup CHAR: > = ] } 0|| not nip ] satisfy repeat1 [ >string ] action ; @@ -96,6 +104,24 @@ C: ebnf #! and it represents the literal value of the identifier. 'identifier' [ ] action ; +: 'foreign-name' ( -- parser ) + #! Parse a valid foreign parser name + [ + { + [ dup blank? ] + [ dup CHAR: > = ] + } 0|| not nip + ] satisfy repeat1 [ >string ] action ; + +: 'foreign' ( -- parser ) + #! A foreign call is a call to a rule in another ebnf grammar + [ + "" syntax , + ] seq* [ first2 ] action ; + : 'any-character' ( -- parser ) #! A parser to match the symbol for any character match. [ CHAR: . = ] satisfy [ drop ] action ; @@ -117,6 +143,7 @@ C: ebnf [ 'non-terminal' , 'terminal' , + 'foreign' , 'range-parser' , 'any-character' , ] choice* , @@ -367,6 +394,15 @@ M: ebnf-var (transform) ( ast -- parser ) M: ebnf-terminal (transform) ( ast -- parser ) symbol>> token ; +M: ebnf-foreign (transform) ( ast -- parser ) + dup word>> search + [ "Foreign word " swap word>> append " not found" append throw ] unless* + swap rule>> dup [ + swap rule + ] [ + execute + ] if ; + : parser-not-found ( name -- * ) [ "Parser " % % " not found." % @@ -411,6 +447,3 @@ M: ebnf-non-terminal (transform) ( ast -- parser ) ";EBNF" parse-multiline-string replace-escapes ebnf>quot swapd 1 1 define-declared "ebnf-parser" set-word-prop ; parsing -: rule ( name word -- parser ) - #! Given an EBNF word produced from EBNF: return the EBNF rule - "ebnf-parser" word-prop at ; \ No newline at end of file From 0841dbb4ad4567652a0acc53d3a0e1d4b24b4855 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Wed, 18 Jun 2008 15:21:10 +1200 Subject: [PATCH 06/47] Fix ebnf unit test --- extra/peg/ebnf/ebnf-tests.factor | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/extra/peg/ebnf/ebnf-tests.factor b/extra/peg/ebnf/ebnf-tests.factor index 425c05f391..04cc01c9d0 100644 --- a/extra/peg/ebnf/ebnf-tests.factor +++ b/extra/peg/ebnf/ebnf-tests.factor @@ -164,23 +164,23 @@ IN: peg.ebnf.tests ] unit-test { 6 } [ - "4+2" [EBNF num=[0-9] => [[ digit> ]] foo=num:x '+' num:y => [[ drop x y + ]] EBNF] call ast>> + "4+2" [EBNF num=[0-9] => [[ digit> ]] foo=num:x '+' num:y => [[ x y + ]] EBNF] call ast>> ] unit-test { 6 } [ - "4+2" [EBNF foo=[0-9]:x '+' [0-9]:y => [[ drop x digit> y digit> + ]] EBNF] call ast>> + "4+2" [EBNF foo=[0-9]:x '+' [0-9]:y => [[ x digit> y digit> + ]] EBNF] call ast>> ] unit-test { 10 } [ - { 1 2 3 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ drop x y + ]] | num EBNF] call ast>> + { 1 2 3 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF] call ast>> ] unit-test { f } [ - { "a" 2 3 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ drop x y + ]] | num EBNF] call + { "a" 2 3 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF] call ] unit-test { 3 } [ - { 1 2 "a" 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ drop x y + ]] | num EBNF] call ast>> + { 1 2 "a" 4 } [EBNF num=. ?[ number? ]? list=list:x num:y => [[ x y + ]] | num EBNF] call ast>> ] unit-test { f } [ @@ -251,7 +251,7 @@ IN: peg.ebnf.tests ] unit-test { t } [ - "abcd='9' | ('8'):x => [[ drop x ]]" 'ebnf' parse parse-result-remaining empty? + "abcd='9' | ('8'):x => [[ x ]]" 'ebnf' parse parse-result-remaining empty? ] unit-test EBNF: primary @@ -366,21 +366,21 @@ main = Primary ] unit-test { V{ "a" "a" "a" } } [ - "aaa" [EBNF a=('a')* b=!('b') a:x => [[ drop x ]] EBNF] call ast>> + "aaa" [EBNF a=('a')* b=!('b') a:x => [[ x ]] EBNF] call ast>> ] unit-test { t } [ - "aaa" [EBNF a=('a')* b=!('b') a:x => [[ drop x ]] EBNF] call ast>> - "aaa" [EBNF a=('a')* b=!('b') (a):x => [[ drop x ]] EBNF] call ast>> = + "aaa" [EBNF a=('a')* b=!('b') a:x => [[ x ]] EBNF] call ast>> + "aaa" [EBNF a=('a')* b=!('b') (a):x => [[ x ]] EBNF] call ast>> = ] unit-test { V{ "a" "a" "a" } } [ - "aaa" [EBNF a=('a')* b=a:x => [[ drop x ]] EBNF] call ast>> + "aaa" [EBNF a=('a')* b=a:x => [[ x ]] EBNF] call ast>> ] unit-test { t } [ - "aaa" [EBNF a=('a')* b=a:x => [[ drop x ]] EBNF] call ast>> - "aaa" [EBNF a=('a')* b=(a):x => [[ drop x ]] EBNF] call ast>> = + "aaa" [EBNF a=('a')* b=a:x => [[ x ]] EBNF] call ast>> + "aaa" [EBNF a=('a')* b=(a):x => [[ x ]] EBNF] call ast>> = ] unit-test { t } [ From f4f4ea7eb6fd5b78f635ecbc019649db1a1dd817 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Wed, 18 Jun 2008 17:34:21 +1200 Subject: [PATCH 07/47] Fix peg.ebnf unit test failures --- extra/peg/ebnf/ebnf.factor | 68 ++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 25 deletions(-) diff --git a/extra/peg/ebnf/ebnf.factor b/extra/peg/ebnf/ebnf.factor index 4828ace9af..215eabdd37 100644 --- a/extra/peg/ebnf/ebnf.factor +++ b/extra/peg/ebnf/ebnf.factor @@ -49,6 +49,10 @@ C: ebnf-var C: ebnf-semantic C: ebnf +: filter-hidden ( seq -- seq ) + #! Remove elements that produce no AST from sequence + [ ebnf-ensure-not? not ] filter [ ebnf-ensure? not ] filter ; + : syntax ( string -- parser ) #! Parses the string, ignoring white space, and #! does not put the result in the AST. @@ -140,12 +144,18 @@ C: ebnf #! The latter indicates that it is the beginning of a #! new rule. [ - [ - 'non-terminal' , - 'terminal' , - 'foreign' , - 'range-parser' , - 'any-character' , + [ + [ + 'non-terminal' , + 'terminal' , + 'foreign' , + 'range-parser' , + 'any-character' , + ] choice* + [ dup , "*" token hide , ] seq* [ first ] action , + [ dup , "+" token hide , ] seq* [ first ] action , + [ dup , "?[" token ensure-not , "?" token hide , ] seq* [ first ] action , + , ] choice* , [ "=" syntax ensure-not , @@ -153,6 +163,8 @@ C: ebnf ] choice* , ] seq* [ first ] action ; +DEFER: 'action' + : 'element' ( -- parser ) [ [ ('element') , ":" syntax , "a-zA-Z" range-pattern repeat1 [ >string ] action , ] seq* [ first2 ] action , @@ -256,7 +268,7 @@ DEFER: 'choice' ] choice* ; : 'choice' ( -- parser ) - 'actioned-sequence' sp "|" token sp list-of [ + 'actioned-sequence' sp repeat1 [ dup length 1 = [ first ] [ ] if ] action "|" token sp list-of [ dup length 1 = [ first ] [ ] if ] action ; @@ -337,23 +349,29 @@ M: ebnf-whitespace (transform) ( ast -- parser ) GENERIC: build-locals ( code ast -- code ) M: ebnf-sequence build-locals ( code ast -- code ) - elements>> dup [ ebnf-var? ] filter empty? [ - drop - ] [ - [ - "USING: locals sequences ; [let* | " % - dup length swap [ - dup ebnf-var? [ - name>> % - " [ " % # " over nth ] " % - ] [ - 2drop - ] if - ] 2each - " | " % - % - " nip ]" % - ] "" make + #! Note the need to filter out this ebnf items that + #! leave nothing in the AST + elements>> filter-hidden dup length 1 = [ + first build-locals + ] [ + dup [ ebnf-var? ] filter empty? [ + drop + ] [ + [ + "USING: locals sequences ; [let* | " % + dup length swap [ + dup ebnf-var? [ + name>> % + " [ " % # " over nth ] " % + ] [ + 2drop + ] if + ] 2each + " | " % + % + " nip ]" % + ] "" make + ] if ] if ; M: ebnf-var build-locals ( code ast -- ) @@ -381,7 +399,7 @@ M: object build-locals ( code ast -- ) } cond ; M: ebnf-action (transform) ( ast -- parser ) - [ parser>> (transform) ] [ code>> ] [ parser>> ] tri build-locals + [ parser>> (transform) ] [ code>> ] [ parser>> ] tri build-locals string-lines parse-lines check-action-effect action ; M: ebnf-semantic (transform) ( ast -- parser ) From b338fc8feaa04050cac2e10be76a4d7cf812c4b3 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Tue, 17 Jun 2008 21:55:53 +1200 Subject: [PATCH 08/47] Javascript parser --- extra/peg/javascript/javascript.factor | 247 +++++++++++++++++++++++++ 1 file changed, 247 insertions(+) create mode 100644 extra/peg/javascript/javascript.factor diff --git a/extra/peg/javascript/javascript.factor b/extra/peg/javascript/javascript.factor new file mode 100644 index 0000000000..33fd6dd069 --- /dev/null +++ b/extra/peg/javascript/javascript.factor @@ -0,0 +1,247 @@ +! Copyright (C) 2007 Chris Double. +! See http://factorcode.org/license.txt for BSD license. +USING: kernel arrays strings math.parser sequences sequences.deep +peg peg.ebnf peg.parsers memoize namespaces math ; +IN: peg.javascript + +#! Grammar for JavaScript. Based on OMeta-JS example from: +#! http://jarrett.cs.ucla.edu/ometa-js/#JavaScript_Compiler + +USE: prettyprint + +TUPLE: ast-keyword value ; +TUPLE: ast-name value ; +TUPLE: ast-number value ; +TUPLE: ast-string value ; +TUPLE: ast-cond-expr condition then else ; +TUPLE: ast-set lhs rhs ; +TUPLE: ast-get value ; +TUPLE: ast-mset lhs rhs operator ; +TUPLE: ast-binop lhs rhs operator ; +TUPLE: ast-unop expr operator ; +TUPLE: ast-postop expr operator ; +TUPLE: ast-preop expr operator ; +TUPLE: ast-getp index expr ; +TUPLE: ast-send method expr args ; +TUPLE: ast-call expr args ; +TUPLE: ast-this ; +TUPLE: ast-new name args ; +TUPLE: ast-array values ; +TUPLE: ast-json bindings ; +TUPLE: ast-binding name value ; +TUPLE: ast-func fs body ; +TUPLE: ast-var name value ; +TUPLE: ast-begin statements ; +TUPLE: ast-if condition true false ; +TUPLE: ast-while condition statements ; +TUPLE: ast-do-while statements condition ; +TUPLE: ast-for i c u statements ; +TUPLE: ast-for-in v e statements ; +TUPLE: ast-switch expr statements ; +TUPLE: ast-break ; +TUPLE: ast-continue ; +TUPLE: ast-throw e ; +TUPLE: ast-try t e c f ; +TUPLE: ast-return e ; +TUPLE: ast-case c cs ; +TUPLE: ast-default cs ; +C: ast-name +C: ast-keyword +C: ast-number +C: ast-string +C: ast-cond-expr +C: ast-set +C: ast-get +C: ast-mset +C: ast-binop +C: ast-unop +C: ast-preop +C: ast-postop +C: ast-getp +C: ast-send +C: ast-call +C: ast-this +C: ast-new +C: ast-array +C: ast-json +C: ast-binding +C: ast-func +C: ast-var +C: ast-begin +C: ast-if +C: ast-while +C: ast-do-while +C: ast-for +C: ast-for-in +C: ast-switch +C: ast-break +C: ast-continue +C: ast-throw +C: ast-try +C: ast-return +C: ast-case +C: ast-default + +EBNF: javascript +Letter = [a-zA-Z] +Digit = [0-9] +Digits = (Digit)+ +SingleLineComment = "//" (!("\n") .)* "\n" => [[ drop ignore ]] +MultiLineComment = "/*" (!("*/") .)* "*/" => [[ drop ignore ]] +Space = " " | "\t" | "\n" | SingleLineComment | MultiLineComment +Spaces = (Space)* => [[ drop ignore ]] +NameFirst = Letter | "$" | "_" +NameRest = NameFirst | Digit +iName = NameFirst (NameRest)* => [[ first2 swap prefix >string ]] +Keyword = ("break" + | "case" + | "catch" + | "continue" + | "default" + | "delete" + | "do" + | "else" + | "finally" + | "for" + | "function" + | "if" + | "in" + | "instanceof" + | "new" + | "return" + | "switch" + | "this" + | "throw" + | "try" + | "typeof" + | "var" + | "void" + | "while" + | "with") => [[ ]] +Name = !(Keyword) (iName):n => [[ drop n ]] +Number = Digits:ws '.' Digits:fs => [[ drop ws "." fs 3array concat >string string>number ]] + | Digits => [[ >string string>number ]] + +EscapeChar = "\\n" => [[ drop 10 ]] + | "\\r" => [[ drop 13 ]] + | "\\t" => [[ drop 9 ]] +StringChars1 = (EscapeChar | !('"""') .)* => [[ >string ]] +StringChars2 = (EscapeChar | !('"') .)* => [[ >string ]] +StringChars3 = (EscapeChar | !("'") .)* => [[ >string ]] +Str = '"""' StringChars1:cs '"""' => [[ drop cs ]] + | '"' StringChars2:cs '"' => [[ drop cs ]] + | "'" StringChars3:cs "'" => [[ drop cs ]] +Special = "(" | ")" | "{" | "}" | "[" | "]" | "," | ";" + | "?" | ":" | "!==" | "~=" | "===" | "==" | "=" | ">=" + | ">" | "<=" | "<" | "++" | "+=" | "+" | "--" | "-=" + | "-" | "*=" | "*" | "/=" | "/" | "%=" | "%" | "&&=" + | "&&" | "||=" | "||" | "." | "!" +Tok = Spaces (Name | Keyword | Number | Str | Special ) +Toks = (Tok)* Spaces +SpacesNoNl = (!("\n") Space)* => [[ drop ignore ]] + +Expr = OrExpr:e "?" Expr:t ":" Expr:f => [[ drop e t f ]] + | OrExpr:e "=" Expr:rhs => [[ drop e rhs ]] + | OrExpr:e "+=" Expr:rhs => [[ drop e rhs "+" ]] + | OrExpr:e "-=" Expr:rhs => [[ drop e rhs "-" ]] + | OrExpr:e "*=" Expr:rhs => [[ drop e rhs "*" ]] + | OrExpr:e "/=" Expr:rhs => [[ drop e rhs "/" ]] + | OrExpr:e "%=" Expr:rhs => [[ drop e rhs "%" ]] + | OrExpr:e "&&=" Expr:rhs => [[ drop e rhs "&&" ]] + | OrExpr:e "||=" Expr:rhs => [[ drop e rhs "||" ]] + | OrExpr:e => [[ drop e ]] + +OrExpr = OrExpr:x "||" AndExpr:y => [[ drop x y "||" ]] + | AndExpr +AndExpr = AndExpr:x "&&" EqExpr:y => [[ drop x y "&&" ]] + | EqExpr +EqExpr = EqExpr:x "==" RelExpr:y => [[ drop x y "==" ]] + | EqExpr:x "!=" RelExpr:y => [[ drop x y "!=" ]] + | EqExpr:x "===" RelExpr:y => [[ drop x y "===" ]] + | EqExpr:x "!==" RelExpr:y => [[ drop x y "!==" ]] + | RelExpr +RelExpr = RelExpr:x ">" AddExpr:y => [[ drop x y ">" ]] + | RelExpr:x ">=" AddExpr:y => [[ drop x y ">=" ]] + | RelExpr:x "<" AddExpr:y => [[ drop x y "<" ]] + | RelExpr:x "<=" AddExpr:y => [[ drop x y "<=" ]] + | RelExpr:x "instanceof" AddExpr:y => [[ drop x y "instanceof" ]] + | AddExpr +AddExpr = AddExpr:x "+" MulExpr:y => [[ drop x y "+" ]] + | AddExpr:x "-" MulExpr:y => [[ drop x y "-" ]] + | MulExpr +MulExpr = MulExpr:x "*" MulExpr:y => [[ drop x y "*" ]] + | MulExpr:x "/" MulExpr:y => [[ drop x y "/" ]] + | MulExpr:x "%" MulExpr:y => [[ drop x y "%" ]] + | Unary +Unary = "-" Postfix:p => [[ drop p "-" ]] + | "+" Postfix:p => [[ drop p ]] + | "++" Postfix:p => [[ drop p "++" ]] + | "--" Postfix:p => [[ drop p "--" ]] + | "!" Postfix:p => [[ drop p "!" ]] + | Postfix +Postfix = PrimExpr:p SpacesNoNl "++" => [[ drop p "++" ]] + | PrimExpr:p SpacesNoNl "--" => [[ drop p "--" ]] + | PrimExpr +Args = Expr ("," Expr)* => [[ first2 swap prefix ]] +PrimExpr = PrimExpr:p "[" Expr:i "]" => [[ drop i p ]] + | PrimExpr:p "." Name:m "(" Args:as ")" => [[ drop m p as ]] + | PrimExpr:p "." Name:f => [[ drop f p ]] + | PrimExpr:p "(" Args:as ")" => [[ drop p as ]] + | PrimExprHd +PrimExprHd = "(" Expr:e ")" => [[ drop e ]] + | "this" => [[ drop ]] + | Name => [[ ]] + | Number => [[ ]] + | Str => [[ ]] + | "function" FuncRest:fr => [[ drop fr ]] + | "new" Name:n "(" Args:as ")" => [[ drop n as ]] + | "[" Args:es "]" => [[ drop es ]] + | Json +JsonBindings = JsonBinding ("," JsonBinding)* => [[ first2 swap prefix ]] +Json = "{" JsonBindings:bs "}" => [[ drop bs ]] +JsonBinding = JsonPropName:n ":" Expr:v => [[ drop n v ]] +JsonPropName = Name | Number | Str +Formal = Spaces Name +Formals = Formal ("," Formal)* => [[ first2 swap prefix ]] +FuncRest = "(" Formals:fs ")" "{" SrcElems:body "}" => [[ drop fs body ]] +Sc = SpacesNoNl ("\n" | "}")| ";" +Binding = Name:n "=" Expr:v => [[ drop n v ]] + | Name:n => [[ drop n "undefined" ]] +Block = "{" SrcElems:ss "}" => [[ drop ss ]] +Bindings = Binding ("," Binding)* => [[ first2 swap prefix ]] +For1 = "var" Binding => [[ second ]] + | Expr + | Spaces => [[ "undefined" ]] +For2 = Expr + | Spaces => [[ "true" ]] +For3 = Expr + | Spaces => [[ "undefined" ]] +ForIn1 = "var" Name:n => [[ drop n "undefined" ]] + | Expr +Switch1 = "case" Expr:c ":" SrcElems:cs => [[ drop c cs ]] + | "default" ":" SrcElems:cs => [[ drop cs ]] +SwitchBody = (Switch1)* +Finally = "finally" Block:b => [[ drop b ]] + | Spaces => [[ drop "undefined" ]] +Stmt = Block + | "var" Bindings:bs Sc => [[ drop bs ]] + | "if" "(" Expr:c ")" Stmt:t "else" Stmt:f => [[ drop c t f ]] + | "if" "(" Expr:c ")" Stmt:t => [[ drop c t "undefined" ]] + | "while" "(" Expr:c ")" Stmt:s => [[ drop c s ]] + | "do" Stmt:s "while" "(" Expr:c ")" Sc => [[ drop s c ]] + | "for" "(" For1:i ";" For2:c ";" For3:u ")" Stmt:s => [[ drop i c u s ]] + | "for" "(" ForIn1:v "in" Expr:e ")" Stmt:s => [[ drop v e s ]] + | "switch" "(" Expr:e ")" "{" SwitchBody:cs "}" => [[ drop e cs ]] + | "break" Sc => [[ drop ]] + | "continue" Sc => [[ drop ]] + | "throw" SpacesNoNl Expr:e Sc => [[ drop e ]] + | "try" Block:t "catch" "(" Name:e ")" Block:c Finally:f => [[ drop t e c f ]] + | "return" Expr:e Sc => [[ drop e ]] + | "return" Sc => [[ drop "undefined" ]] + | Expr:e Sc => [[ drop e ]] + | ";" => [[ drop "undefined" ]] +SrcElem = "function" Name:n FuncRest:f => [[ drop n f ]] + | Stmt +SrcElems = (SrcElem)* => [[ ]] +TopLevel = SrcElems Spaces +;EBNF \ No newline at end of file From 79dfe2806a873b9bef9f405bb8f222eab4b86f50 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Tue, 17 Jun 2008 22:07:57 +1200 Subject: [PATCH 09/47] Remove javascript boa constructors --- extra/peg/javascript/javascript.factor | 190 ++++++++++--------------- 1 file changed, 77 insertions(+), 113 deletions(-) diff --git a/extra/peg/javascript/javascript.factor b/extra/peg/javascript/javascript.factor index 33fd6dd069..5c76c45f4c 100644 --- a/extra/peg/javascript/javascript.factor +++ b/extra/peg/javascript/javascript.factor @@ -45,42 +45,6 @@ TUPLE: ast-try t e c f ; TUPLE: ast-return e ; TUPLE: ast-case c cs ; TUPLE: ast-default cs ; -C: ast-name -C: ast-keyword -C: ast-number -C: ast-string -C: ast-cond-expr -C: ast-set -C: ast-get -C: ast-mset -C: ast-binop -C: ast-unop -C: ast-preop -C: ast-postop -C: ast-getp -C: ast-send -C: ast-call -C: ast-this -C: ast-new -C: ast-array -C: ast-json -C: ast-binding -C: ast-func -C: ast-var -C: ast-begin -C: ast-if -C: ast-while -C: ast-do-while -C: ast-for -C: ast-for-in -C: ast-switch -C: ast-break -C: ast-continue -C: ast-throw -C: ast-try -C: ast-return -C: ast-case -C: ast-default EBNF: javascript Letter = [a-zA-Z] @@ -117,10 +81,10 @@ Keyword = ("break" | "var" | "void" | "while" - | "with") => [[ ]] -Name = !(Keyword) (iName):n => [[ drop n ]] -Number = Digits:ws '.' Digits:fs => [[ drop ws "." fs 3array concat >string string>number ]] - | Digits => [[ >string string>number ]] + | "with") => [[ ast-keyword boa ]] +Name = !(Keyword) (iName):n => [[ drop n ast-name boa ]] +Number = Digits:ws '.' Digits:fs => [[ drop ws "." fs 3array concat >string string>number ast-number boa ]] + | Digits => [[ >string string>number ast-number boa ]] EscapeChar = "\\n" => [[ drop 10 ]] | "\\r" => [[ drop 13 ]] @@ -128,9 +92,9 @@ EscapeChar = "\\n" => [[ drop 10 ]] StringChars1 = (EscapeChar | !('"""') .)* => [[ >string ]] StringChars2 = (EscapeChar | !('"') .)* => [[ >string ]] StringChars3 = (EscapeChar | !("'") .)* => [[ >string ]] -Str = '"""' StringChars1:cs '"""' => [[ drop cs ]] - | '"' StringChars2:cs '"' => [[ drop cs ]] - | "'" StringChars3:cs "'" => [[ drop cs ]] +Str = '"""' StringChars1:cs '"""' => [[ drop cs ast-string boa ]] + | '"' StringChars2:cs '"' => [[ drop cs ast-string boa ]] + | "'" StringChars3:cs "'" => [[ drop cs ast-string boa ]] Special = "(" | ")" | "{" | "}" | "[" | "]" | "," | ";" | "?" | ":" | "!==" | "~=" | "===" | "==" | "=" | ">=" | ">" | "<=" | "<" | "++" | "+=" | "+" | "--" | "-=" @@ -140,108 +104,108 @@ Tok = Spaces (Name | Keyword | Number | Str | Special ) Toks = (Tok)* Spaces SpacesNoNl = (!("\n") Space)* => [[ drop ignore ]] -Expr = OrExpr:e "?" Expr:t ":" Expr:f => [[ drop e t f ]] - | OrExpr:e "=" Expr:rhs => [[ drop e rhs ]] - | OrExpr:e "+=" Expr:rhs => [[ drop e rhs "+" ]] - | OrExpr:e "-=" Expr:rhs => [[ drop e rhs "-" ]] - | OrExpr:e "*=" Expr:rhs => [[ drop e rhs "*" ]] - | OrExpr:e "/=" Expr:rhs => [[ drop e rhs "/" ]] - | OrExpr:e "%=" Expr:rhs => [[ drop e rhs "%" ]] - | OrExpr:e "&&=" Expr:rhs => [[ drop e rhs "&&" ]] - | OrExpr:e "||=" Expr:rhs => [[ drop e rhs "||" ]] +Expr = OrExpr:e "?" Expr:t ":" Expr:f => [[ drop e t f ast-cond-expr boa ]] + | OrExpr:e "=" Expr:rhs => [[ drop e rhs ast-set boa ]] + | OrExpr:e "+=" Expr:rhs => [[ drop e rhs "+" ast-mset boa ]] + | OrExpr:e "-=" Expr:rhs => [[ drop e rhs "-" ast-mset boa ]] + | OrExpr:e "*=" Expr:rhs => [[ drop e rhs "*" ast-mset boa ]] + | OrExpr:e "/=" Expr:rhs => [[ drop e rhs "/" ast-mset boa ]] + | OrExpr:e "%=" Expr:rhs => [[ drop e rhs "%" ast-mset boa ]] + | OrExpr:e "&&=" Expr:rhs => [[ drop e rhs "&&" ast-mset boa ]] + | OrExpr:e "||=" Expr:rhs => [[ drop e rhs "||" ast-mset boa ]] | OrExpr:e => [[ drop e ]] -OrExpr = OrExpr:x "||" AndExpr:y => [[ drop x y "||" ]] +OrExpr = OrExpr:x "||" AndExpr:y => [[ drop x y "||" ast-binop boa ]] | AndExpr -AndExpr = AndExpr:x "&&" EqExpr:y => [[ drop x y "&&" ]] +AndExpr = AndExpr:x "&&" EqExpr:y => [[ drop x y "&&" ast-binop boa ]] | EqExpr -EqExpr = EqExpr:x "==" RelExpr:y => [[ drop x y "==" ]] - | EqExpr:x "!=" RelExpr:y => [[ drop x y "!=" ]] - | EqExpr:x "===" RelExpr:y => [[ drop x y "===" ]] - | EqExpr:x "!==" RelExpr:y => [[ drop x y "!==" ]] +EqExpr = EqExpr:x "==" RelExpr:y => [[ drop x y "==" ast-binop boa ]] + | EqExpr:x "!=" RelExpr:y => [[ drop x y "!=" ast-binop boa ]] + | EqExpr:x "===" RelExpr:y => [[ drop x y "===" ast-binop boa ]] + | EqExpr:x "!==" RelExpr:y => [[ drop x y "!==" ast-binop boa ]] | RelExpr -RelExpr = RelExpr:x ">" AddExpr:y => [[ drop x y ">" ]] - | RelExpr:x ">=" AddExpr:y => [[ drop x y ">=" ]] - | RelExpr:x "<" AddExpr:y => [[ drop x y "<" ]] - | RelExpr:x "<=" AddExpr:y => [[ drop x y "<=" ]] - | RelExpr:x "instanceof" AddExpr:y => [[ drop x y "instanceof" ]] +RelExpr = RelExpr:x ">" AddExpr:y => [[ drop x y ">" ast-binop boa ]] + | RelExpr:x ">=" AddExpr:y => [[ drop x y ">=" ast-binop boa ]] + | RelExpr:x "<" AddExpr:y => [[ drop x y "<" ast-binop boa ]] + | RelExpr:x "<=" AddExpr:y => [[ drop x y "<=" ast-binop boa ]] + | RelExpr:x "instanceof" AddExpr:y => [[ drop x y "instanceof" ast-binop boa ]] | AddExpr -AddExpr = AddExpr:x "+" MulExpr:y => [[ drop x y "+" ]] - | AddExpr:x "-" MulExpr:y => [[ drop x y "-" ]] +AddExpr = AddExpr:x "+" MulExpr:y => [[ drop x y "+" ast-binop boa ]] + | AddExpr:x "-" MulExpr:y => [[ drop x y "-" ast-binop boa ]] | MulExpr -MulExpr = MulExpr:x "*" MulExpr:y => [[ drop x y "*" ]] - | MulExpr:x "/" MulExpr:y => [[ drop x y "/" ]] - | MulExpr:x "%" MulExpr:y => [[ drop x y "%" ]] +MulExpr = MulExpr:x "*" MulExpr:y => [[ drop x y "*" ast-binop boa ]] + | MulExpr:x "/" MulExpr:y => [[ drop x y "/" ast-binop boa ]] + | MulExpr:x "%" MulExpr:y => [[ drop x y "%" ast-binop boa ]] | Unary -Unary = "-" Postfix:p => [[ drop p "-" ]] +Unary = "-" Postfix:p => [[ drop p "-" ast-unop boa ]] | "+" Postfix:p => [[ drop p ]] - | "++" Postfix:p => [[ drop p "++" ]] - | "--" Postfix:p => [[ drop p "--" ]] - | "!" Postfix:p => [[ drop p "!" ]] + | "++" Postfix:p => [[ drop p "++" ast-preop boa ]] + | "--" Postfix:p => [[ drop p "--" ast-preop boa ]] + | "!" Postfix:p => [[ drop p "!" ast-unop boa ]] | Postfix -Postfix = PrimExpr:p SpacesNoNl "++" => [[ drop p "++" ]] - | PrimExpr:p SpacesNoNl "--" => [[ drop p "--" ]] +Postfix = PrimExpr:p SpacesNoNl "++" => [[ drop p "++" ast-postop boa ]] + | PrimExpr:p SpacesNoNl "--" => [[ drop p "--" ast-postop boa ]] | PrimExpr Args = Expr ("," Expr)* => [[ first2 swap prefix ]] -PrimExpr = PrimExpr:p "[" Expr:i "]" => [[ drop i p ]] - | PrimExpr:p "." Name:m "(" Args:as ")" => [[ drop m p as ]] - | PrimExpr:p "." Name:f => [[ drop f p ]] - | PrimExpr:p "(" Args:as ")" => [[ drop p as ]] +PrimExpr = PrimExpr:p "[" Expr:i "]" => [[ drop i p ast-getp boa ]] + | PrimExpr:p "." Name:m "(" Args:as ")" => [[ drop m p as ast-send boa ]] + | PrimExpr:p "." Name:f => [[ drop f p ast-getp boa ]] + | PrimExpr:p "(" Args:as ")" => [[ drop p as ast-call boa ]] | PrimExprHd PrimExprHd = "(" Expr:e ")" => [[ drop e ]] - | "this" => [[ drop ]] - | Name => [[ ]] - | Number => [[ ]] - | Str => [[ ]] + | "this" => [[ drop ast-this boa ]] + | Name => [[ ast-get boa ]] + | Number => [[ ast-number boa ]] + | Str => [[ ast-string boa ]] | "function" FuncRest:fr => [[ drop fr ]] - | "new" Name:n "(" Args:as ")" => [[ drop n as ]] - | "[" Args:es "]" => [[ drop es ]] + | "new" Name:n "(" Args:as ")" => [[ drop n as ast-new boa ]] + | "[" Args:es "]" => [[ drop es ast-array boa ]] | Json JsonBindings = JsonBinding ("," JsonBinding)* => [[ first2 swap prefix ]] -Json = "{" JsonBindings:bs "}" => [[ drop bs ]] -JsonBinding = JsonPropName:n ":" Expr:v => [[ drop n v ]] +Json = "{" JsonBindings:bs "}" => [[ drop bs ast-json boa ]] +JsonBinding = JsonPropName:n ":" Expr:v => [[ drop n v ast-binding boa ]] JsonPropName = Name | Number | Str Formal = Spaces Name Formals = Formal ("," Formal)* => [[ first2 swap prefix ]] -FuncRest = "(" Formals:fs ")" "{" SrcElems:body "}" => [[ drop fs body ]] +FuncRest = "(" Formals:fs ")" "{" SrcElems:body "}" => [[ drop fs body ast-func boa ]] Sc = SpacesNoNl ("\n" | "}")| ";" -Binding = Name:n "=" Expr:v => [[ drop n v ]] - | Name:n => [[ drop n "undefined" ]] +Binding = Name:n "=" Expr:v => [[ drop n v ast-var boa ]] + | Name:n => [[ drop n "undefined" ast-get boa ast-var boa ]] Block = "{" SrcElems:ss "}" => [[ drop ss ]] Bindings = Binding ("," Binding)* => [[ first2 swap prefix ]] For1 = "var" Binding => [[ second ]] | Expr - | Spaces => [[ "undefined" ]] + | Spaces => [[ "undefined" ast-get boa ]] For2 = Expr - | Spaces => [[ "true" ]] + | Spaces => [[ "true" ast-get boa ]] For3 = Expr - | Spaces => [[ "undefined" ]] -ForIn1 = "var" Name:n => [[ drop n "undefined" ]] + | Spaces => [[ "undefined" ast-get boa ]] +ForIn1 = "var" Name:n => [[ drop n "undefined" ast-get boa ast-var boa ]] | Expr -Switch1 = "case" Expr:c ":" SrcElems:cs => [[ drop c cs ]] - | "default" ":" SrcElems:cs => [[ drop cs ]] +Switch1 = "case" Expr:c ":" SrcElems:cs => [[ drop c cs ast-case boa ]] + | "default" ":" SrcElems:cs => [[ drop cs ast-default boa ]] SwitchBody = (Switch1)* Finally = "finally" Block:b => [[ drop b ]] - | Spaces => [[ drop "undefined" ]] + | Spaces => [[ drop "undefined" ast-get boa ]] Stmt = Block - | "var" Bindings:bs Sc => [[ drop bs ]] - | "if" "(" Expr:c ")" Stmt:t "else" Stmt:f => [[ drop c t f ]] - | "if" "(" Expr:c ")" Stmt:t => [[ drop c t "undefined" ]] - | "while" "(" Expr:c ")" Stmt:s => [[ drop c s ]] - | "do" Stmt:s "while" "(" Expr:c ")" Sc => [[ drop s c ]] - | "for" "(" For1:i ";" For2:c ";" For3:u ")" Stmt:s => [[ drop i c u s ]] - | "for" "(" ForIn1:v "in" Expr:e ")" Stmt:s => [[ drop v e s ]] - | "switch" "(" Expr:e ")" "{" SwitchBody:cs "}" => [[ drop e cs ]] - | "break" Sc => [[ drop ]] - | "continue" Sc => [[ drop ]] - | "throw" SpacesNoNl Expr:e Sc => [[ drop e ]] - | "try" Block:t "catch" "(" Name:e ")" Block:c Finally:f => [[ drop t e c f ]] - | "return" Expr:e Sc => [[ drop e ]] - | "return" Sc => [[ drop "undefined" ]] + | "var" Bindings:bs Sc => [[ drop bs ast-begin boa ]] + | "if" "(" Expr:c ")" Stmt:t "else" Stmt:f => [[ drop c t f ast-if boa ]] + | "if" "(" Expr:c ")" Stmt:t => [[ drop c t "undefined" ast-get boa ast-if boa ]] + | "while" "(" Expr:c ")" Stmt:s => [[ drop c s ast-while boa ]] + | "do" Stmt:s "while" "(" Expr:c ")" Sc => [[ drop s c ast-do-while boa ]] + | "for" "(" For1:i ";" For2:c ";" For3:u ")" Stmt:s => [[ drop i c u s ast-for boa ]] + | "for" "(" ForIn1:v "in" Expr:e ")" Stmt:s => [[ drop v e s ast-for-in boa ]] + | "switch" "(" Expr:e ")" "{" SwitchBody:cs "}" => [[ drop e cs ast-switch boa ]] + | "break" Sc => [[ drop ast-break boa ]] + | "continue" Sc => [[ drop ast-continue boa ]] + | "throw" SpacesNoNl Expr:e Sc => [[ drop e ast-throw boa ]] + | "try" Block:t "catch" "(" Name:e ")" Block:c Finally:f => [[ drop t e c f ast-try boa ]] + | "return" Expr:e Sc => [[ drop e ast-return boa ]] + | "return" Sc => [[ drop "undefined" ast-get boa ast-return boa ]] | Expr:e Sc => [[ drop e ]] - | ";" => [[ drop "undefined" ]] -SrcElem = "function" Name:n FuncRest:f => [[ drop n f ]] + | ";" => [[ drop "undefined" ast-get boa ]] +SrcElem = "function" Name:n FuncRest:f => [[ drop n f ast-var boa ]] | Stmt -SrcElems = (SrcElem)* => [[ ]] +SrcElems = (SrcElem)* => [[ ast-begin boa ]] TopLevel = SrcElems Spaces ;EBNF \ No newline at end of file From 55216a990dc5eaf37dee3345426454876994e70a Mon Sep 17 00:00:00 2001 From: Chris Double Date: Tue, 17 Jun 2008 22:47:05 +1200 Subject: [PATCH 10/47] Remove drop from actions --- extra/peg/javascript/javascript.factor | 164 ++++++++++++------------- 1 file changed, 82 insertions(+), 82 deletions(-) diff --git a/extra/peg/javascript/javascript.factor b/extra/peg/javascript/javascript.factor index 5c76c45f4c..54b9d8aa0a 100644 --- a/extra/peg/javascript/javascript.factor +++ b/extra/peg/javascript/javascript.factor @@ -50,10 +50,10 @@ EBNF: javascript Letter = [a-zA-Z] Digit = [0-9] Digits = (Digit)+ -SingleLineComment = "//" (!("\n") .)* "\n" => [[ drop ignore ]] -MultiLineComment = "/*" (!("*/") .)* "*/" => [[ drop ignore ]] +SingleLineComment = "//" (!("\n") .)* "\n" => [[ ignore ]] +MultiLineComment = "/*" (!("*/") .)* "*/" => [[ ignore ]] Space = " " | "\t" | "\n" | SingleLineComment | MultiLineComment -Spaces = (Space)* => [[ drop ignore ]] +Spaces = (Space)* => [[ ignore ]] NameFirst = Letter | "$" | "_" NameRest = NameFirst | Digit iName = NameFirst (NameRest)* => [[ first2 swap prefix >string ]] @@ -82,19 +82,19 @@ Keyword = ("break" | "void" | "while" | "with") => [[ ast-keyword boa ]] -Name = !(Keyword) (iName):n => [[ drop n ast-name boa ]] -Number = Digits:ws '.' Digits:fs => [[ drop ws "." fs 3array concat >string string>number ast-number boa ]] +Name = !(Keyword) (iName):n => [[ n ast-name boa ]] +Number = Digits:ws '.' Digits:fs => [[ ws "." fs 3array concat >string string>number ast-number boa ]] | Digits => [[ >string string>number ast-number boa ]] -EscapeChar = "\\n" => [[ drop 10 ]] - | "\\r" => [[ drop 13 ]] - | "\\t" => [[ drop 9 ]] +EscapeChar = "\\n" => [[ 10 ]] + | "\\r" => [[ 13 ]] + | "\\t" => [[ 9 ]] StringChars1 = (EscapeChar | !('"""') .)* => [[ >string ]] StringChars2 = (EscapeChar | !('"') .)* => [[ >string ]] StringChars3 = (EscapeChar | !("'") .)* => [[ >string ]] -Str = '"""' StringChars1:cs '"""' => [[ drop cs ast-string boa ]] - | '"' StringChars2:cs '"' => [[ drop cs ast-string boa ]] - | "'" StringChars3:cs "'" => [[ drop cs ast-string boa ]] +Str = '"""' StringChars1:cs '"""' => [[ cs ast-string boa ]] + | '"' StringChars2:cs '"' => [[ cs ast-string boa ]] + | "'" StringChars3:cs "'" => [[ cs ast-string boa ]] Special = "(" | ")" | "{" | "}" | "[" | "]" | "," | ";" | "?" | ":" | "!==" | "~=" | "===" | "==" | "=" | ">=" | ">" | "<=" | "<" | "++" | "+=" | "+" | "--" | "-=" @@ -102,76 +102,76 @@ Special = "(" | ")" | "{" | "}" | "[" | "]" | "," | " | "&&" | "||=" | "||" | "." | "!" Tok = Spaces (Name | Keyword | Number | Str | Special ) Toks = (Tok)* Spaces -SpacesNoNl = (!("\n") Space)* => [[ drop ignore ]] +SpacesNoNl = (!("\n") Space)* => [[ ignore ]] -Expr = OrExpr:e "?" Expr:t ":" Expr:f => [[ drop e t f ast-cond-expr boa ]] - | OrExpr:e "=" Expr:rhs => [[ drop e rhs ast-set boa ]] - | OrExpr:e "+=" Expr:rhs => [[ drop e rhs "+" ast-mset boa ]] - | OrExpr:e "-=" Expr:rhs => [[ drop e rhs "-" ast-mset boa ]] - | OrExpr:e "*=" Expr:rhs => [[ drop e rhs "*" ast-mset boa ]] - | OrExpr:e "/=" Expr:rhs => [[ drop e rhs "/" ast-mset boa ]] - | OrExpr:e "%=" Expr:rhs => [[ drop e rhs "%" ast-mset boa ]] - | OrExpr:e "&&=" Expr:rhs => [[ drop e rhs "&&" ast-mset boa ]] - | OrExpr:e "||=" Expr:rhs => [[ drop e rhs "||" ast-mset boa ]] - | OrExpr:e => [[ drop e ]] +Expr = OrExpr:e "?" Expr:t ":" Expr:f => [[ e t f ast-cond-expr boa ]] + | OrExpr:e "=" Expr:rhs => [[ e rhs ast-set boa ]] + | OrExpr:e "+=" Expr:rhs => [[ e rhs "+" ast-mset boa ]] + | OrExpr:e "-=" Expr:rhs => [[ e rhs "-" ast-mset boa ]] + | OrExpr:e "*=" Expr:rhs => [[ e rhs "*" ast-mset boa ]] + | OrExpr:e "/=" Expr:rhs => [[ e rhs "/" ast-mset boa ]] + | OrExpr:e "%=" Expr:rhs => [[ e rhs "%" ast-mset boa ]] + | OrExpr:e "&&=" Expr:rhs => [[ e rhs "&&" ast-mset boa ]] + | OrExpr:e "||=" Expr:rhs => [[ e rhs "||" ast-mset boa ]] + | OrExpr:e => [[ e ]] -OrExpr = OrExpr:x "||" AndExpr:y => [[ drop x y "||" ast-binop boa ]] +OrExpr = OrExpr:x "||" AndExpr:y => [[ x y "||" ast-binop boa ]] | AndExpr -AndExpr = AndExpr:x "&&" EqExpr:y => [[ drop x y "&&" ast-binop boa ]] +AndExpr = AndExpr:x "&&" EqExpr:y => [[ x y "&&" ast-binop boa ]] | EqExpr -EqExpr = EqExpr:x "==" RelExpr:y => [[ drop x y "==" ast-binop boa ]] - | EqExpr:x "!=" RelExpr:y => [[ drop x y "!=" ast-binop boa ]] - | EqExpr:x "===" RelExpr:y => [[ drop x y "===" ast-binop boa ]] - | EqExpr:x "!==" RelExpr:y => [[ drop x y "!==" ast-binop boa ]] +EqExpr = EqExpr:x "==" RelExpr:y => [[ x y "==" ast-binop boa ]] + | EqExpr:x "!=" RelExpr:y => [[ x y "!=" ast-binop boa ]] + | EqExpr:x "===" RelExpr:y => [[ x y "===" ast-binop boa ]] + | EqExpr:x "!==" RelExpr:y => [[ x y "!==" ast-binop boa ]] | RelExpr -RelExpr = RelExpr:x ">" AddExpr:y => [[ drop x y ">" ast-binop boa ]] - | RelExpr:x ">=" AddExpr:y => [[ drop x y ">=" ast-binop boa ]] - | RelExpr:x "<" AddExpr:y => [[ drop x y "<" ast-binop boa ]] - | RelExpr:x "<=" AddExpr:y => [[ drop x y "<=" ast-binop boa ]] - | RelExpr:x "instanceof" AddExpr:y => [[ drop x y "instanceof" ast-binop boa ]] +RelExpr = RelExpr:x ">" AddExpr:y => [[ x y ">" ast-binop boa ]] + | RelExpr:x ">=" AddExpr:y => [[ x y ">=" ast-binop boa ]] + | RelExpr:x "<" AddExpr:y => [[ x y "<" ast-binop boa ]] + | RelExpr:x "<=" AddExpr:y => [[ x y "<=" ast-binop boa ]] + | RelExpr:x "instanceof" AddExpr:y => [[ x y "instanceof" ast-binop boa ]] | AddExpr -AddExpr = AddExpr:x "+" MulExpr:y => [[ drop x y "+" ast-binop boa ]] - | AddExpr:x "-" MulExpr:y => [[ drop x y "-" ast-binop boa ]] +AddExpr = AddExpr:x "+" MulExpr:y => [[ x y "+" ast-binop boa ]] + | AddExpr:x "-" MulExpr:y => [[ x y "-" ast-binop boa ]] | MulExpr -MulExpr = MulExpr:x "*" MulExpr:y => [[ drop x y "*" ast-binop boa ]] - | MulExpr:x "/" MulExpr:y => [[ drop x y "/" ast-binop boa ]] - | MulExpr:x "%" MulExpr:y => [[ drop x y "%" ast-binop boa ]] +MulExpr = MulExpr:x "*" MulExpr:y => [[ x y "*" ast-binop boa ]] + | MulExpr:x "/" MulExpr:y => [[ x y "/" ast-binop boa ]] + | MulExpr:x "%" MulExpr:y => [[ x y "%" ast-binop boa ]] | Unary -Unary = "-" Postfix:p => [[ drop p "-" ast-unop boa ]] - | "+" Postfix:p => [[ drop p ]] - | "++" Postfix:p => [[ drop p "++" ast-preop boa ]] - | "--" Postfix:p => [[ drop p "--" ast-preop boa ]] - | "!" Postfix:p => [[ drop p "!" ast-unop boa ]] +Unary = "-" Postfix:p => [[ p "-" ast-unop boa ]] + | "+" Postfix:p => [[ p ]] + | "++" Postfix:p => [[ p "++" ast-preop boa ]] + | "--" Postfix:p => [[ p "--" ast-preop boa ]] + | "!" Postfix:p => [[ p "!" ast-unop boa ]] | Postfix -Postfix = PrimExpr:p SpacesNoNl "++" => [[ drop p "++" ast-postop boa ]] - | PrimExpr:p SpacesNoNl "--" => [[ drop p "--" ast-postop boa ]] +Postfix = PrimExpr:p SpacesNoNl "++" => [[ p "++" ast-postop boa ]] + | PrimExpr:p SpacesNoNl "--" => [[ p "--" ast-postop boa ]] | PrimExpr Args = Expr ("," Expr)* => [[ first2 swap prefix ]] -PrimExpr = PrimExpr:p "[" Expr:i "]" => [[ drop i p ast-getp boa ]] - | PrimExpr:p "." Name:m "(" Args:as ")" => [[ drop m p as ast-send boa ]] - | PrimExpr:p "." Name:f => [[ drop f p ast-getp boa ]] - | PrimExpr:p "(" Args:as ")" => [[ drop p as ast-call boa ]] +PrimExpr = PrimExpr:p "[" Expr:i "]" => [[ i p ast-getp boa ]] + | PrimExpr:p "." Name:m "(" Args:as ")" => [[ m p as ast-send boa ]] + | PrimExpr:p "." Name:f => [[ f p ast-getp boa ]] + | PrimExpr:p "(" Args:as ")" => [[ p as ast-call boa ]] | PrimExprHd -PrimExprHd = "(" Expr:e ")" => [[ drop e ]] - | "this" => [[ drop ast-this boa ]] +PrimExprHd = "(" Expr:e ")" => [[ e ]] + | "this" => [[ ast-this boa ]] | Name => [[ ast-get boa ]] | Number => [[ ast-number boa ]] | Str => [[ ast-string boa ]] - | "function" FuncRest:fr => [[ drop fr ]] - | "new" Name:n "(" Args:as ")" => [[ drop n as ast-new boa ]] - | "[" Args:es "]" => [[ drop es ast-array boa ]] + | "function" FuncRest:fr => [[ fr ]] + | "new" Name:n "(" Args:as ")" => [[ n as ast-new boa ]] + | "[" Args:es "]" => [[ es ast-array boa ]] | Json JsonBindings = JsonBinding ("," JsonBinding)* => [[ first2 swap prefix ]] -Json = "{" JsonBindings:bs "}" => [[ drop bs ast-json boa ]] -JsonBinding = JsonPropName:n ":" Expr:v => [[ drop n v ast-binding boa ]] +Json = "{" JsonBindings:bs "}" => [[ bs ast-json boa ]] +JsonBinding = JsonPropName:n ":" Expr:v => [[ n v ast-binding boa ]] JsonPropName = Name | Number | Str Formal = Spaces Name Formals = Formal ("," Formal)* => [[ first2 swap prefix ]] -FuncRest = "(" Formals:fs ")" "{" SrcElems:body "}" => [[ drop fs body ast-func boa ]] +FuncRest = "(" Formals:fs ")" "{" SrcElems:body "}" => [[ fs body ast-func boa ]] Sc = SpacesNoNl ("\n" | "}")| ";" -Binding = Name:n "=" Expr:v => [[ drop n v ast-var boa ]] - | Name:n => [[ drop n "undefined" ast-get boa ast-var boa ]] -Block = "{" SrcElems:ss "}" => [[ drop ss ]] +Binding = Name:n "=" Expr:v => [[ n v ast-var boa ]] + | Name:n => [[ n "undefined" ast-get boa ast-var boa ]] +Block = "{" SrcElems:ss "}" => [[ ss ]] Bindings = Binding ("," Binding)* => [[ first2 swap prefix ]] For1 = "var" Binding => [[ second ]] | Expr @@ -180,31 +180,31 @@ For2 = Expr | Spaces => [[ "true" ast-get boa ]] For3 = Expr | Spaces => [[ "undefined" ast-get boa ]] -ForIn1 = "var" Name:n => [[ drop n "undefined" ast-get boa ast-var boa ]] +ForIn1 = "var" Name:n => [[ n "undefined" ast-get boa ast-var boa ]] | Expr -Switch1 = "case" Expr:c ":" SrcElems:cs => [[ drop c cs ast-case boa ]] - | "default" ":" SrcElems:cs => [[ drop cs ast-default boa ]] +Switch1 = "case" Expr:c ":" SrcElems:cs => [[ c cs ast-case boa ]] + | "default" ":" SrcElems:cs => [[ cs ast-default boa ]] SwitchBody = (Switch1)* -Finally = "finally" Block:b => [[ drop b ]] - | Spaces => [[ drop "undefined" ast-get boa ]] +Finally = "finally" Block:b => [[ b ]] + | Spaces => [[ "undefined" ast-get boa ]] Stmt = Block - | "var" Bindings:bs Sc => [[ drop bs ast-begin boa ]] - | "if" "(" Expr:c ")" Stmt:t "else" Stmt:f => [[ drop c t f ast-if boa ]] - | "if" "(" Expr:c ")" Stmt:t => [[ drop c t "undefined" ast-get boa ast-if boa ]] - | "while" "(" Expr:c ")" Stmt:s => [[ drop c s ast-while boa ]] - | "do" Stmt:s "while" "(" Expr:c ")" Sc => [[ drop s c ast-do-while boa ]] - | "for" "(" For1:i ";" For2:c ";" For3:u ")" Stmt:s => [[ drop i c u s ast-for boa ]] - | "for" "(" ForIn1:v "in" Expr:e ")" Stmt:s => [[ drop v e s ast-for-in boa ]] - | "switch" "(" Expr:e ")" "{" SwitchBody:cs "}" => [[ drop e cs ast-switch boa ]] - | "break" Sc => [[ drop ast-break boa ]] - | "continue" Sc => [[ drop ast-continue boa ]] - | "throw" SpacesNoNl Expr:e Sc => [[ drop e ast-throw boa ]] - | "try" Block:t "catch" "(" Name:e ")" Block:c Finally:f => [[ drop t e c f ast-try boa ]] - | "return" Expr:e Sc => [[ drop e ast-return boa ]] - | "return" Sc => [[ drop "undefined" ast-get boa ast-return boa ]] - | Expr:e Sc => [[ drop e ]] - | ";" => [[ drop "undefined" ast-get boa ]] -SrcElem = "function" Name:n FuncRest:f => [[ drop n f ast-var boa ]] + | "var" Bindings:bs Sc => [[ bs ast-begin boa ]] + | "if" "(" Expr:c ")" Stmt:t "else" Stmt:f => [[ c t f ast-if boa ]] + | "if" "(" Expr:c ")" Stmt:t => [[ c t "undefined" ast-get boa ast-if boa ]] + | "while" "(" Expr:c ")" Stmt:s => [[ c s ast-while boa ]] + | "do" Stmt:s "while" "(" Expr:c ")" Sc => [[ s c ast-do-while boa ]] + | "for" "(" For1:i ";" For2:c ";" For3:u ")" Stmt:s => [[ i c u s ast-for boa ]] + | "for" "(" ForIn1:v "in" Expr:e ")" Stmt:s => [[ v e s ast-for-in boa ]] + | "switch" "(" Expr:e ")" "{" SwitchBody:cs "}" => [[ e cs ast-switch boa ]] + | "break" Sc => [[ ast-break boa ]] + | "continue" Sc => [[ ast-continue boa ]] + | "throw" SpacesNoNl Expr:e Sc => [[ e ast-throw boa ]] + | "try" Block:t "catch" "(" Name:e ")" Block:c Finally:f => [[ t e c f ast-try boa ]] + | "return" Expr:e Sc => [[ e ast-return boa ]] + | "return" Sc => [[ "undefined" ast-get boa ast-return boa ]] + | Expr:e Sc => [[ e ]] + | ";" => [[ "undefined" ast-get boa ]] +SrcElem = "function" Name:n FuncRest:f => [[ n f ast-var boa ]] | Stmt SrcElems = (SrcElem)* => [[ ast-begin boa ]] TopLevel = SrcElems Spaces From 258951d954343a8e9289425ca9c1180ba285023c Mon Sep 17 00:00:00 2001 From: Chris Double Date: Tue, 17 Jun 2008 22:59:13 +1200 Subject: [PATCH 11/47] Split out javascript tokenizer --- extra/peg/javascript/javascript.factor | 58 ++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/extra/peg/javascript/javascript.factor b/extra/peg/javascript/javascript.factor index 54b9d8aa0a..3db962420a 100644 --- a/extra/peg/javascript/javascript.factor +++ b/extra/peg/javascript/javascript.factor @@ -46,6 +46,64 @@ TUPLE: ast-return e ; TUPLE: ast-case c cs ; TUPLE: ast-default cs ; +EBNF: tokenizer +Letter = [a-zA-Z] +Digit = [0-9] +Digits = (Digit)+ +SingleLineComment = "//" (!("\n") .)* "\n" => [[ ignore ]] +MultiLineComment = "/*" (!("*/") .)* "*/" => [[ ignore ]] +Space = " " | "\t" | "\n" | SingleLineComment | MultiLineComment +Spaces = (Space)* => [[ ignore ]] +NameFirst = Letter | "$" | "_" +NameRest = NameFirst | Digit +iName = NameFirst (NameRest)* => [[ first2 swap prefix >string ]] +Keyword = ("break" + | "case" + | "catch" + | "continue" + | "default" + | "delete" + | "do" + | "else" + | "finally" + | "for" + | "function" + | "if" + | "in" + | "instanceof" + | "new" + | "return" + | "switch" + | "this" + | "throw" + | "try" + | "typeof" + | "var" + | "void" + | "while" + | "with") => [[ ast-keyword boa ]] +Name = !(Keyword) (iName):n => [[ n ast-name boa ]] +Number = Digits:ws '.' Digits:fs => [[ ws "." fs 3array concat >string string>number ast-number boa ]] + | Digits => [[ >string string>number ast-number boa ]] + +EscapeChar = "\\n" => [[ 10 ]] + | "\\r" => [[ 13 ]] + | "\\t" => [[ 9 ]] +StringChars1 = (EscapeChar | !('"""') .)* => [[ >string ]] +StringChars2 = (EscapeChar | !('"') .)* => [[ >string ]] +StringChars3 = (EscapeChar | !("'") .)* => [[ >string ]] +Str = '"""' StringChars1:cs '"""' => [[ cs ast-string boa ]] + | '"' StringChars2:cs '"' => [[ cs ast-string boa ]] + | "'" StringChars3:cs "'" => [[ cs ast-string boa ]] +Special = "(" | ")" | "{" | "}" | "[" | "]" | "," | ";" + | "?" | ":" | "!==" | "~=" | "===" | "==" | "=" | ">=" + | ">" | "<=" | "<" | "++" | "+=" | "+" | "--" | "-=" + | "-" | "*=" | "*" | "/=" | "/" | "%=" | "%" | "&&=" + | "&&" | "||=" | "||" | "." | "!" +Tok = Spaces (Name | Keyword | Number | Str | Special ) +Toks = (Tok)* Spaces +;EBNF + EBNF: javascript Letter = [a-zA-Z] Digit = [0-9] From 4050ebcbde098f0b09a34a2123a12dbdc78d134c Mon Sep 17 00:00:00 2001 From: Chris Double Date: Tue, 17 Jun 2008 23:42:28 +1200 Subject: [PATCH 12/47] Javascript parser now works on token sequence --- extra/peg/ebnf/ebnf.factor | 2 +- extra/peg/javascript/javascript.factor | 66 ++++---------------------- 2 files changed, 9 insertions(+), 59 deletions(-) diff --git a/extra/peg/ebnf/ebnf.factor b/extra/peg/ebnf/ebnf.factor index 215eabdd37..36b3742b64 100644 --- a/extra/peg/ebnf/ebnf.factor +++ b/extra/peg/ebnf/ebnf.factor @@ -410,7 +410,7 @@ M: ebnf-var (transform) ( ast -- parser ) parser>> (transform) ; M: ebnf-terminal (transform) ( ast -- parser ) - symbol>> token ; + symbol>> [ token ] keep [ = ] curry satisfy 2choice ; M: ebnf-foreign (transform) ( ast -- parser ) dup word>> search diff --git a/extra/peg/javascript/javascript.factor b/extra/peg/javascript/javascript.factor index 3db962420a..c9bef2f6d3 100644 --- a/extra/peg/javascript/javascript.factor +++ b/extra/peg/javascript/javascript.factor @@ -1,7 +1,7 @@ ! Copyright (C) 2007 Chris Double. ! See http://factorcode.org/license.txt for BSD license. USING: kernel arrays strings math.parser sequences sequences.deep -peg peg.ebnf peg.parsers memoize namespaces math ; +peg peg.ebnf peg.parsers memoize namespaces math accessors ; IN: peg.javascript #! Grammar for JavaScript. Based on OMeta-JS example from: @@ -81,7 +81,7 @@ Keyword = ("break" | "var" | "void" | "while" - | "with") => [[ ast-keyword boa ]] + | "with") Name = !(Keyword) (iName):n => [[ n ast-name boa ]] Number = Digits:ws '.' Digits:fs => [[ ws "." fs 3array concat >string string>number ast-number boa ]] | Digits => [[ >string string>number ast-number boa ]] @@ -105,61 +105,11 @@ Toks = (Tok)* Spaces ;EBNF EBNF: javascript -Letter = [a-zA-Z] -Digit = [0-9] -Digits = (Digit)+ -SingleLineComment = "//" (!("\n") .)* "\n" => [[ ignore ]] -MultiLineComment = "/*" (!("*/") .)* "*/" => [[ ignore ]] -Space = " " | "\t" | "\n" | SingleLineComment | MultiLineComment +Space = " " | "\t" | "\n" Spaces = (Space)* => [[ ignore ]] -NameFirst = Letter | "$" | "_" -NameRest = NameFirst | Digit -iName = NameFirst (NameRest)* => [[ first2 swap prefix >string ]] -Keyword = ("break" - | "case" - | "catch" - | "continue" - | "default" - | "delete" - | "do" - | "else" - | "finally" - | "for" - | "function" - | "if" - | "in" - | "instanceof" - | "new" - | "return" - | "switch" - | "this" - | "throw" - | "try" - | "typeof" - | "var" - | "void" - | "while" - | "with") => [[ ast-keyword boa ]] -Name = !(Keyword) (iName):n => [[ n ast-name boa ]] -Number = Digits:ws '.' Digits:fs => [[ ws "." fs 3array concat >string string>number ast-number boa ]] - | Digits => [[ >string string>number ast-number boa ]] - -EscapeChar = "\\n" => [[ 10 ]] - | "\\r" => [[ 13 ]] - | "\\t" => [[ 9 ]] -StringChars1 = (EscapeChar | !('"""') .)* => [[ >string ]] -StringChars2 = (EscapeChar | !('"') .)* => [[ >string ]] -StringChars3 = (EscapeChar | !("'") .)* => [[ >string ]] -Str = '"""' StringChars1:cs '"""' => [[ cs ast-string boa ]] - | '"' StringChars2:cs '"' => [[ cs ast-string boa ]] - | "'" StringChars3:cs "'" => [[ cs ast-string boa ]] -Special = "(" | ")" | "{" | "}" | "[" | "]" | "," | ";" - | "?" | ":" | "!==" | "~=" | "===" | "==" | "=" | ">=" - | ">" | "<=" | "<" | "++" | "+=" | "+" | "--" | "-=" - | "-" | "*=" | "*" | "/=" | "/" | "%=" | "%" | "&&=" - | "&&" | "||=" | "||" | "." | "!" -Tok = Spaces (Name | Keyword | Number | Str | Special ) -Toks = (Tok)* Spaces +Name = . ?[ ast-name? ]? => [[ value>> ]] +Number = . ?[ ast-number? ]? => [[ value>> ]] +String = . ?[ ast-string? ]? => [[ value>> ]] SpacesNoNl = (!("\n") Space)* => [[ ignore ]] Expr = OrExpr:e "?" Expr:t ":" Expr:f => [[ e t f ast-cond-expr boa ]] @@ -214,7 +164,7 @@ PrimExprHd = "(" Expr:e ")" => [[ e ]] | "this" => [[ ast-this boa ]] | Name => [[ ast-get boa ]] | Number => [[ ast-number boa ]] - | Str => [[ ast-string boa ]] + | String => [[ ast-string boa ]] | "function" FuncRest:fr => [[ fr ]] | "new" Name:n "(" Args:as ")" => [[ n as ast-new boa ]] | "[" Args:es "]" => [[ es ast-array boa ]] @@ -222,7 +172,7 @@ PrimExprHd = "(" Expr:e ")" => [[ e ]] JsonBindings = JsonBinding ("," JsonBinding)* => [[ first2 swap prefix ]] Json = "{" JsonBindings:bs "}" => [[ bs ast-json boa ]] JsonBinding = JsonPropName:n ":" Expr:v => [[ n v ast-binding boa ]] -JsonPropName = Name | Number | Str +JsonPropName = Name | Number | String Formal = Spaces Name Formals = Formal ("," Formal)* => [[ first2 swap prefix ]] FuncRest = "(" Formals:fs ")" "{" SrcElems:body "}" => [[ fs body ast-func boa ]] From e99ff9fa6b996ba1fcf6199cfe5979bcfe221757 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Tue, 17 Jun 2008 23:57:51 +1200 Subject: [PATCH 13/47] Fix list AST in javascript parser --- extra/peg/javascript/javascript.factor | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/extra/peg/javascript/javascript.factor b/extra/peg/javascript/javascript.factor index c9bef2f6d3..c4d87e3ce5 100644 --- a/extra/peg/javascript/javascript.factor +++ b/extra/peg/javascript/javascript.factor @@ -154,7 +154,7 @@ Unary = "-" Postfix:p => [[ p "-" ast-unop boa Postfix = PrimExpr:p SpacesNoNl "++" => [[ p "++" ast-postop boa ]] | PrimExpr:p SpacesNoNl "--" => [[ p "--" ast-postop boa ]] | PrimExpr -Args = Expr ("," Expr)* => [[ first2 swap prefix ]] +Args = Expr ("," Expr => [[ second ]])* => [[ first2 swap prefix ]] PrimExpr = PrimExpr:p "[" Expr:i "]" => [[ i p ast-getp boa ]] | PrimExpr:p "." Name:m "(" Args:as ")" => [[ m p as ast-send boa ]] | PrimExpr:p "." Name:f => [[ f p ast-getp boa ]] @@ -169,18 +169,18 @@ PrimExprHd = "(" Expr:e ")" => [[ e ]] | "new" Name:n "(" Args:as ")" => [[ n as ast-new boa ]] | "[" Args:es "]" => [[ es ast-array boa ]] | Json -JsonBindings = JsonBinding ("," JsonBinding)* => [[ first2 swap prefix ]] +JsonBindings = JsonBinding ("," JsonBinding => [[ second ]])* => [[ first2 swap prefix ]] Json = "{" JsonBindings:bs "}" => [[ bs ast-json boa ]] JsonBinding = JsonPropName:n ":" Expr:v => [[ n v ast-binding boa ]] JsonPropName = Name | Number | String Formal = Spaces Name -Formals = Formal ("," Formal)* => [[ first2 swap prefix ]] +Formals = Formal ("," Formal => [[ second ]])* => [[ first2 swap prefix ]] FuncRest = "(" Formals:fs ")" "{" SrcElems:body "}" => [[ fs body ast-func boa ]] Sc = SpacesNoNl ("\n" | "}")| ";" Binding = Name:n "=" Expr:v => [[ n v ast-var boa ]] | Name:n => [[ n "undefined" ast-get boa ast-var boa ]] Block = "{" SrcElems:ss "}" => [[ ss ]] -Bindings = Binding ("," Binding)* => [[ first2 swap prefix ]] +Bindings = Binding ("," Binding => [[ second ]])* => [[ first2 swap prefix ]] For1 = "var" Binding => [[ second ]] | Expr | Spaces => [[ "undefined" ast-get boa ]] From 7694dfd68827394d5df1e61bcc0a3acba7db9e0c Mon Sep 17 00:00:00 2001 From: Chris Double Date: Wed, 18 Jun 2008 00:10:14 +1200 Subject: [PATCH 14/47] Allow zero arguments in javascript list handling --- extra/peg/javascript/javascript.factor | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/extra/peg/javascript/javascript.factor b/extra/peg/javascript/javascript.factor index c4d87e3ce5..5368881377 100644 --- a/extra/peg/javascript/javascript.factor +++ b/extra/peg/javascript/javascript.factor @@ -154,7 +154,7 @@ Unary = "-" Postfix:p => [[ p "-" ast-unop boa Postfix = PrimExpr:p SpacesNoNl "++" => [[ p "++" ast-postop boa ]] | PrimExpr:p SpacesNoNl "--" => [[ p "--" ast-postop boa ]] | PrimExpr -Args = Expr ("," Expr => [[ second ]])* => [[ first2 swap prefix ]] +Args = (Expr ("," Expr => [[ second ]])* => [[ first2 swap prefix ]])? PrimExpr = PrimExpr:p "[" Expr:i "]" => [[ i p ast-getp boa ]] | PrimExpr:p "." Name:m "(" Args:as ")" => [[ m p as ast-send boa ]] | PrimExpr:p "." Name:f => [[ f p ast-getp boa ]] @@ -169,18 +169,18 @@ PrimExprHd = "(" Expr:e ")" => [[ e ]] | "new" Name:n "(" Args:as ")" => [[ n as ast-new boa ]] | "[" Args:es "]" => [[ es ast-array boa ]] | Json -JsonBindings = JsonBinding ("," JsonBinding => [[ second ]])* => [[ first2 swap prefix ]] +JsonBindings = (JsonBinding ("," JsonBinding => [[ second ]])* => [[ first2 swap prefix ]])? Json = "{" JsonBindings:bs "}" => [[ bs ast-json boa ]] JsonBinding = JsonPropName:n ":" Expr:v => [[ n v ast-binding boa ]] JsonPropName = Name | Number | String Formal = Spaces Name -Formals = Formal ("," Formal => [[ second ]])* => [[ first2 swap prefix ]] +Formals = (Formal ("," Formal => [[ second ]])* => [[ first2 swap prefix ]])? FuncRest = "(" Formals:fs ")" "{" SrcElems:body "}" => [[ fs body ast-func boa ]] Sc = SpacesNoNl ("\n" | "}")| ";" Binding = Name:n "=" Expr:v => [[ n v ast-var boa ]] | Name:n => [[ n "undefined" ast-get boa ast-var boa ]] Block = "{" SrcElems:ss "}" => [[ ss ]] -Bindings = Binding ("," Binding => [[ second ]])* => [[ first2 swap prefix ]] +Bindings = (Binding ("," Binding => [[ second ]])* => [[ first2 swap prefix ]])? For1 = "var" Binding => [[ second ]] | Expr | Spaces => [[ "undefined" ast-get boa ]] From 778573106c10aeddf03ed7384ea6270f3ea07123 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Wed, 18 Jun 2008 12:16:47 +1200 Subject: [PATCH 15/47] Fix Sc rule --- extra/peg/javascript/javascript.factor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/peg/javascript/javascript.factor b/extra/peg/javascript/javascript.factor index 5368881377..bdf5f4b369 100644 --- a/extra/peg/javascript/javascript.factor +++ b/extra/peg/javascript/javascript.factor @@ -176,7 +176,7 @@ JsonPropName = Name | Number | String Formal = Spaces Name Formals = (Formal ("," Formal => [[ second ]])* => [[ first2 swap prefix ]])? FuncRest = "(" Formals:fs ")" "{" SrcElems:body "}" => [[ fs body ast-func boa ]] -Sc = SpacesNoNl ("\n" | "}")| ";" +Sc = SpacesNoNl ("\n" | &("}"))| ";" Binding = Name:n "=" Expr:v => [[ n v ast-var boa ]] | Name:n => [[ n "undefined" ast-get boa ast-var boa ]] Block = "{" SrcElems:ss "}" => [[ ss ]] From a5719e33976ae6ebfc75ab81edd6d56dd0f0ee0a Mon Sep 17 00:00:00 2001 From: Chris Double Date: Wed, 18 Jun 2008 21:30:21 +1200 Subject: [PATCH 16/47] Add javascript tests. Minor changes to javascript grammar --- extra/peg/javascript/javascript-tests.factor | 42 ++++++++++++++++++++ extra/peg/javascript/javascript.factor | 16 ++++---- 2 files changed, 50 insertions(+), 8 deletions(-) create mode 100644 extra/peg/javascript/javascript-tests.factor diff --git a/extra/peg/javascript/javascript-tests.factor b/extra/peg/javascript/javascript-tests.factor new file mode 100644 index 0000000000..70410a3838 --- /dev/null +++ b/extra/peg/javascript/javascript-tests.factor @@ -0,0 +1,42 @@ +! Copyright (C) 2008 Chris Double. +! See http://factorcode.org/license.txt for BSD license. +! +USING: kernel tools.test peg peg.javascript accessors ; +IN: peg.javascript.tests + +\ javascript must-infer + +{ + V{ + T{ ast-number f 123 } + ";" + T{ ast-string f "hello" } + ";" + T{ ast-name f "foo" } + "(" + T{ ast-name f "x" } + ")" + ";" + } +} [ + "123; 'hello'; foo(x);" tokenizer ast>> +] unit-test + +{ + T{ + ast-begin + f + V{ + T{ ast-number f 123 } + T{ ast-string f "hello" } + T{ + ast-call + f + T{ ast-get f "foo" } + V{ T{ ast-get f "x" } } + } + } + } +} [ + "123; 'hello'; foo(x);" tokenizer ast>> javascript ast>> +] unit-test \ No newline at end of file diff --git a/extra/peg/javascript/javascript.factor b/extra/peg/javascript/javascript.factor index bdf5f4b369..030d2f1728 100644 --- a/extra/peg/javascript/javascript.factor +++ b/extra/peg/javascript/javascript.factor @@ -1,4 +1,4 @@ -! Copyright (C) 2007 Chris Double. +! Copyright (C) 2008 Chris Double. ! See http://factorcode.org/license.txt for BSD license. USING: kernel arrays strings math.parser sequences sequences.deep peg peg.ebnf peg.parsers memoize namespaces math accessors ; @@ -49,14 +49,14 @@ TUPLE: ast-default cs ; EBNF: tokenizer Letter = [a-zA-Z] Digit = [0-9] -Digits = (Digit)+ +Digits = Digit+ SingleLineComment = "//" (!("\n") .)* "\n" => [[ ignore ]] MultiLineComment = "/*" (!("*/") .)* "*/" => [[ ignore ]] Space = " " | "\t" | "\n" | SingleLineComment | MultiLineComment -Spaces = (Space)* => [[ ignore ]] +Spaces = Space* => [[ ignore ]] NameFirst = Letter | "$" | "_" NameRest = NameFirst | Digit -iName = NameFirst (NameRest)* => [[ first2 swap prefix >string ]] +iName = NameFirst NameRest* => [[ first2 swap prefix >string ]] Keyword = ("break" | "case" | "catch" @@ -101,12 +101,12 @@ Special = "(" | ")" | "{" | "}" | "[" | "]" | "," | " | "-" | "*=" | "*" | "/=" | "/" | "%=" | "%" | "&&=" | "&&" | "||=" | "||" | "." | "!" Tok = Spaces (Name | Keyword | Number | Str | Special ) -Toks = (Tok)* Spaces +Toks = Tok* Spaces ;EBNF EBNF: javascript Space = " " | "\t" | "\n" -Spaces = (Space)* => [[ ignore ]] +Spaces = Space* => [[ ignore ]] Name = . ?[ ast-name? ]? => [[ value>> ]] Number = . ?[ ast-number? ]? => [[ value>> ]] String = . ?[ ast-string? ]? => [[ value>> ]] @@ -192,7 +192,7 @@ ForIn1 = "var" Name:n => [[ n "undefined" ast-get boa ast-var boa | Expr Switch1 = "case" Expr:c ":" SrcElems:cs => [[ c cs ast-case boa ]] | "default" ":" SrcElems:cs => [[ cs ast-default boa ]] -SwitchBody = (Switch1)* +SwitchBody = Switch1* Finally = "finally" Block:b => [[ b ]] | Spaces => [[ "undefined" ast-get boa ]] Stmt = Block @@ -214,6 +214,6 @@ Stmt = Block | ";" => [[ "undefined" ast-get boa ]] SrcElem = "function" Name:n FuncRest:f => [[ n f ast-var boa ]] | Stmt -SrcElems = (SrcElem)* => [[ ast-begin boa ]] +SrcElems = SrcElem* => [[ ast-begin boa ]] TopLevel = SrcElems Spaces ;EBNF \ No newline at end of file From fea65df1dfeb1f56bf40fb232aae99c430b2d731 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Wed, 18 Jun 2008 23:07:26 +1200 Subject: [PATCH 17/47] Fix ebnf foreign call breakage and add tests --- extra/peg/ebnf/ebnf-tests.factor | 36 +++++++++++++++++++++++++++++++- extra/peg/ebnf/ebnf.factor | 6 +++--- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/extra/peg/ebnf/ebnf-tests.factor b/extra/peg/ebnf/ebnf-tests.factor index 04cc01c9d0..73db626685 100644 --- a/extra/peg/ebnf/ebnf-tests.factor +++ b/extra/peg/ebnf/ebnf-tests.factor @@ -2,7 +2,7 @@ ! See http://factorcode.org/license.txt for BSD license. ! USING: kernel tools.test peg peg.ebnf words math math.parser - sequences accessors ; + sequences accessors peg.parsers ; IN: peg.ebnf.tests { T{ ebnf-non-terminal f "abc" } } [ @@ -397,4 +397,38 @@ main = Primary { t } [ "number=digit+:n 'a'" 'ebnf' parse remaining>> length zero? +] unit-test + +<< +EBNF: parser1 +foo='a' +;EBNF +>> + +EBNF: parser2 +foo= 'b' +;EBNF + +EBNF: parser3 +foo= 'c' +;EBNF + +EBNF: parser4 +foo= 'd' +;EBNF + +{ "a" } [ + "a" parser1 ast>> +] unit-test + +{ V{ "a" "b" } } [ + "ab" parser2 ast>> +] unit-test + +{ V{ "a" "c" } } [ + "ac" parser3 ast>> +] unit-test + +{ V{ CHAR: a "d" } } [ + "ad" parser4 ast>> ] unit-test \ No newline at end of file diff --git a/extra/peg/ebnf/ebnf.factor b/extra/peg/ebnf/ebnf.factor index 36b3742b64..2ee0958051 100644 --- a/extra/peg/ebnf/ebnf.factor +++ b/extra/peg/ebnf/ebnf.factor @@ -415,11 +415,11 @@ M: ebnf-terminal (transform) ( ast -- parser ) M: ebnf-foreign (transform) ( ast -- parser ) dup word>> search [ "Foreign word " swap word>> append " not found" append throw ] unless* - swap rule>> dup [ - swap rule + swap rule>> [ main ] unless* dupd swap rule [ + nip ] [ execute - ] if ; + ] if* ; : parser-not-found ( name -- * ) [ From ea6974d5dac52706a67666a50ecaf22c915280eb Mon Sep 17 00:00:00 2001 From: Chris Double Date: Wed, 18 Jun 2008 23:50:25 +1200 Subject: [PATCH 18/47] Add \r to whitespace in javascript tokenizer --- extra/peg/javascript/javascript.factor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/peg/javascript/javascript.factor b/extra/peg/javascript/javascript.factor index 030d2f1728..127b13130a 100644 --- a/extra/peg/javascript/javascript.factor +++ b/extra/peg/javascript/javascript.factor @@ -52,7 +52,7 @@ Digit = [0-9] Digits = Digit+ SingleLineComment = "//" (!("\n") .)* "\n" => [[ ignore ]] MultiLineComment = "/*" (!("*/") .)* "*/" => [[ ignore ]] -Space = " " | "\t" | "\n" | SingleLineComment | MultiLineComment +Space = " " | "\t" | "\r" | "\n" | SingleLineComment | MultiLineComment Spaces = Space* => [[ ignore ]] NameFirst = Letter | "$" | "_" NameRest = NameFirst | Digit From 6f8e2a4b0ddf74ea0f7bd43aed53984faaceab9e Mon Sep 17 00:00:00 2001 From: Chris Double Date: Thu, 19 Jun 2008 00:42:11 +1200 Subject: [PATCH 19/47] Refactor JavaScript parser --- extra/peg/javascript/ast/ast.factor | 42 ++++ extra/peg/javascript/ast/authors.txt | 1 + extra/peg/javascript/ast/summary.txt | 1 + extra/peg/javascript/ast/tags.txt | 3 + extra/peg/javascript/authors.txt | 1 + extra/peg/javascript/javascript-docs.factor | 14 ++ extra/peg/javascript/javascript-tests.factor | 39 +-- extra/peg/javascript/javascript.factor | 225 +----------------- extra/peg/javascript/parser/authors.txt | 1 + .../peg/javascript/parser/parser-tests.factor | 27 +++ extra/peg/javascript/parser/parser.factor | 121 ++++++++++ extra/peg/javascript/parser/summary.txt | 1 + extra/peg/javascript/parser/tags.txt | 3 + extra/peg/javascript/summary.txt | 1 + extra/peg/javascript/tags.txt | 3 + extra/peg/javascript/tokenizer/authors.txt | 1 + extra/peg/javascript/tokenizer/summary.txt | 1 + extra/peg/javascript/tokenizer/tags.txt | 3 + .../tokenizer/tokenizer-tests.factor | 23 ++ .../peg/javascript/tokenizer/tokenizer.factor | 68 ++++++ 20 files changed, 330 insertions(+), 249 deletions(-) create mode 100644 extra/peg/javascript/ast/ast.factor create mode 100644 extra/peg/javascript/ast/authors.txt create mode 100644 extra/peg/javascript/ast/summary.txt create mode 100644 extra/peg/javascript/ast/tags.txt create mode 100644 extra/peg/javascript/authors.txt create mode 100644 extra/peg/javascript/javascript-docs.factor create mode 100644 extra/peg/javascript/parser/authors.txt create mode 100644 extra/peg/javascript/parser/parser-tests.factor create mode 100644 extra/peg/javascript/parser/parser.factor create mode 100644 extra/peg/javascript/parser/summary.txt create mode 100644 extra/peg/javascript/parser/tags.txt create mode 100644 extra/peg/javascript/summary.txt create mode 100644 extra/peg/javascript/tags.txt create mode 100644 extra/peg/javascript/tokenizer/authors.txt create mode 100644 extra/peg/javascript/tokenizer/summary.txt create mode 100644 extra/peg/javascript/tokenizer/tags.txt create mode 100644 extra/peg/javascript/tokenizer/tokenizer-tests.factor create mode 100644 extra/peg/javascript/tokenizer/tokenizer.factor diff --git a/extra/peg/javascript/ast/ast.factor b/extra/peg/javascript/ast/ast.factor new file mode 100644 index 0000000000..b857dc51bb --- /dev/null +++ b/extra/peg/javascript/ast/ast.factor @@ -0,0 +1,42 @@ +! Copyright (C) 2008 Chris Double. +! See http://factorcode.org/license.txt for BSD license. +USING: kernel ; +IN: peg.javascript.ast + +TUPLE: ast-keyword value ; +TUPLE: ast-name value ; +TUPLE: ast-number value ; +TUPLE: ast-string value ; +TUPLE: ast-regexp value ; +TUPLE: ast-cond-expr condition then else ; +TUPLE: ast-set lhs rhs ; +TUPLE: ast-get value ; +TUPLE: ast-mset lhs rhs operator ; +TUPLE: ast-binop lhs rhs operator ; +TUPLE: ast-unop expr operator ; +TUPLE: ast-postop expr operator ; +TUPLE: ast-preop expr operator ; +TUPLE: ast-getp index expr ; +TUPLE: ast-send method expr args ; +TUPLE: ast-call expr args ; +TUPLE: ast-this ; +TUPLE: ast-new name args ; +TUPLE: ast-array values ; +TUPLE: ast-json bindings ; +TUPLE: ast-binding name value ; +TUPLE: ast-func fs body ; +TUPLE: ast-var name value ; +TUPLE: ast-begin statements ; +TUPLE: ast-if condition true false ; +TUPLE: ast-while condition statements ; +TUPLE: ast-do-while statements condition ; +TUPLE: ast-for i c u statements ; +TUPLE: ast-for-in v e statements ; +TUPLE: ast-switch expr statements ; +TUPLE: ast-break ; +TUPLE: ast-continue ; +TUPLE: ast-throw e ; +TUPLE: ast-try t e c f ; +TUPLE: ast-return e ; +TUPLE: ast-case c cs ; +TUPLE: ast-default cs ; diff --git a/extra/peg/javascript/ast/authors.txt b/extra/peg/javascript/ast/authors.txt new file mode 100644 index 0000000000..44b06f94bc --- /dev/null +++ b/extra/peg/javascript/ast/authors.txt @@ -0,0 +1 @@ +Chris Double diff --git a/extra/peg/javascript/ast/summary.txt b/extra/peg/javascript/ast/summary.txt new file mode 100644 index 0000000000..543a2e6373 --- /dev/null +++ b/extra/peg/javascript/ast/summary.txt @@ -0,0 +1 @@ +Abstract Syntax Tree for JavaScript parser diff --git a/extra/peg/javascript/ast/tags.txt b/extra/peg/javascript/ast/tags.txt new file mode 100644 index 0000000000..c2aac2932f --- /dev/null +++ b/extra/peg/javascript/ast/tags.txt @@ -0,0 +1,3 @@ +text +javascript +parsing diff --git a/extra/peg/javascript/authors.txt b/extra/peg/javascript/authors.txt new file mode 100644 index 0000000000..44b06f94bc --- /dev/null +++ b/extra/peg/javascript/authors.txt @@ -0,0 +1 @@ +Chris Double diff --git a/extra/peg/javascript/javascript-docs.factor b/extra/peg/javascript/javascript-docs.factor new file mode 100644 index 0000000000..5fdc3e8587 --- /dev/null +++ b/extra/peg/javascript/javascript-docs.factor @@ -0,0 +1,14 @@ +! Copyright (C) 2007 Chris Double. +! See http://factorcode.org/license.txt for BSD license. +USING: help.markup help.syntax ; +IN: peg.javascript + +HELP: parse-javascript +{ $values + { "string" "a string" } + { "ast" "a JavaScript abstract syntax tree" } +} +{ $description + "Parse the input string using the JavaScript parser. Throws an error if " + "the string does not contain valid JavaScript. Returns the abstract syntax tree " + "if successful." } ; diff --git a/extra/peg/javascript/javascript-tests.factor b/extra/peg/javascript/javascript-tests.factor index 70410a3838..0d6899714d 100644 --- a/extra/peg/javascript/javascript-tests.factor +++ b/extra/peg/javascript/javascript-tests.factor @@ -1,42 +1,11 @@ ! Copyright (C) 2008 Chris Double. ! See http://factorcode.org/license.txt for BSD license. ! -USING: kernel tools.test peg peg.javascript accessors ; +USING: kernel tools.test peg.javascript peg.javascript.ast accessors ; IN: peg.javascript.tests -\ javascript must-infer +\ parse-javascript must-infer -{ - V{ - T{ ast-number f 123 } - ";" - T{ ast-string f "hello" } - ";" - T{ ast-name f "foo" } - "(" - T{ ast-name f "x" } - ")" - ";" - } -} [ - "123; 'hello'; foo(x);" tokenizer ast>> -] unit-test - -{ - T{ - ast-begin - f - V{ - T{ ast-number f 123 } - T{ ast-string f "hello" } - T{ - ast-call - f - T{ ast-get f "foo" } - V{ T{ ast-get f "x" } } - } - } - } -} [ - "123; 'hello'; foo(x);" tokenizer ast>> javascript ast>> +{ T{ ast-begin f V{ T{ ast-number f 123 } } } } [ + "123;" parse-javascript ] unit-test \ No newline at end of file diff --git a/extra/peg/javascript/javascript.factor b/extra/peg/javascript/javascript.factor index 127b13130a..23a4b4f7f0 100644 --- a/extra/peg/javascript/javascript.factor +++ b/extra/peg/javascript/javascript.factor @@ -1,219 +1,16 @@ ! Copyright (C) 2008 Chris Double. ! See http://factorcode.org/license.txt for BSD license. -USING: kernel arrays strings math.parser sequences sequences.deep -peg peg.ebnf peg.parsers memoize namespaces math accessors ; +USING: kernel accessors peg.javascript.tokenizer peg.javascript.parser ; IN: peg.javascript -#! Grammar for JavaScript. Based on OMeta-JS example from: -#! http://jarrett.cs.ucla.edu/ometa-js/#JavaScript_Compiler +: parse-javascript ( string -- ast ) + tokenizer [ + ast>> javascript [ + ast>> + ] [ + "Unable to parse JavaScript" throw + ] if* + ] [ + "Unable to tokenize JavaScript" throw + ] if* ; -USE: prettyprint - -TUPLE: ast-keyword value ; -TUPLE: ast-name value ; -TUPLE: ast-number value ; -TUPLE: ast-string value ; -TUPLE: ast-cond-expr condition then else ; -TUPLE: ast-set lhs rhs ; -TUPLE: ast-get value ; -TUPLE: ast-mset lhs rhs operator ; -TUPLE: ast-binop lhs rhs operator ; -TUPLE: ast-unop expr operator ; -TUPLE: ast-postop expr operator ; -TUPLE: ast-preop expr operator ; -TUPLE: ast-getp index expr ; -TUPLE: ast-send method expr args ; -TUPLE: ast-call expr args ; -TUPLE: ast-this ; -TUPLE: ast-new name args ; -TUPLE: ast-array values ; -TUPLE: ast-json bindings ; -TUPLE: ast-binding name value ; -TUPLE: ast-func fs body ; -TUPLE: ast-var name value ; -TUPLE: ast-begin statements ; -TUPLE: ast-if condition true false ; -TUPLE: ast-while condition statements ; -TUPLE: ast-do-while statements condition ; -TUPLE: ast-for i c u statements ; -TUPLE: ast-for-in v e statements ; -TUPLE: ast-switch expr statements ; -TUPLE: ast-break ; -TUPLE: ast-continue ; -TUPLE: ast-throw e ; -TUPLE: ast-try t e c f ; -TUPLE: ast-return e ; -TUPLE: ast-case c cs ; -TUPLE: ast-default cs ; - -EBNF: tokenizer -Letter = [a-zA-Z] -Digit = [0-9] -Digits = Digit+ -SingleLineComment = "//" (!("\n") .)* "\n" => [[ ignore ]] -MultiLineComment = "/*" (!("*/") .)* "*/" => [[ ignore ]] -Space = " " | "\t" | "\r" | "\n" | SingleLineComment | MultiLineComment -Spaces = Space* => [[ ignore ]] -NameFirst = Letter | "$" | "_" -NameRest = NameFirst | Digit -iName = NameFirst NameRest* => [[ first2 swap prefix >string ]] -Keyword = ("break" - | "case" - | "catch" - | "continue" - | "default" - | "delete" - | "do" - | "else" - | "finally" - | "for" - | "function" - | "if" - | "in" - | "instanceof" - | "new" - | "return" - | "switch" - | "this" - | "throw" - | "try" - | "typeof" - | "var" - | "void" - | "while" - | "with") -Name = !(Keyword) (iName):n => [[ n ast-name boa ]] -Number = Digits:ws '.' Digits:fs => [[ ws "." fs 3array concat >string string>number ast-number boa ]] - | Digits => [[ >string string>number ast-number boa ]] - -EscapeChar = "\\n" => [[ 10 ]] - | "\\r" => [[ 13 ]] - | "\\t" => [[ 9 ]] -StringChars1 = (EscapeChar | !('"""') .)* => [[ >string ]] -StringChars2 = (EscapeChar | !('"') .)* => [[ >string ]] -StringChars3 = (EscapeChar | !("'") .)* => [[ >string ]] -Str = '"""' StringChars1:cs '"""' => [[ cs ast-string boa ]] - | '"' StringChars2:cs '"' => [[ cs ast-string boa ]] - | "'" StringChars3:cs "'" => [[ cs ast-string boa ]] -Special = "(" | ")" | "{" | "}" | "[" | "]" | "," | ";" - | "?" | ":" | "!==" | "~=" | "===" | "==" | "=" | ">=" - | ">" | "<=" | "<" | "++" | "+=" | "+" | "--" | "-=" - | "-" | "*=" | "*" | "/=" | "/" | "%=" | "%" | "&&=" - | "&&" | "||=" | "||" | "." | "!" -Tok = Spaces (Name | Keyword | Number | Str | Special ) -Toks = Tok* Spaces -;EBNF - -EBNF: javascript -Space = " " | "\t" | "\n" -Spaces = Space* => [[ ignore ]] -Name = . ?[ ast-name? ]? => [[ value>> ]] -Number = . ?[ ast-number? ]? => [[ value>> ]] -String = . ?[ ast-string? ]? => [[ value>> ]] -SpacesNoNl = (!("\n") Space)* => [[ ignore ]] - -Expr = OrExpr:e "?" Expr:t ":" Expr:f => [[ e t f ast-cond-expr boa ]] - | OrExpr:e "=" Expr:rhs => [[ e rhs ast-set boa ]] - | OrExpr:e "+=" Expr:rhs => [[ e rhs "+" ast-mset boa ]] - | OrExpr:e "-=" Expr:rhs => [[ e rhs "-" ast-mset boa ]] - | OrExpr:e "*=" Expr:rhs => [[ e rhs "*" ast-mset boa ]] - | OrExpr:e "/=" Expr:rhs => [[ e rhs "/" ast-mset boa ]] - | OrExpr:e "%=" Expr:rhs => [[ e rhs "%" ast-mset boa ]] - | OrExpr:e "&&=" Expr:rhs => [[ e rhs "&&" ast-mset boa ]] - | OrExpr:e "||=" Expr:rhs => [[ e rhs "||" ast-mset boa ]] - | OrExpr:e => [[ e ]] - -OrExpr = OrExpr:x "||" AndExpr:y => [[ x y "||" ast-binop boa ]] - | AndExpr -AndExpr = AndExpr:x "&&" EqExpr:y => [[ x y "&&" ast-binop boa ]] - | EqExpr -EqExpr = EqExpr:x "==" RelExpr:y => [[ x y "==" ast-binop boa ]] - | EqExpr:x "!=" RelExpr:y => [[ x y "!=" ast-binop boa ]] - | EqExpr:x "===" RelExpr:y => [[ x y "===" ast-binop boa ]] - | EqExpr:x "!==" RelExpr:y => [[ x y "!==" ast-binop boa ]] - | RelExpr -RelExpr = RelExpr:x ">" AddExpr:y => [[ x y ">" ast-binop boa ]] - | RelExpr:x ">=" AddExpr:y => [[ x y ">=" ast-binop boa ]] - | RelExpr:x "<" AddExpr:y => [[ x y "<" ast-binop boa ]] - | RelExpr:x "<=" AddExpr:y => [[ x y "<=" ast-binop boa ]] - | RelExpr:x "instanceof" AddExpr:y => [[ x y "instanceof" ast-binop boa ]] - | AddExpr -AddExpr = AddExpr:x "+" MulExpr:y => [[ x y "+" ast-binop boa ]] - | AddExpr:x "-" MulExpr:y => [[ x y "-" ast-binop boa ]] - | MulExpr -MulExpr = MulExpr:x "*" MulExpr:y => [[ x y "*" ast-binop boa ]] - | MulExpr:x "/" MulExpr:y => [[ x y "/" ast-binop boa ]] - | MulExpr:x "%" MulExpr:y => [[ x y "%" ast-binop boa ]] - | Unary -Unary = "-" Postfix:p => [[ p "-" ast-unop boa ]] - | "+" Postfix:p => [[ p ]] - | "++" Postfix:p => [[ p "++" ast-preop boa ]] - | "--" Postfix:p => [[ p "--" ast-preop boa ]] - | "!" Postfix:p => [[ p "!" ast-unop boa ]] - | Postfix -Postfix = PrimExpr:p SpacesNoNl "++" => [[ p "++" ast-postop boa ]] - | PrimExpr:p SpacesNoNl "--" => [[ p "--" ast-postop boa ]] - | PrimExpr -Args = (Expr ("," Expr => [[ second ]])* => [[ first2 swap prefix ]])? -PrimExpr = PrimExpr:p "[" Expr:i "]" => [[ i p ast-getp boa ]] - | PrimExpr:p "." Name:m "(" Args:as ")" => [[ m p as ast-send boa ]] - | PrimExpr:p "." Name:f => [[ f p ast-getp boa ]] - | PrimExpr:p "(" Args:as ")" => [[ p as ast-call boa ]] - | PrimExprHd -PrimExprHd = "(" Expr:e ")" => [[ e ]] - | "this" => [[ ast-this boa ]] - | Name => [[ ast-get boa ]] - | Number => [[ ast-number boa ]] - | String => [[ ast-string boa ]] - | "function" FuncRest:fr => [[ fr ]] - | "new" Name:n "(" Args:as ")" => [[ n as ast-new boa ]] - | "[" Args:es "]" => [[ es ast-array boa ]] - | Json -JsonBindings = (JsonBinding ("," JsonBinding => [[ second ]])* => [[ first2 swap prefix ]])? -Json = "{" JsonBindings:bs "}" => [[ bs ast-json boa ]] -JsonBinding = JsonPropName:n ":" Expr:v => [[ n v ast-binding boa ]] -JsonPropName = Name | Number | String -Formal = Spaces Name -Formals = (Formal ("," Formal => [[ second ]])* => [[ first2 swap prefix ]])? -FuncRest = "(" Formals:fs ")" "{" SrcElems:body "}" => [[ fs body ast-func boa ]] -Sc = SpacesNoNl ("\n" | &("}"))| ";" -Binding = Name:n "=" Expr:v => [[ n v ast-var boa ]] - | Name:n => [[ n "undefined" ast-get boa ast-var boa ]] -Block = "{" SrcElems:ss "}" => [[ ss ]] -Bindings = (Binding ("," Binding => [[ second ]])* => [[ first2 swap prefix ]])? -For1 = "var" Binding => [[ second ]] - | Expr - | Spaces => [[ "undefined" ast-get boa ]] -For2 = Expr - | Spaces => [[ "true" ast-get boa ]] -For3 = Expr - | Spaces => [[ "undefined" ast-get boa ]] -ForIn1 = "var" Name:n => [[ n "undefined" ast-get boa ast-var boa ]] - | Expr -Switch1 = "case" Expr:c ":" SrcElems:cs => [[ c cs ast-case boa ]] - | "default" ":" SrcElems:cs => [[ cs ast-default boa ]] -SwitchBody = Switch1* -Finally = "finally" Block:b => [[ b ]] - | Spaces => [[ "undefined" ast-get boa ]] -Stmt = Block - | "var" Bindings:bs Sc => [[ bs ast-begin boa ]] - | "if" "(" Expr:c ")" Stmt:t "else" Stmt:f => [[ c t f ast-if boa ]] - | "if" "(" Expr:c ")" Stmt:t => [[ c t "undefined" ast-get boa ast-if boa ]] - | "while" "(" Expr:c ")" Stmt:s => [[ c s ast-while boa ]] - | "do" Stmt:s "while" "(" Expr:c ")" Sc => [[ s c ast-do-while boa ]] - | "for" "(" For1:i ";" For2:c ";" For3:u ")" Stmt:s => [[ i c u s ast-for boa ]] - | "for" "(" ForIn1:v "in" Expr:e ")" Stmt:s => [[ v e s ast-for-in boa ]] - | "switch" "(" Expr:e ")" "{" SwitchBody:cs "}" => [[ e cs ast-switch boa ]] - | "break" Sc => [[ ast-break boa ]] - | "continue" Sc => [[ ast-continue boa ]] - | "throw" SpacesNoNl Expr:e Sc => [[ e ast-throw boa ]] - | "try" Block:t "catch" "(" Name:e ")" Block:c Finally:f => [[ t e c f ast-try boa ]] - | "return" Expr:e Sc => [[ e ast-return boa ]] - | "return" Sc => [[ "undefined" ast-get boa ast-return boa ]] - | Expr:e Sc => [[ e ]] - | ";" => [[ "undefined" ast-get boa ]] -SrcElem = "function" Name:n FuncRest:f => [[ n f ast-var boa ]] - | Stmt -SrcElems = SrcElem* => [[ ast-begin boa ]] -TopLevel = SrcElems Spaces -;EBNF \ No newline at end of file diff --git a/extra/peg/javascript/parser/authors.txt b/extra/peg/javascript/parser/authors.txt new file mode 100644 index 0000000000..44b06f94bc --- /dev/null +++ b/extra/peg/javascript/parser/authors.txt @@ -0,0 +1 @@ +Chris Double diff --git a/extra/peg/javascript/parser/parser-tests.factor b/extra/peg/javascript/parser/parser-tests.factor new file mode 100644 index 0000000000..933d4cf10e --- /dev/null +++ b/extra/peg/javascript/parser/parser-tests.factor @@ -0,0 +1,27 @@ +! Copyright (C) 2008 Chris Double. +! See http://factorcode.org/license.txt for BSD license. +! +USING: kernel tools.test peg peg.javascript.ast peg.javascript.tokenizer + peg.javascript.parser accessors ; +IN: peg.javascript.parser.tests + +\ javascript must-infer + +{ + T{ + ast-begin + f + V{ + T{ ast-number f 123 } + T{ ast-string f "hello" } + T{ + ast-call + f + T{ ast-get f "foo" } + V{ T{ ast-get f "x" } } + } + } + } +} [ + "123; 'hello'; foo(x);" tokenizer ast>> javascript ast>> +] unit-test \ No newline at end of file diff --git a/extra/peg/javascript/parser/parser.factor b/extra/peg/javascript/parser/parser.factor new file mode 100644 index 0000000000..a38cf4aea8 --- /dev/null +++ b/extra/peg/javascript/parser/parser.factor @@ -0,0 +1,121 @@ +! Copyright (C) 2008 Chris Double. +! See http://factorcode.org/license.txt for BSD license. +USING: kernel accessors peg peg.ebnf peg.javascript.ast ; +IN: peg.javascript.parser + +#! Grammar for JavaScript. Based on OMeta-JS example from: +#! http://jarrett.cs.ucla.edu/ometa-js/#JavaScript_Compiler + +EBNF: javascript +Space = " " | "\t" | "\n" +Spaces = Space* => [[ ignore ]] +Name = . ?[ ast-name? ]? => [[ value>> ]] +Number = . ?[ ast-number? ]? => [[ value>> ]] +String = . ?[ ast-string? ]? => [[ value>> ]] +SpacesNoNl = (!("\n") Space)* => [[ ignore ]] + +Expr = OrExpr:e "?" Expr:t ":" Expr:f => [[ e t f ast-cond-expr boa ]] + | OrExpr:e "=" Expr:rhs => [[ e rhs ast-set boa ]] + | OrExpr:e "+=" Expr:rhs => [[ e rhs "+" ast-mset boa ]] + | OrExpr:e "-=" Expr:rhs => [[ e rhs "-" ast-mset boa ]] + | OrExpr:e "*=" Expr:rhs => [[ e rhs "*" ast-mset boa ]] + | OrExpr:e "/=" Expr:rhs => [[ e rhs "/" ast-mset boa ]] + | OrExpr:e "%=" Expr:rhs => [[ e rhs "%" ast-mset boa ]] + | OrExpr:e "&&=" Expr:rhs => [[ e rhs "&&" ast-mset boa ]] + | OrExpr:e "||=" Expr:rhs => [[ e rhs "||" ast-mset boa ]] + | OrExpr:e => [[ e ]] + +OrExpr = OrExpr:x "||" AndExpr:y => [[ x y "||" ast-binop boa ]] + | AndExpr +AndExpr = AndExpr:x "&&" EqExpr:y => [[ x y "&&" ast-binop boa ]] + | EqExpr +EqExpr = EqExpr:x "==" RelExpr:y => [[ x y "==" ast-binop boa ]] + | EqExpr:x "!=" RelExpr:y => [[ x y "!=" ast-binop boa ]] + | EqExpr:x "===" RelExpr:y => [[ x y "===" ast-binop boa ]] + | EqExpr:x "!==" RelExpr:y => [[ x y "!==" ast-binop boa ]] + | RelExpr +RelExpr = RelExpr:x ">" AddExpr:y => [[ x y ">" ast-binop boa ]] + | RelExpr:x ">=" AddExpr:y => [[ x y ">=" ast-binop boa ]] + | RelExpr:x "<" AddExpr:y => [[ x y "<" ast-binop boa ]] + | RelExpr:x "<=" AddExpr:y => [[ x y "<=" ast-binop boa ]] + | RelExpr:x "instanceof" AddExpr:y => [[ x y "instanceof" ast-binop boa ]] + | AddExpr +AddExpr = AddExpr:x "+" MulExpr:y => [[ x y "+" ast-binop boa ]] + | AddExpr:x "-" MulExpr:y => [[ x y "-" ast-binop boa ]] + | MulExpr +MulExpr = MulExpr:x "*" MulExpr:y => [[ x y "*" ast-binop boa ]] + | MulExpr:x "/" MulExpr:y => [[ x y "/" ast-binop boa ]] + | MulExpr:x "%" MulExpr:y => [[ x y "%" ast-binop boa ]] + | Unary +Unary = "-" Postfix:p => [[ p "-" ast-unop boa ]] + | "+" Postfix:p => [[ p ]] + | "++" Postfix:p => [[ p "++" ast-preop boa ]] + | "--" Postfix:p => [[ p "--" ast-preop boa ]] + | "!" Postfix:p => [[ p "!" ast-unop boa ]] + | Postfix +Postfix = PrimExpr:p SpacesNoNl "++" => [[ p "++" ast-postop boa ]] + | PrimExpr:p SpacesNoNl "--" => [[ p "--" ast-postop boa ]] + | PrimExpr +Args = (Expr ("," Expr => [[ second ]])* => [[ first2 swap prefix ]])? +PrimExpr = PrimExpr:p "[" Expr:i "]" => [[ i p ast-getp boa ]] + | PrimExpr:p "." Name:m "(" Args:as ")" => [[ m p as ast-send boa ]] + | PrimExpr:p "." Name:f => [[ f p ast-getp boa ]] + | PrimExpr:p "(" Args:as ")" => [[ p as ast-call boa ]] + | PrimExprHd +PrimExprHd = "(" Expr:e ")" => [[ e ]] + | "this" => [[ ast-this boa ]] + | Name => [[ ast-get boa ]] + | Number => [[ ast-number boa ]] + | String => [[ ast-string boa ]] + | "function" FuncRest:fr => [[ fr ]] + | "new" Name:n "(" Args:as ")" => [[ n as ast-new boa ]] + | "[" Args:es "]" => [[ es ast-array boa ]] + | Json +JsonBindings = (JsonBinding ("," JsonBinding => [[ second ]])* => [[ first2 swap prefix ]])? +Json = "{" JsonBindings:bs "}" => [[ bs ast-json boa ]] +JsonBinding = JsonPropName:n ":" Expr:v => [[ n v ast-binding boa ]] +JsonPropName = Name | Number | String +Formal = Spaces Name +Formals = (Formal ("," Formal => [[ second ]])* => [[ first2 swap prefix ]])? +FuncRest = "(" Formals:fs ")" "{" SrcElems:body "}" => [[ fs body ast-func boa ]] +Sc = SpacesNoNl ("\n" | &("}"))| ";" +Binding = Name:n "=" Expr:v => [[ n v ast-var boa ]] + | Name:n => [[ n "undefined" ast-get boa ast-var boa ]] +Block = "{" SrcElems:ss "}" => [[ ss ]] +Bindings = (Binding ("," Binding => [[ second ]])* => [[ first2 swap prefix ]])? +For1 = "var" Binding => [[ second ]] + | Expr + | Spaces => [[ "undefined" ast-get boa ]] +For2 = Expr + | Spaces => [[ "true" ast-get boa ]] +For3 = Expr + | Spaces => [[ "undefined" ast-get boa ]] +ForIn1 = "var" Name:n => [[ n "undefined" ast-get boa ast-var boa ]] + | Expr +Switch1 = "case" Expr:c ":" SrcElems:cs => [[ c cs ast-case boa ]] + | "default" ":" SrcElems:cs => [[ cs ast-default boa ]] +SwitchBody = Switch1* +Finally = "finally" Block:b => [[ b ]] + | Spaces => [[ "undefined" ast-get boa ]] +Stmt = Block + | "var" Bindings:bs Sc => [[ bs ast-begin boa ]] + | "if" "(" Expr:c ")" Stmt:t "else" Stmt:f => [[ c t f ast-if boa ]] + | "if" "(" Expr:c ")" Stmt:t => [[ c t "undefined" ast-get boa ast-if boa ]] + | "while" "(" Expr:c ")" Stmt:s => [[ c s ast-while boa ]] + | "do" Stmt:s "while" "(" Expr:c ")" Sc => [[ s c ast-do-while boa ]] + | "for" "(" For1:i ";" For2:c ";" For3:u ")" Stmt:s => [[ i c u s ast-for boa ]] + | "for" "(" ForIn1:v "in" Expr:e ")" Stmt:s => [[ v e s ast-for-in boa ]] + | "switch" "(" Expr:e ")" "{" SwitchBody:cs "}" => [[ e cs ast-switch boa ]] + | "break" Sc => [[ ast-break boa ]] + | "continue" Sc => [[ ast-continue boa ]] + | "throw" SpacesNoNl Expr:e Sc => [[ e ast-throw boa ]] + | "try" Block:t "catch" "(" Name:e ")" Block:c Finally:f => [[ t e c f ast-try boa ]] + | "return" Expr:e Sc => [[ e ast-return boa ]] + | "return" Sc => [[ "undefined" ast-get boa ast-return boa ]] + | Expr:e Sc => [[ e ]] + | ";" => [[ "undefined" ast-get boa ]] +SrcElem = "function" Name:n FuncRest:f => [[ n f ast-var boa ]] + | Stmt +SrcElems = SrcElem* => [[ ast-begin boa ]] +TopLevel = SrcElems Spaces +;EBNF \ No newline at end of file diff --git a/extra/peg/javascript/parser/summary.txt b/extra/peg/javascript/parser/summary.txt new file mode 100644 index 0000000000..bae5a461d2 --- /dev/null +++ b/extra/peg/javascript/parser/summary.txt @@ -0,0 +1 @@ +JavaScript Parser diff --git a/extra/peg/javascript/parser/tags.txt b/extra/peg/javascript/parser/tags.txt new file mode 100644 index 0000000000..c2aac2932f --- /dev/null +++ b/extra/peg/javascript/parser/tags.txt @@ -0,0 +1,3 @@ +text +javascript +parsing diff --git a/extra/peg/javascript/summary.txt b/extra/peg/javascript/summary.txt new file mode 100644 index 0000000000..12f092dcf7 --- /dev/null +++ b/extra/peg/javascript/summary.txt @@ -0,0 +1 @@ +JavaScript parser diff --git a/extra/peg/javascript/tags.txt b/extra/peg/javascript/tags.txt new file mode 100644 index 0000000000..c2aac2932f --- /dev/null +++ b/extra/peg/javascript/tags.txt @@ -0,0 +1,3 @@ +text +javascript +parsing diff --git a/extra/peg/javascript/tokenizer/authors.txt b/extra/peg/javascript/tokenizer/authors.txt new file mode 100644 index 0000000000..44b06f94bc --- /dev/null +++ b/extra/peg/javascript/tokenizer/authors.txt @@ -0,0 +1 @@ +Chris Double diff --git a/extra/peg/javascript/tokenizer/summary.txt b/extra/peg/javascript/tokenizer/summary.txt new file mode 100644 index 0000000000..ce94386ed9 --- /dev/null +++ b/extra/peg/javascript/tokenizer/summary.txt @@ -0,0 +1 @@ +Tokenizer for JavaScript language diff --git a/extra/peg/javascript/tokenizer/tags.txt b/extra/peg/javascript/tokenizer/tags.txt new file mode 100644 index 0000000000..c2aac2932f --- /dev/null +++ b/extra/peg/javascript/tokenizer/tags.txt @@ -0,0 +1,3 @@ +text +javascript +parsing diff --git a/extra/peg/javascript/tokenizer/tokenizer-tests.factor b/extra/peg/javascript/tokenizer/tokenizer-tests.factor new file mode 100644 index 0000000000..1300b3c9c7 --- /dev/null +++ b/extra/peg/javascript/tokenizer/tokenizer-tests.factor @@ -0,0 +1,23 @@ +! Copyright (C) 2008 Chris Double. +! See http://factorcode.org/license.txt for BSD license. +! +USING: kernel tools.test peg peg.javascript.ast peg.javascript.tokenizer accessors ; +IN: peg.javascript.tokenizer.tests + +\ tokenizer must-infer + +{ + V{ + T{ ast-number f 123 } + ";" + T{ ast-string f "hello" } + ";" + T{ ast-name f "foo" } + "(" + T{ ast-name f "x" } + ")" + ";" + } +} [ + "123; 'hello'; foo(x);" tokenizer ast>> +] unit-test diff --git a/extra/peg/javascript/tokenizer/tokenizer.factor b/extra/peg/javascript/tokenizer/tokenizer.factor new file mode 100644 index 0000000000..d62bb9395b --- /dev/null +++ b/extra/peg/javascript/tokenizer/tokenizer.factor @@ -0,0 +1,68 @@ +! Copyright (C) 2008 Chris Double. +! See http://factorcode.org/license.txt for BSD license. +USING: kernel sequences strings arrays math.parser peg peg.ebnf peg.javascript.ast ; +IN: peg.javascript.tokenizer + +#! Grammar for JavaScript. Based on OMeta-JS example from: +#! http://jarrett.cs.ucla.edu/ometa-js/#JavaScript_Compiler + +EBNF: tokenizer +Letter = [a-zA-Z] +Digit = [0-9] +Digits = Digit+ +SingleLineComment = "//" (!("\n") .)* "\n" => [[ ignore ]] +MultiLineComment = "/*" (!("*/") .)* "*/" => [[ ignore ]] +Space = " " | "\t" | "\r" | "\n" | SingleLineComment | MultiLineComment +Spaces = Space* => [[ ignore ]] +NameFirst = Letter | "$" | "_" +NameRest = NameFirst | Digit +iName = NameFirst NameRest* => [[ first2 swap prefix >string ]] +Keyword = ("break" + | "case" + | "catch" + | "continue" + | "default" + | "delete" + | "do" + | "else" + | "finally" + | "for" + | "function" + | "if" + | "in" + | "instanceof" + | "new" + | "return" + | "switch" + | "this" + | "throw" + | "try" + | "typeof" + | "var" + | "void" + | "while" + | "with") +Name = !(Keyword) (iName):n => [[ n ast-name boa ]] +Number = Digits:ws '.' Digits:fs => [[ ws "." fs 3array concat >string string>number ast-number boa ]] + | Digits => [[ >string string>number ast-number boa ]] + +EscapeChar = "\\n" => [[ 10 ]] + | "\\r" => [[ 13 ]] + | "\\t" => [[ 9 ]] +StringChars1 = (EscapeChar | !('"""') .)* => [[ >string ]] +StringChars2 = (EscapeChar | !('"') .)* => [[ >string ]] +StringChars3 = (EscapeChar | !("'") .)* => [[ >string ]] +Str = '"""' StringChars1:cs '"""' => [[ cs ast-string boa ]] + | '"' StringChars2:cs '"' => [[ cs ast-string boa ]] + | "'" StringChars3:cs "'" => [[ cs ast-string boa ]] +RegExpBody = (!("/" | "\n" | "\r") .)* => [[ >string ]] +RegExp = "/" RegExpBody:r "/" => [[ r ast-regexp boa ]] +Special = "(" | ")" | "{" | "}" | "[" | "]" | "," | ";" + | "?" | ":" | "!==" | "~=" | "===" | "==" | "=" | ">=" + | ">" | "<=" | "<" | "++" | "+=" | "+" | "--" | "-=" + | "-" | "*=" | "*" | "/=" | "/" | "%=" | "%" | "&&=" + | "&&" | "||=" | "||" | "." | "!" +Tok = Spaces (Name | Keyword | Number | Str | RegExp | Special ) +Toks = Tok* Spaces +;EBNF + From fc7baebacbcedc177b16c5377991e156e9f62d26 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Thu, 19 Jun 2008 00:51:47 +1200 Subject: [PATCH 20/47] Fix handling of _ and $ in Javascript names --- extra/peg/javascript/tokenizer/tokenizer.factor | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/extra/peg/javascript/tokenizer/tokenizer.factor b/extra/peg/javascript/tokenizer/tokenizer.factor index d62bb9395b..420abff442 100644 --- a/extra/peg/javascript/tokenizer/tokenizer.factor +++ b/extra/peg/javascript/tokenizer/tokenizer.factor @@ -6,6 +6,8 @@ IN: peg.javascript.tokenizer #! Grammar for JavaScript. Based on OMeta-JS example from: #! http://jarrett.cs.ucla.edu/ometa-js/#JavaScript_Compiler +USE: prettyprint + EBNF: tokenizer Letter = [a-zA-Z] Digit = [0-9] @@ -14,7 +16,7 @@ SingleLineComment = "//" (!("\n") .)* "\n" => [[ ignore ]] MultiLineComment = "/*" (!("*/") .)* "*/" => [[ ignore ]] Space = " " | "\t" | "\r" | "\n" | SingleLineComment | MultiLineComment Spaces = Space* => [[ ignore ]] -NameFirst = Letter | "$" | "_" +NameFirst = Letter | "$" => [[ CHAR: $ ]] | "_" => [[ CHAR: _ ]] NameRest = NameFirst | Digit iName = NameFirst NameRest* => [[ first2 swap prefix >string ]] Keyword = ("break" From c26d87e11788c107d7467f38ae2a0bded3666c05 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Thu, 19 Jun 2008 01:01:46 +1200 Subject: [PATCH 21/47] Fix handling of JavasScript names which partially match reserved words --- extra/peg/javascript/tokenizer/tokenizer.factor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/peg/javascript/tokenizer/tokenizer.factor b/extra/peg/javascript/tokenizer/tokenizer.factor index 420abff442..70fabb10f6 100644 --- a/extra/peg/javascript/tokenizer/tokenizer.factor +++ b/extra/peg/javascript/tokenizer/tokenizer.factor @@ -44,7 +44,7 @@ Keyword = ("break" | "void" | "while" | "with") -Name = !(Keyword) (iName):n => [[ n ast-name boa ]] +Name = iName !(Keyword) => [[ ast-name boa ]] Number = Digits:ws '.' Digits:fs => [[ ws "." fs 3array concat >string string>number ast-number boa ]] | Digits => [[ >string string>number ast-number boa ]] From 179ea21c11d95f257a5f3172dab4ffd7fa91ae5a Mon Sep 17 00:00:00 2001 From: Chris Double Date: Thu, 19 Jun 2008 01:10:44 +1200 Subject: [PATCH 22/47] Add a couple of failing peg.ebnf tests --- extra/peg/ebnf/ebnf-tests.factor | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/extra/peg/ebnf/ebnf-tests.factor b/extra/peg/ebnf/ebnf-tests.factor index 73db626685..ed38d37421 100644 --- a/extra/peg/ebnf/ebnf-tests.factor +++ b/extra/peg/ebnf/ebnf-tests.factor @@ -399,6 +399,16 @@ main = Primary "number=digit+:n 'a'" 'ebnf' parse remaining>> length zero? ] unit-test +{ t } [ + "foo=(name):n !(keyword) => [[ n ]]" 'rule' parse ast>> + "foo=name:n !(keyword) => [[ n ]]" 'rule' parse ast>> = +] unit-test + +{ t } [ + "foo=!(keyword) (name):n => [[ n ]]" 'rule' parse ast>> + "foo=!(keyword) name:n => [[ n ]]" 'rule' parse ast>> = +] unit-test + << EBNF: parser1 foo='a' @@ -431,4 +441,4 @@ foo= 'd' { V{ CHAR: a "d" } } [ "ad" parser4 ast>> -] unit-test \ No newline at end of file +] unit-test From fcd1e39834052d7df4548d4c411870a8417ec10c Mon Sep 17 00:00:00 2001 From: Chris Double Date: Thu, 19 Jun 2008 01:20:33 +1200 Subject: [PATCH 23/47] More JavaScript fixes for keyword handling --- extra/peg/javascript/tokenizer/tokenizer.factor | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extra/peg/javascript/tokenizer/tokenizer.factor b/extra/peg/javascript/tokenizer/tokenizer.factor index 70fabb10f6..a1cff8a3db 100644 --- a/extra/peg/javascript/tokenizer/tokenizer.factor +++ b/extra/peg/javascript/tokenizer/tokenizer.factor @@ -43,8 +43,8 @@ Keyword = ("break" | "var" | "void" | "while" - | "with") -Name = iName !(Keyword) => [[ ast-name boa ]] + | "with") !(NameRest) +Name = !(Keyword) iName => [[ ast-name boa ]] Number = Digits:ws '.' Digits:fs => [[ ws "." fs 3array concat >string string>number ast-number boa ]] | Digits => [[ >string string>number ast-number boa ]] From d58a085598e930566c003aaff406e5996f91c73f Mon Sep 17 00:00:00 2001 From: Chris Double Date: Thu, 19 Jun 2008 01:24:17 +1200 Subject: [PATCH 24/47] Add additional javascript test --- extra/peg/javascript/parser/parser-tests.factor | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/extra/peg/javascript/parser/parser-tests.factor b/extra/peg/javascript/parser/parser-tests.factor index 933d4cf10e..6741e059f9 100644 --- a/extra/peg/javascript/parser/parser-tests.factor +++ b/extra/peg/javascript/parser/parser-tests.factor @@ -2,7 +2,7 @@ ! See http://factorcode.org/license.txt for BSD license. ! USING: kernel tools.test peg peg.javascript.ast peg.javascript.tokenizer - peg.javascript.parser accessors ; + peg.javascript.parser accessors multiline sequences math ; IN: peg.javascript.parser.tests \ javascript must-infer @@ -24,4 +24,14 @@ IN: peg.javascript.parser.tests } } [ "123; 'hello'; foo(x);" tokenizer ast>> javascript ast>> -] unit-test \ No newline at end of file +] unit-test + +{ t } [ +<" +function foldl(f, initial, seq) { + for(var i=0; i< seq.length; ++i) + initial = f(initial, seq[i]); + return initial; +} +"> tokenizer ast>> javascript remaining>> length zero? +] unit-test From d5e5e47944736585195b66aab7e997b0e7c9a666 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Thu, 19 Jun 2008 01:39:51 +1200 Subject: [PATCH 25/47] Fix bug in javascript automatic semicolon insertion rule --- extra/peg/javascript/parser/parser-tests.factor | 11 +++++++++++ extra/peg/javascript/parser/parser.factor | 3 ++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/extra/peg/javascript/parser/parser-tests.factor b/extra/peg/javascript/parser/parser-tests.factor index 6741e059f9..ec7a30845f 100644 --- a/extra/peg/javascript/parser/parser-tests.factor +++ b/extra/peg/javascript/parser/parser-tests.factor @@ -35,3 +35,14 @@ function foldl(f, initial, seq) { } "> tokenizer ast>> javascript remaining>> length zero? ] unit-test + +{ t } [ +<" +ParseState.prototype.from = function(index) { + var r = new ParseState(this.input, this.index + index); + r.cache = this.cache; + r.length = this.length - index; + return r; +} +"> tokenizer ast>> javascript remaining>> length zero? +] unit-test diff --git a/extra/peg/javascript/parser/parser.factor b/extra/peg/javascript/parser/parser.factor index a38cf4aea8..45aa0f022c 100644 --- a/extra/peg/javascript/parser/parser.factor +++ b/extra/peg/javascript/parser/parser.factor @@ -7,6 +7,7 @@ IN: peg.javascript.parser #! http://jarrett.cs.ucla.edu/ometa-js/#JavaScript_Compiler EBNF: javascript +End = !(.) Space = " " | "\t" | "\n" Spaces = Space* => [[ ignore ]] Name = . ?[ ast-name? ]? => [[ value>> ]] @@ -78,7 +79,7 @@ JsonPropName = Name | Number | String Formal = Spaces Name Formals = (Formal ("," Formal => [[ second ]])* => [[ first2 swap prefix ]])? FuncRest = "(" Formals:fs ")" "{" SrcElems:body "}" => [[ fs body ast-func boa ]] -Sc = SpacesNoNl ("\n" | &("}"))| ";" +Sc = SpacesNoNl ("\n" | &("}") | End)| ";" Binding = Name:n "=" Expr:v => [[ n v ast-var boa ]] | Name:n => [[ n "undefined" ast-get boa ast-var boa ]] Block = "{" SrcElems:ss "}" => [[ ss ]] From 2b2ede0a89411421649af182bb69439bc6424b17 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Thu, 19 Jun 2008 01:45:09 +1200 Subject: [PATCH 26/47] Add a javascript sc test --- extra/peg/javascript/parser/parser-tests.factor | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/extra/peg/javascript/parser/parser-tests.factor b/extra/peg/javascript/parser/parser-tests.factor index ec7a30845f..d911a27285 100644 --- a/extra/peg/javascript/parser/parser-tests.factor +++ b/extra/peg/javascript/parser/parser-tests.factor @@ -26,6 +26,14 @@ IN: peg.javascript.parser.tests "123; 'hello'; foo(x);" tokenizer ast>> javascript ast>> ] unit-test +{ t } [ +<" +var x=5 +var y=10 +"> tokenizer ast>> javascript remaining>> length zero? +] unit-test + + { t } [ <" function foldl(f, initial, seq) { @@ -46,3 +54,4 @@ ParseState.prototype.from = function(index) { } "> tokenizer ast>> javascript remaining>> length zero? ] unit-test + From 04453b242157e5966971fd86dc62c86ab6a56757 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Thu, 19 Jun 2008 14:23:18 +1200 Subject: [PATCH 27/47] Fix (foo):n usage --- extra/peg/ebnf/ebnf.factor | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/extra/peg/ebnf/ebnf.factor b/extra/peg/ebnf/ebnf.factor index 2ee0958051..d982d73229 100644 --- a/extra/peg/ebnf/ebnf.factor +++ b/extra/peg/ebnf/ebnf.factor @@ -232,14 +232,18 @@ DEFER: 'choice' : ('sequence') ( -- parser ) #! A sequence of terminals and non-terminals, including #! groupings of those. - [ - 'ensure-not' sp , - 'ensure' sp , - 'element' sp , - 'group' sp , - 'repeat0' sp , - 'repeat1' sp , - 'optional' sp , + [ + [ + 'ensure-not' sp , + 'ensure' sp , + 'element' sp , + 'group' sp , + 'repeat0' sp , + 'repeat1' sp , + 'optional' sp , + ] choice* + [ dup , ":" syntax , "a-zA-Z" range-pattern repeat1 [ >string ] action , ] seq* [ first2 ] action , + , ] choice* ; : 'action' ( -- parser ) From 3dc3a6f8996d46f3717302407b197d3b0aa6da1a Mon Sep 17 00:00:00 2001 From: Chris Double Date: Thu, 19 Jun 2008 14:31:14 +1200 Subject: [PATCH 28/47] Remove obsolete ebnf stuff --- extra/peg/ebnf/ebnf.factor | 2 -- 1 file changed, 2 deletions(-) diff --git a/extra/peg/ebnf/ebnf.factor b/extra/peg/ebnf/ebnf.factor index d982d73229..08ac24e535 100644 --- a/extra/peg/ebnf/ebnf.factor +++ b/extra/peg/ebnf/ebnf.factor @@ -266,8 +266,6 @@ DEFER: 'choice' : 'actioned-sequence' ( -- parser ) [ [ 'sequence' , "=>" syntax , 'action' , ] seq* [ first2 ] action , - [ 'sequence' , ":" syntax , "a-zA-Z" range-pattern repeat1 [ >string ] action , "=>" syntax , 'action' , ] seq* [ first3 >r r> ] action , - [ 'sequence' , ":" syntax , "a-zA-Z" range-pattern repeat1 [ >string ] action , ] seq* [ first2 ] action , 'sequence' , ] choice* ; From c92224f5b9b94ea2bdc1224c10cf640921c283df Mon Sep 17 00:00:00 2001 From: Chris Double Date: Thu, 19 Jun 2008 14:34:09 +1200 Subject: [PATCH 29/47] Put '..' around parser error messages --- extra/peg/ebnf/ebnf.factor | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extra/peg/ebnf/ebnf.factor b/extra/peg/ebnf/ebnf.factor index 08ac24e535..2aec8b9aea 100644 --- a/extra/peg/ebnf/ebnf.factor +++ b/extra/peg/ebnf/ebnf.factor @@ -416,7 +416,7 @@ M: ebnf-terminal (transform) ( ast -- parser ) M: ebnf-foreign (transform) ( ast -- parser ) dup word>> search - [ "Foreign word " swap word>> append " not found" append throw ] unless* + [ "Foreign word '" swap word>> append "' not found" append throw ] unless* swap rule>> [ main ] unless* dupd swap rule [ nip ] [ @@ -425,7 +425,7 @@ M: ebnf-foreign (transform) ( ast -- parser ) : parser-not-found ( name -- * ) [ - "Parser " % % " not found." % + "Parser '" % % "' not found." % ] "" make throw ; M: ebnf-non-terminal (transform) ( ast -- parser ) From 9b7e2bacc960c03ff28780b0a469018a098b5540 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Thu, 19 Jun 2008 14:44:13 +1200 Subject: [PATCH 30/47] Throw an error if there are duplicate rules in ebnf --- extra/peg/ebnf/ebnf.factor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/peg/ebnf/ebnf.factor b/extra/peg/ebnf/ebnf.factor index 2aec8b9aea..e78757be34 100644 --- a/extra/peg/ebnf/ebnf.factor +++ b/extra/peg/ebnf/ebnf.factor @@ -305,7 +305,7 @@ M: ebnf (transform) ( ast -- parser ) M: ebnf-rule (transform) ( ast -- parser ) dup elements>> (transform) [ - swap symbol>> set + swap symbol>> dup get [ "Rule '" over append "' defined more than once" append throw ] [ set ] if ] keep ; M: ebnf-sequence (transform) ( ast -- parser ) From 001e866b13e8fbc65d6d704b2e9794fca8eb9dc6 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Fri, 20 Jun 2008 11:49:08 +1200 Subject: [PATCH 31/47] Add yet another failing ebnf unit test --- extra/peg/ebnf/ebnf-tests.factor | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/extra/peg/ebnf/ebnf-tests.factor b/extra/peg/ebnf/ebnf-tests.factor index ed38d37421..f613002fdf 100644 --- a/extra/peg/ebnf/ebnf-tests.factor +++ b/extra/peg/ebnf/ebnf-tests.factor @@ -442,3 +442,7 @@ foo= 'd' { V{ CHAR: a "d" } } [ "ad" parser4 ast>> ] unit-test + +{ V{ "a" "\n" } } [ + "a\n" [EBNF foo='a' '\n' => [[ drop '\n' ]] EBNF] call ast>> +] unit-test \ No newline at end of file From 91768f171c5bd8721d2a1c20a75478b9692857b0 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Fri, 20 Jun 2008 11:53:07 +1200 Subject: [PATCH 32/47] Rule 'x' defined more than once unit test --- extra/peg/ebnf/ebnf-tests.factor | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/extra/peg/ebnf/ebnf-tests.factor b/extra/peg/ebnf/ebnf-tests.factor index f613002fdf..1824c0342a 100644 --- a/extra/peg/ebnf/ebnf-tests.factor +++ b/extra/peg/ebnf/ebnf-tests.factor @@ -445,4 +445,8 @@ foo= 'd' { V{ "a" "\n" } } [ "a\n" [EBNF foo='a' '\n' => [[ drop '\n' ]] EBNF] call ast>> +] unit-test + +{ t } [ + [EBNF foo='a' foo='b' EBNF] drop t ] unit-test \ No newline at end of file From 6bd761e4609a1ac2048e7bde627c4ccc384adc4a Mon Sep 17 00:00:00 2001 From: Chris Double Date: Fri, 20 Jun 2008 12:35:33 +1200 Subject: [PATCH 33/47] Another breaking unit test for ebnf --- extra/peg/ebnf/ebnf-tests.factor | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/extra/peg/ebnf/ebnf-tests.factor b/extra/peg/ebnf/ebnf-tests.factor index 1824c0342a..0a16fc8007 100644 --- a/extra/peg/ebnf/ebnf-tests.factor +++ b/extra/peg/ebnf/ebnf-tests.factor @@ -2,7 +2,7 @@ ! See http://factorcode.org/license.txt for BSD license. ! USING: kernel tools.test peg peg.ebnf words math math.parser - sequences accessors peg.parsers ; + sequences accessors peg.parsers parser namespaces ; IN: peg.ebnf.tests { T{ ebnf-non-terminal f "abc" } } [ @@ -443,10 +443,17 @@ foo= 'd' "ad" parser4 ast>> ] unit-test -{ V{ "a" "\n" } } [ - "a\n" [EBNF foo='a' '\n' => [[ drop '\n' ]] EBNF] call ast>> +{ t } [ + "USING: kernel peg.ebnf ; [EBNF foo='a' '\n' => [[ drop '\n' ]] EBNF]" eval drop t ] unit-test +[ + "USING: peg.ebnf ; [EBNF foo='a' foo='b' EBNF]" eval +] must-fail + + { t } [ - [EBNF foo='a' foo='b' EBNF] drop t + #! Rule lookup occurs in a namespace. This causes an incorrect duplicate rule + #! if a var in a namespace is set. This unit test is to remind me to fix this. + [ "fail" "foo" set "foo='a'" 'ebnf' parse ast>> transform drop t ] with-scope ] unit-test \ No newline at end of file From d22a24a90eef38975b6eb5bacfc010477730d453 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Fri, 20 Jun 2008 14:13:50 +1200 Subject: [PATCH 34/47] Fix some failing ebnf unit tests --- extra/peg/ebnf/ebnf-tests.factor | 2 +- extra/peg/ebnf/ebnf.factor | 31 +++++++++++++++++++------------ 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/extra/peg/ebnf/ebnf-tests.factor b/extra/peg/ebnf/ebnf-tests.factor index 0a16fc8007..e3c6586c89 100644 --- a/extra/peg/ebnf/ebnf-tests.factor +++ b/extra/peg/ebnf/ebnf-tests.factor @@ -444,7 +444,7 @@ foo= 'd' ] unit-test { t } [ - "USING: kernel peg.ebnf ; [EBNF foo='a' '\n' => [[ drop '\n' ]] EBNF]" eval drop t + "USING: kernel peg.ebnf ; [EBNF foo='a' '\n' => [[ drop \"\n\" ]] EBNF]" eval drop t ] unit-test [ diff --git a/extra/peg/ebnf/ebnf.factor b/extra/peg/ebnf/ebnf.factor index e78757be34..cba48f5892 100644 --- a/extra/peg/ebnf/ebnf.factor +++ b/extra/peg/ebnf/ebnf.factor @@ -63,6 +63,20 @@ C: ebnf #! begin and end. [ syntax ] 2dip syntax pack ; +: replace-escapes ( string -- string ) + [ + "\\t" token [ drop "\t" ] action , + "\\n" token [ drop "\n" ] action , + "\\r" token [ drop "\r" ] action , + ] choice* replace ; + +: insert-escapes ( string -- string ) + [ + "\t" token [ drop "\\t" ] action , + "\n" token [ drop "\\n" ] action , + "\r" token [ drop "\\r" ] action , + ] choice* replace ; + : 'identifier' ( -- parser ) #! Return a parser that parses an identifer delimited by #! a quotation character. The quotation can be single @@ -71,7 +85,7 @@ C: ebnf [ [ CHAR: " = not ] satisfy repeat1 "\"" "\"" surrounded-by , [ CHAR: ' = not ] satisfy repeat1 "'" "'" surrounded-by , - ] choice* [ >string ] action ; + ] choice* [ >string replace-escapes ] action ; : 'non-terminal' ( -- parser ) #! A non-terminal is the name of another rule. It can @@ -401,11 +415,11 @@ M: object build-locals ( code ast -- ) } cond ; M: ebnf-action (transform) ( ast -- parser ) - [ parser>> (transform) ] [ code>> ] [ parser>> ] tri build-locals + [ parser>> (transform) ] [ code>> insert-escapes ] [ parser>> ] tri build-locals string-lines parse-lines check-action-effect action ; M: ebnf-semantic (transform) ( ast -- parser ) - [ parser>> (transform) ] [ code>> ] [ parser>> ] tri build-locals + [ parser>> (transform) ] [ code>> insert-escapes ] [ parser>> ] tri build-locals string-lines parse-lines semantic ; M: ebnf-var (transform) ( ast -- parser ) @@ -453,17 +467,10 @@ M: ebnf-non-terminal (transform) ( ast -- parser ) parse-result-ast transform dup dup parser [ main swap at compile ] with-variable [ compiled-parse ] curry [ with-scope ] curry ; -: replace-escapes ( string -- string ) - [ - "\\t" token [ drop "\t" ] action , - "\\n" token [ drop "\n" ] action , - "\\r" token [ drop "\r" ] action , - ] choice* replace ; - -: [EBNF "EBNF]" parse-multiline-string replace-escapes ebnf>quot nip parsed ; parsing +: [EBNF "EBNF]" parse-multiline-string ebnf>quot nip parsed ; parsing : EBNF: CREATE-WORD dup - ";EBNF" parse-multiline-string replace-escapes + ";EBNF" parse-multiline-string ebnf>quot swapd 1 1 define-declared "ebnf-parser" set-word-prop ; parsing From 7239c4d79f590997ddfa25ce3e6423c568b5cb62 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Fri, 20 Jun 2008 15:14:30 +1200 Subject: [PATCH 35/47] Make replace-escapes infer and fix remaining unit test breakage in ebnf --- extra/peg/ebnf/ebnf-tests.factor | 2 +- extra/peg/ebnf/ebnf.factor | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/extra/peg/ebnf/ebnf-tests.factor b/extra/peg/ebnf/ebnf-tests.factor index e3c6586c89..5a4ecc5c2f 100644 --- a/extra/peg/ebnf/ebnf-tests.factor +++ b/extra/peg/ebnf/ebnf-tests.factor @@ -448,7 +448,7 @@ foo= 'd' ] unit-test [ - "USING: peg.ebnf ; [EBNF foo='a' foo='b' EBNF]" eval + "USING: peg.ebnf ; [EBNF foo='a' foo='b' EBNF]" eval drop ] must-fail diff --git a/extra/peg/ebnf/ebnf.factor b/extra/peg/ebnf/ebnf.factor index cba48f5892..8f36218b61 100644 --- a/extra/peg/ebnf/ebnf.factor +++ b/extra/peg/ebnf/ebnf.factor @@ -63,12 +63,17 @@ C: ebnf #! begin and end. [ syntax ] 2dip syntax pack ; -: replace-escapes ( string -- string ) +#! Don't want to use 'replace' in an action since replace doesn't infer. +#! Do the compilation of the peg at parse time and call (replace). +PEG: escaper ( string -- ast ) [ "\\t" token [ drop "\t" ] action , "\\n" token [ drop "\n" ] action , "\\r" token [ drop "\r" ] action , - ] choice* replace ; + ] choice* any-char-parser 2array choice repeat0 ; + +: replace-escapes ( string -- string ) + escaper sift [ [ tree-write ] each ] with-string-writer ; : insert-escapes ( string -- string ) [ @@ -319,7 +324,11 @@ M: ebnf (transform) ( ast -- parser ) M: ebnf-rule (transform) ( ast -- parser ) dup elements>> (transform) [ - swap symbol>> dup get [ "Rule '" over append "' defined more than once" append throw ] [ set ] if + swap symbol>> dup get { [ tuple? ] [ delegate parser? ] } 1&& [ + "Rule '" over append "' defined more than once" append throw + ] [ + set + ] if ] keep ; M: ebnf-sequence (transform) ( ast -- parser ) From b6b5f12732914b36a0939f1f0221eaf29f9867d0 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Fri, 20 Jun 2008 22:29:53 +1200 Subject: [PATCH 36/47] Provide ability to plug in tokenizers to ebnf parsers --- extra/peg/ebnf/ebnf.factor | 49 ++++++++++++++++--- .../peg/javascript/tokenizer/tokenizer.factor | 2 +- 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/extra/peg/ebnf/ebnf.factor b/extra/peg/ebnf/ebnf.factor index 8f36218b61..921ba7be67 100644 --- a/extra/peg/ebnf/ebnf.factor +++ b/extra/peg/ebnf/ebnf.factor @@ -11,6 +11,31 @@ IN: peg.ebnf #! Given an EBNF word produced from EBNF: return the EBNF rule "ebnf-parser" word-prop at ; +TUPLE: tokenizer any one many ; + +: default-tokenizer ( -- tokenizer ) + T{ tokenizer f + [ [ drop t ] satisfy ] + [ token ] + [ [ = ] curry satisfy ] + } ; + +: parser-tokenizer ( parser -- tokenizer ) + 1quotation [ [ = ] curry satisfy ] dup tokenizer boa ; + +: rule-tokenizer ( name word -- tokenizer ) + rule parser-tokenizer ; + +: tokenizer ( -- word ) + \ tokenizer get [ default-tokenizer ] unless* ; + +: reset-tokenizer ( -- ) + default-tokenizer \ tokenizer set-global ; + +: TOKENIZER: + scan search [ "Tokenizer not found" throw ] unless* + execute \ tokenizer set-global ; parsing + TUPLE: ebnf-non-terminal symbol ; TUPLE: ebnf-terminal symbol ; TUPLE: ebnf-foreign word rule ; @@ -344,7 +369,7 @@ M: ebnf-choice (transform) ( ast -- parser ) options>> [ (transform) ] map choice ; M: ebnf-any-character (transform) ( ast -- parser ) - drop any-char ; + drop [ tokenizer any>> call ] box ; M: ebnf-range (transform) ( ast -- parser ) pattern>> range-pattern ; @@ -435,7 +460,7 @@ M: ebnf-var (transform) ( ast -- parser ) parser>> (transform) ; M: ebnf-terminal (transform) ( ast -- parser ) - symbol>> [ token ] keep [ = ] curry satisfy 2choice ; + symbol>> [ tokenizer one>> call ] curry box ; M: ebnf-foreign (transform) ( ast -- parser ) dup word>> search @@ -476,10 +501,22 @@ M: ebnf-non-terminal (transform) ( ast -- parser ) parse-result-ast transform dup dup parser [ main swap at compile ] with-variable [ compiled-parse ] curry [ with-scope ] curry ; -: [EBNF "EBNF]" parse-multiline-string ebnf>quot nip parsed ; parsing +: [EBNF + scan { + { "+" [ scan-word execute "" swap ] } + [ " " append default-tokenizer ] + } case \ tokenizer [ + [ "EBNF]" parse-multiline-string ] [ drop "" ] recover append ebnf>quot nip parsed + ] with-variable ; parsing : EBNF: - CREATE-WORD dup - ";EBNF" parse-multiline-string - ebnf>quot swapd 1 1 define-declared "ebnf-parser" set-word-prop ; parsing + CREATE-WORD scan { + { "+" [ scan-word execute "" swap ] } + [ " " append default-tokenizer ] + } case \ tokenizer [ + dupd [ ";EBNF" parse-multiline-string ] [ drop "" ] recover append + ebnf>quot swapd 1 1 define-declared "ebnf-parser" set-word-prop + ] with-variable ; parsing + + diff --git a/extra/peg/javascript/tokenizer/tokenizer.factor b/extra/peg/javascript/tokenizer/tokenizer.factor index a1cff8a3db..5bf6a639fa 100644 --- a/extra/peg/javascript/tokenizer/tokenizer.factor +++ b/extra/peg/javascript/tokenizer/tokenizer.factor @@ -8,7 +8,7 @@ IN: peg.javascript.tokenizer USE: prettyprint -EBNF: tokenizer +EBNF: javascript-tokenizer Letter = [a-zA-Z] Digit = [0-9] Digits = Digit+ From eca8260799dde2a4da0df6ac78e83356ad34f3df Mon Sep 17 00:00:00 2001 From: Chris Double Date: Sat, 21 Jun 2008 00:28:13 +1200 Subject: [PATCH 37/47] peg.ebnf syntax for tokenizers --- extra/peg/ebnf/ebnf-tests.factor | 59 ++++++++++++++++++++++++++++++-- extra/peg/ebnf/ebnf.factor | 41 ++++++++++++++++------ 2 files changed, 87 insertions(+), 13 deletions(-) diff --git a/extra/peg/ebnf/ebnf-tests.factor b/extra/peg/ebnf/ebnf-tests.factor index 5a4ecc5c2f..a90fa98436 100644 --- a/extra/peg/ebnf/ebnf-tests.factor +++ b/extra/peg/ebnf/ebnf-tests.factor @@ -2,7 +2,8 @@ ! See http://factorcode.org/license.txt for BSD license. ! USING: kernel tools.test peg peg.ebnf words math math.parser - sequences accessors peg.parsers parser namespaces ; + sequences accessors peg.parsers parser namespaces arrays + strings ; IN: peg.ebnf.tests { T{ ebnf-non-terminal f "abc" } } [ @@ -451,9 +452,63 @@ foo= 'd' "USING: peg.ebnf ; [EBNF foo='a' foo='b' EBNF]" eval drop ] must-fail - { t } [ #! Rule lookup occurs in a namespace. This causes an incorrect duplicate rule #! if a var in a namespace is set. This unit test is to remind me to fix this. [ "fail" "foo" set "foo='a'" 'ebnf' parse ast>> transform drop t ] with-scope +] unit-test + +#! Tokenizer tests +{ V{ "a" CHAR: b } } [ + "ab" [EBNF tokenizer=default foo="a" . EBNF] call ast>> +] unit-test + +TUPLE: ast-number value ; + +EBNF: a-tokenizer +Letter = [a-zA-Z] +Digit = [0-9] +Digits = Digit+ +SingleLineComment = "//" (!("\n") .)* "\n" => [[ ignore ]] +MultiLineComment = "/*" (!("*/") .)* "*/" => [[ ignore ]] +Space = " " | "\t" | "\r" | "\n" | SingleLineComment | MultiLineComment +Spaces = Space* => [[ ignore ]] +Number = Digits:ws '.' Digits:fs => [[ ws "." fs 3array concat >string string>number ast-number boa ]] + | Digits => [[ >string string>number ast-number boa ]] +Special = "(" | ")" | "{" | "}" | "[" | "]" | "," | ";" + | "?" | ":" | "!==" | "~=" | "===" | "==" | "=" | ">=" + | ">" | "<=" | "<" | "++" | "+=" | "+" | "--" | "-=" + | "-" | "*=" | "*" | "/=" | "/" | "%=" | "%" | "&&=" + | "&&" | "||=" | "||" | "." | "!" +Tok = Spaces (Number | Special ) +;EBNF + +{ V{ CHAR: 1 T{ ast-number f 23 } ";" CHAR: x } } [ + "123;x" [EBNF bar = . + tokenizer = foo=. + tokenizer=default baz=. + main = bar foo foo baz + EBNF] call ast>> +] unit-test + +{ V{ CHAR: 5 "+" CHAR: 2 } } [ + "5+2" [EBNF + space=(" " | "\n") + number=[0-9] + operator=("*" | "+") + spaces=space* => [[ ignore ]] + tokenizer=spaces (number | operator) + main= . . . + EBNF] call ast>> +] unit-test + +{ V{ CHAR: 5 "+" CHAR: 2 } } [ + "5 + 2" [EBNF + space=(" " | "\n") + number=[0-9] + operator=("*" | "+") + spaces=space* => [[ ignore ]] + tokenizer=spaces (number | operator) + main= . . . + EBNF] call ast>> ] unit-test \ No newline at end of file diff --git a/extra/peg/ebnf/ebnf.factor b/extra/peg/ebnf/ebnf.factor index 921ba7be67..25889fe44d 100644 --- a/extra/peg/ebnf/ebnf.factor +++ b/extra/peg/ebnf/ebnf.factor @@ -27,7 +27,7 @@ TUPLE: tokenizer any one many ; rule parser-tokenizer ; : tokenizer ( -- word ) - \ tokenizer get [ default-tokenizer ] unless* ; + \ tokenizer get-global [ default-tokenizer ] unless* ; : reset-tokenizer ( -- ) default-tokenizer \ tokenizer set-global ; @@ -49,6 +49,7 @@ TUPLE: ebnf-repeat0 group ; TUPLE: ebnf-repeat1 group ; TUPLE: ebnf-optional group ; TUPLE: ebnf-whitespace group ; +TUPLE: ebnf-tokenizer elements ; TUPLE: ebnf-rule symbol elements ; TUPLE: ebnf-action parser code ; TUPLE: ebnf-var parser name ; @@ -68,6 +69,7 @@ C: ebnf-repeat0 C: ebnf-repeat1 C: ebnf-optional C: ebnf-whitespace +C: ebnf-tokenizer C: ebnf-rule C: ebnf-action C: ebnf-var @@ -318,8 +320,17 @@ DEFER: 'choice' dup length 1 = [ first ] [ ] if ] action ; +: 'tokenizer' ( -- parser ) + [ + "tokenizer" syntax , + "=" syntax , + ">" token ensure-not , + [ "default" token sp , 'choice' , ] choice* , + ] seq* [ first ] action ; + : 'rule' ( -- parser ) [ + "tokenizer" token ensure-not , 'non-terminal' [ symbol>> ] action , "=" syntax , ">" token ensure-not , @@ -327,7 +338,7 @@ DEFER: 'choice' ] seq* [ first2 ] action ; : 'ebnf' ( -- parser ) - 'rule' sp repeat1 [ ] action ; + [ 'tokenizer' sp , 'rule' sp , ] choice* repeat1 [ ] action ; GENERIC: (transform) ( ast -- parser ) @@ -345,6 +356,14 @@ SYMBOL: ignore-ws M: ebnf (transform) ( ast -- parser ) rules>> [ (transform) ] map peek ; + +M: ebnf-tokenizer (transform) ( ast -- parser ) + elements>> dup "default" = [ + drop default-tokenizer \ tokenizer set-global any-char + ] [ + (transform) + dup parser-tokenizer \ tokenizer set-global + ] if ; M: ebnf-rule (transform) ( ast -- parser ) dup elements>> @@ -369,7 +388,7 @@ M: ebnf-choice (transform) ( ast -- parser ) options>> [ (transform) ] map choice ; M: ebnf-any-character (transform) ( ast -- parser ) - drop [ tokenizer any>> call ] box ; + drop tokenizer any>> call ; M: ebnf-range (transform) ( ast -- parser ) pattern>> range-pattern ; @@ -460,7 +479,7 @@ M: ebnf-var (transform) ( ast -- parser ) parser>> (transform) ; M: ebnf-terminal (transform) ( ast -- parser ) - symbol>> [ tokenizer one>> call ] curry box ; + symbol>> tokenizer one>> call ; M: ebnf-foreign (transform) ( ast -- parser ) dup word>> search @@ -505,18 +524,18 @@ M: ebnf-non-terminal (transform) ( ast -- parser ) scan { { "+" [ scan-word execute "" swap ] } [ " " append default-tokenizer ] - } case \ tokenizer [ - [ "EBNF]" parse-multiline-string ] [ drop "" ] recover append ebnf>quot nip parsed - ] with-variable ; parsing + } case \ tokenizer set-global + [ "EBNF]" parse-multiline-string ] [ drop "" ] recover append ebnf>quot nip parsed + reset-tokenizer ; parsing : EBNF: CREATE-WORD scan { { "+" [ scan-word execute "" swap ] } [ " " append default-tokenizer ] - } case \ tokenizer [ - dupd [ ";EBNF" parse-multiline-string ] [ drop "" ] recover append - ebnf>quot swapd 1 1 define-declared "ebnf-parser" set-word-prop - ] with-variable ; parsing + } case \ tokenizer set-global + dupd [ ";EBNF" parse-multiline-string ] [ drop "" ] recover append + ebnf>quot swapd 1 1 define-declared "ebnf-parser" set-word-prop + reset-tokenizer ; parsing From 5433553571fd0c2382cb334fc94336515faa5c7d Mon Sep 17 00:00:00 2001 From: Chris Double Date: Sat, 21 Jun 2008 01:00:27 +1200 Subject: [PATCH 38/47] Remove EBNF: and [EBNF EBNF] +tokenizer syntax replaced with tokenizer support in ebnf grammar itself. --- extra/peg/ebnf/ebnf.factor | 14 ++------------ extra/peg/javascript/javascript.factor | 4 ++-- extra/peg/javascript/parser/parser-tests.factor | 10 +++++----- extra/peg/javascript/parser/parser.factor | 2 +- .../javascript/tokenizer/tokenizer-tests.factor | 4 ++-- extra/peg/javascript/tokenizer/tokenizer.factor | 2 +- 6 files changed, 13 insertions(+), 23 deletions(-) diff --git a/extra/peg/ebnf/ebnf.factor b/extra/peg/ebnf/ebnf.factor index 25889fe44d..564b376b29 100644 --- a/extra/peg/ebnf/ebnf.factor +++ b/extra/peg/ebnf/ebnf.factor @@ -520,20 +520,10 @@ M: ebnf-non-terminal (transform) ( ast -- parser ) parse-result-ast transform dup dup parser [ main swap at compile ] with-variable [ compiled-parse ] curry [ with-scope ] curry ; -: [EBNF - scan { - { "+" [ scan-word execute "" swap ] } - [ " " append default-tokenizer ] - } case \ tokenizer set-global - [ "EBNF]" parse-multiline-string ] [ drop "" ] recover append ebnf>quot nip parsed - reset-tokenizer ; parsing +: [EBNF "EBNF]" reset-tokenizer parse-multiline-string ebnf>quot nip parsed reset-tokenizer ; parsing : EBNF: - CREATE-WORD scan { - { "+" [ scan-word execute "" swap ] } - [ " " append default-tokenizer ] - } case \ tokenizer set-global - dupd [ ";EBNF" parse-multiline-string ] [ drop "" ] recover append + reset-tokenizer CREATE-WORD dup ";EBNF" parse-multiline-string ebnf>quot swapd 1 1 define-declared "ebnf-parser" set-word-prop reset-tokenizer ; parsing diff --git a/extra/peg/javascript/javascript.factor b/extra/peg/javascript/javascript.factor index 23a4b4f7f0..d27a06e9d2 100644 --- a/extra/peg/javascript/javascript.factor +++ b/extra/peg/javascript/javascript.factor @@ -4,8 +4,8 @@ USING: kernel accessors peg.javascript.tokenizer peg.javascript.parser ; IN: peg.javascript : parse-javascript ( string -- ast ) - tokenizer [ - ast>> javascript [ + tokenize-javascript [ + ast>> parse-javascript [ ast>> ] [ "Unable to parse JavaScript" throw diff --git a/extra/peg/javascript/parser/parser-tests.factor b/extra/peg/javascript/parser/parser-tests.factor index d911a27285..b19bb314bb 100644 --- a/extra/peg/javascript/parser/parser-tests.factor +++ b/extra/peg/javascript/parser/parser-tests.factor @@ -5,7 +5,7 @@ USING: kernel tools.test peg peg.javascript.ast peg.javascript.tokenizer peg.javascript.parser accessors multiline sequences math ; IN: peg.javascript.parser.tests -\ javascript must-infer +\ parse-javascript must-infer { T{ @@ -23,14 +23,14 @@ IN: peg.javascript.parser.tests } } } [ - "123; 'hello'; foo(x);" tokenizer ast>> javascript ast>> + "123; 'hello'; foo(x);" tokenize-javascript ast>> parse-javascript ast>> ] unit-test { t } [ <" var x=5 var y=10 -"> tokenizer ast>> javascript remaining>> length zero? +"> tokenize-javascript ast>> parse-javascript remaining>> length zero? ] unit-test @@ -41,7 +41,7 @@ function foldl(f, initial, seq) { initial = f(initial, seq[i]); return initial; } -"> tokenizer ast>> javascript remaining>> length zero? +"> tokenize-javascript ast>> parse-javascript remaining>> length zero? ] unit-test { t } [ @@ -52,6 +52,6 @@ ParseState.prototype.from = function(index) { r.length = this.length - index; return r; } -"> tokenizer ast>> javascript remaining>> length zero? +"> tokenize-javascript ast>> parse-javascript remaining>> length zero? ] unit-test diff --git a/extra/peg/javascript/parser/parser.factor b/extra/peg/javascript/parser/parser.factor index 45aa0f022c..1a074090f3 100644 --- a/extra/peg/javascript/parser/parser.factor +++ b/extra/peg/javascript/parser/parser.factor @@ -6,7 +6,7 @@ IN: peg.javascript.parser #! Grammar for JavaScript. Based on OMeta-JS example from: #! http://jarrett.cs.ucla.edu/ometa-js/#JavaScript_Compiler -EBNF: javascript +EBNF: parse-javascript End = !(.) Space = " " | "\t" | "\n" Spaces = Space* => [[ ignore ]] diff --git a/extra/peg/javascript/tokenizer/tokenizer-tests.factor b/extra/peg/javascript/tokenizer/tokenizer-tests.factor index 1300b3c9c7..509ff4a0fe 100644 --- a/extra/peg/javascript/tokenizer/tokenizer-tests.factor +++ b/extra/peg/javascript/tokenizer/tokenizer-tests.factor @@ -4,7 +4,7 @@ USING: kernel tools.test peg peg.javascript.ast peg.javascript.tokenizer accessors ; IN: peg.javascript.tokenizer.tests -\ tokenizer must-infer +\ tokenize-javascript must-infer { V{ @@ -19,5 +19,5 @@ IN: peg.javascript.tokenizer.tests ";" } } [ - "123; 'hello'; foo(x);" tokenizer ast>> + "123; 'hello'; foo(x);" tokenize-javascript ast>> ] unit-test diff --git a/extra/peg/javascript/tokenizer/tokenizer.factor b/extra/peg/javascript/tokenizer/tokenizer.factor index 5bf6a639fa..b72173f956 100644 --- a/extra/peg/javascript/tokenizer/tokenizer.factor +++ b/extra/peg/javascript/tokenizer/tokenizer.factor @@ -8,7 +8,7 @@ IN: peg.javascript.tokenizer USE: prettyprint -EBNF: javascript-tokenizer +EBNF: tokenize-javascript Letter = [a-zA-Z] Digit = [0-9] Digits = Digit+ From 6b83ab9d9060f8972b43d7082253a3a957d1f96f Mon Sep 17 00:00:00 2001 From: Chris Double Date: Sat, 21 Jun 2008 01:49:07 +1200 Subject: [PATCH 39/47] Fix terminal parser with non-default tokenizers --- extra/peg/ebnf/ebnf-tests.factor | 4 ++++ extra/peg/ebnf/ebnf.factor | 7 ++++--- extra/peg/javascript/javascript.factor | 2 +- extra/peg/javascript/parser/parser-tests.factor | 14 +++++++------- extra/peg/javascript/parser/parser.factor | 5 +++-- 5 files changed, 19 insertions(+), 13 deletions(-) diff --git a/extra/peg/ebnf/ebnf-tests.factor b/extra/peg/ebnf/ebnf-tests.factor index a90fa98436..2269af6625 100644 --- a/extra/peg/ebnf/ebnf-tests.factor +++ b/extra/peg/ebnf/ebnf-tests.factor @@ -511,4 +511,8 @@ Tok = Spaces (Number | Special ) tokenizer=spaces (number | operator) main= . . . EBNF] call ast>> +] unit-test + +{ "++" } [ + "++--" [EBNF tokenizer=("++" | "--") main="++" EBNF] call ast>> ] unit-test \ No newline at end of file diff --git a/extra/peg/ebnf/ebnf.factor b/extra/peg/ebnf/ebnf.factor index 564b376b29..4725534178 100644 --- a/extra/peg/ebnf/ebnf.factor +++ b/extra/peg/ebnf/ebnf.factor @@ -15,13 +15,14 @@ TUPLE: tokenizer any one many ; : default-tokenizer ( -- tokenizer ) T{ tokenizer f - [ [ drop t ] satisfy ] + [ any-char ] [ token ] - [ [ = ] curry satisfy ] + [ [ = ] curry any-char swap semantic ] } ; : parser-tokenizer ( parser -- tokenizer ) - 1quotation [ [ = ] curry satisfy ] dup tokenizer boa ; + [ 1quotation ] keep + [ swap [ = ] curry semantic ] curry dup tokenizer boa ; : rule-tokenizer ( name word -- tokenizer ) rule parser-tokenizer ; diff --git a/extra/peg/javascript/javascript.factor b/extra/peg/javascript/javascript.factor index d27a06e9d2..791f63c56b 100644 --- a/extra/peg/javascript/javascript.factor +++ b/extra/peg/javascript/javascript.factor @@ -5,7 +5,7 @@ IN: peg.javascript : parse-javascript ( string -- ast ) tokenize-javascript [ - ast>> parse-javascript [ + ast>> javascript [ ast>> ] [ "Unable to parse JavaScript" throw diff --git a/extra/peg/javascript/parser/parser-tests.factor b/extra/peg/javascript/parser/parser-tests.factor index b19bb314bb..fd0e27b6d4 100644 --- a/extra/peg/javascript/parser/parser-tests.factor +++ b/extra/peg/javascript/parser/parser-tests.factor @@ -1,11 +1,11 @@ ! Copyright (C) 2008 Chris Double. ! See http://factorcode.org/license.txt for BSD license. ! -USING: kernel tools.test peg peg.javascript.ast peg.javascript.tokenizer - peg.javascript.parser accessors multiline sequences math ; +USING: kernel tools.test peg peg.javascript.ast peg.javascript.parser + accessors multiline sequences math ; IN: peg.javascript.parser.tests -\ parse-javascript must-infer +\ javascript must-infer { T{ @@ -23,14 +23,14 @@ IN: peg.javascript.parser.tests } } } [ - "123; 'hello'; foo(x);" tokenize-javascript ast>> parse-javascript ast>> + "123; 'hello'; foo(x);" javascript ast>> ] unit-test { t } [ <" var x=5 var y=10 -"> tokenize-javascript ast>> parse-javascript remaining>> length zero? +"> javascript remaining>> length zero? ] unit-test @@ -41,7 +41,7 @@ function foldl(f, initial, seq) { initial = f(initial, seq[i]); return initial; } -"> tokenize-javascript ast>> parse-javascript remaining>> length zero? +"> javascript remaining>> length zero? ] unit-test { t } [ @@ -52,6 +52,6 @@ ParseState.prototype.from = function(index) { r.length = this.length - index; return r; } -"> tokenize-javascript ast>> parse-javascript remaining>> length zero? +"> javascript remaining>> length zero? ] unit-test diff --git a/extra/peg/javascript/parser/parser.factor b/extra/peg/javascript/parser/parser.factor index 1a074090f3..0239ce882c 100644 --- a/extra/peg/javascript/parser/parser.factor +++ b/extra/peg/javascript/parser/parser.factor @@ -1,12 +1,13 @@ ! Copyright (C) 2008 Chris Double. ! See http://factorcode.org/license.txt for BSD license. -USING: kernel accessors peg peg.ebnf peg.javascript.ast ; +USING: kernel accessors peg peg.ebnf peg.javascript.ast peg.javascript.tokenizer ; IN: peg.javascript.parser #! Grammar for JavaScript. Based on OMeta-JS example from: #! http://jarrett.cs.ucla.edu/ometa-js/#JavaScript_Compiler -EBNF: parse-javascript +EBNF: javascript +tokenizer = End = !(.) Space = " " | "\t" | "\n" Spaces = Space* => [[ ignore ]] From ff8e52190725b8b3f1790c7909380a88b713eafb Mon Sep 17 00:00:00 2001 From: Chris Double Date: Sat, 21 Jun 2008 02:06:58 +1200 Subject: [PATCH 40/47] Fix automatic semicolon insertion rule in JavaScript parser --- extra/peg/javascript/parser/parser.factor | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/extra/peg/javascript/parser/parser.factor b/extra/peg/javascript/parser/parser.factor index 0239ce882c..f9a070dd86 100644 --- a/extra/peg/javascript/parser/parser.factor +++ b/extra/peg/javascript/parser/parser.factor @@ -6,7 +6,21 @@ IN: peg.javascript.parser #! Grammar for JavaScript. Based on OMeta-JS example from: #! http://jarrett.cs.ucla.edu/ometa-js/#JavaScript_Compiler +#! The interesting thing about this parser is the mixing of +#! a default and non-default tokenizer. The JavaScript tokenizer +#! removes all newlines. So when operating on tokens there is no +#! need for newline and space skipping in the grammar. But JavaScript +#! uses the newline in the 'automatic semicolon insertion' rule. +#! +#! If a statement ends in a newline, sometimes the semicolon can be +#! skipped. So we define an 'nl' rule using the default tokenizer. +#! This operates a character at a time. Using this 'nl' in the parser +#! allows us to detect newlines when we need to for the semicolon +#! insertion rule, but ignore it in all other places. EBNF: javascript +tokenizer = default +nl = "\n" + tokenizer = End = !(.) Space = " " | "\t" | "\n" @@ -14,7 +28,7 @@ Spaces = Space* => [[ ignore ]] Name = . ?[ ast-name? ]? => [[ value>> ]] Number = . ?[ ast-number? ]? => [[ value>> ]] String = . ?[ ast-string? ]? => [[ value>> ]] -SpacesNoNl = (!("\n") Space)* => [[ ignore ]] +SpacesNoNl = (!(nl) Space)* => [[ ignore ]] Expr = OrExpr:e "?" Expr:t ":" Expr:f => [[ e t f ast-cond-expr boa ]] | OrExpr:e "=" Expr:rhs => [[ e rhs ast-set boa ]] @@ -80,7 +94,7 @@ JsonPropName = Name | Number | String Formal = Spaces Name Formals = (Formal ("," Formal => [[ second ]])* => [[ first2 swap prefix ]])? FuncRest = "(" Formals:fs ")" "{" SrcElems:body "}" => [[ fs body ast-func boa ]] -Sc = SpacesNoNl ("\n" | &("}") | End)| ";" +Sc = SpacesNoNl (nl | &("}") | End)| ";" Binding = Name:n "=" Expr:v => [[ n v ast-var boa ]] | Name:n => [[ n "undefined" ast-get boa ast-var boa ]] Block = "{" SrcElems:ss "}" => [[ ss ]] From 4a47346bda1522f08a725a5c85f7516a0342c8b2 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Sat, 21 Jun 2008 02:09:46 +1200 Subject: [PATCH 41/47] Fix parse-javascript word for recent changes --- extra/peg/javascript/javascript.factor | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/extra/peg/javascript/javascript.factor b/extra/peg/javascript/javascript.factor index 791f63c56b..8fe0538eae 100644 --- a/extra/peg/javascript/javascript.factor +++ b/extra/peg/javascript/javascript.factor @@ -4,13 +4,8 @@ USING: kernel accessors peg.javascript.tokenizer peg.javascript.parser ; IN: peg.javascript : parse-javascript ( string -- ast ) - tokenize-javascript [ - ast>> javascript [ - ast>> - ] [ - "Unable to parse JavaScript" throw - ] if* + javascript [ + ast>> ] [ - "Unable to tokenize JavaScript" throw + "Unable to parse JavaScript" throw ] if* ; - From e8f990454905d86241f824ba29452ed82d3a0c02 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Sat, 21 Jun 2008 02:21:54 +1200 Subject: [PATCH 42/47] Add \r to the JavaScript automatic semicolon insertion routine --- extra/peg/javascript/parser/parser.factor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/peg/javascript/parser/parser.factor b/extra/peg/javascript/parser/parser.factor index f9a070dd86..15ab8ee1c4 100644 --- a/extra/peg/javascript/parser/parser.factor +++ b/extra/peg/javascript/parser/parser.factor @@ -19,7 +19,7 @@ IN: peg.javascript.parser #! insertion rule, but ignore it in all other places. EBNF: javascript tokenizer = default -nl = "\n" +nl = "\r" "\n" | "\n" tokenizer = End = !(.) From 8fa0c80d4e93780d3a31623702fe721ea1c5e1d1 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Sat, 21 Jun 2008 02:29:47 +1200 Subject: [PATCH 43/47] Add regular expressions to JavaScript parser --- extra/peg/javascript/parser/parser.factor | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/extra/peg/javascript/parser/parser.factor b/extra/peg/javascript/parser/parser.factor index 15ab8ee1c4..6abe2fb385 100644 --- a/extra/peg/javascript/parser/parser.factor +++ b/extra/peg/javascript/parser/parser.factor @@ -28,6 +28,7 @@ Spaces = Space* => [[ ignore ]] Name = . ?[ ast-name? ]? => [[ value>> ]] Number = . ?[ ast-number? ]? => [[ value>> ]] String = . ?[ ast-string? ]? => [[ value>> ]] +RegExp = . ?[ ast-regexp? ]? => [[ value>> ]] SpacesNoNl = (!(nl) Space)* => [[ ignore ]] Expr = OrExpr:e "?" Expr:t ":" Expr:f => [[ e t f ast-cond-expr boa ]] @@ -83,6 +84,7 @@ PrimExprHd = "(" Expr:e ")" => [[ e ]] | Name => [[ ast-get boa ]] | Number => [[ ast-number boa ]] | String => [[ ast-string boa ]] + | RegExp => [[ ast-regexp boa ]] | "function" FuncRest:fr => [[ fr ]] | "new" Name:n "(" Args:as ")" => [[ n as ast-new boa ]] | "[" Args:es "]" => [[ es ast-array boa ]] @@ -90,7 +92,7 @@ PrimExprHd = "(" Expr:e ")" => [[ e ]] JsonBindings = (JsonBinding ("," JsonBinding => [[ second ]])* => [[ first2 swap prefix ]])? Json = "{" JsonBindings:bs "}" => [[ bs ast-json boa ]] JsonBinding = JsonPropName:n ":" Expr:v => [[ n v ast-binding boa ]] -JsonPropName = Name | Number | String +JsonPropName = Name | Number | String | RegExp Formal = Spaces Name Formals = (Formal ("," Formal => [[ second ]])* => [[ first2 swap prefix ]])? FuncRest = "(" Formals:fs ")" "{" SrcElems:body "}" => [[ fs body ast-func boa ]] From c5a9ee0e160ca7fd087198bf50a750df7f8f2f81 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Sat, 21 Jun 2008 02:40:08 +1200 Subject: [PATCH 44/47] Add typeof, void and delete to JavaScript grammar --- extra/peg/javascript/parser/parser.factor | 3 +++ 1 file changed, 3 insertions(+) diff --git a/extra/peg/javascript/parser/parser.factor b/extra/peg/javascript/parser/parser.factor index 6abe2fb385..5eb42daa31 100644 --- a/extra/peg/javascript/parser/parser.factor +++ b/extra/peg/javascript/parser/parser.factor @@ -69,6 +69,9 @@ Unary = "-" Postfix:p => [[ p "-" ast-unop boa | "++" Postfix:p => [[ p "++" ast-preop boa ]] | "--" Postfix:p => [[ p "--" ast-preop boa ]] | "!" Postfix:p => [[ p "!" ast-unop boa ]] + | "typeof" Postfix:p => [[ p "typeof" ast-unop boa ]] + | "void" Postfix:p => [[ p "void" ast-unop boa ]] + | "delete" Postfix:p => [[ p "delete" ast-unop boa ]] | Postfix Postfix = PrimExpr:p SpacesNoNl "++" => [[ p "++" ast-postop boa ]] | PrimExpr:p SpacesNoNl "--" => [[ p "--" ast-postop boa ]] From a15783cabb4250bcab5376e6b2869e3aff270fbe Mon Sep 17 00:00:00 2001 From: Chris Double Date: Sat, 21 Jun 2008 10:32:15 +1200 Subject: [PATCH 45/47] Fix typo in != in JavaScript parser --- extra/peg/javascript/tokenizer/tokenizer.factor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/peg/javascript/tokenizer/tokenizer.factor b/extra/peg/javascript/tokenizer/tokenizer.factor index b72173f956..195184a16c 100644 --- a/extra/peg/javascript/tokenizer/tokenizer.factor +++ b/extra/peg/javascript/tokenizer/tokenizer.factor @@ -60,7 +60,7 @@ Str = '"""' StringChars1:cs '"""' => [[ cs ast-string boa ]] RegExpBody = (!("/" | "\n" | "\r") .)* => [[ >string ]] RegExp = "/" RegExpBody:r "/" => [[ r ast-regexp boa ]] Special = "(" | ")" | "{" | "}" | "[" | "]" | "," | ";" - | "?" | ":" | "!==" | "~=" | "===" | "==" | "=" | ">=" + | "?" | ":" | "!==" | "!=" | "===" | "==" | "=" | ">=" | ">" | "<=" | "<" | "++" | "+=" | "+" | "--" | "-=" | "-" | "*=" | "*" | "/=" | "/" | "%=" | "%" | "&&=" | "&&" | "||=" | "||" | "." | "!" From f0a1f8fe5fa709e318ccf1c3a4ac57a5408a3c29 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Sat, 21 Jun 2008 16:56:21 +1200 Subject: [PATCH 46/47] Fix JavaScript new expression --- extra/peg/javascript/parser/parser.factor | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extra/peg/javascript/parser/parser.factor b/extra/peg/javascript/parser/parser.factor index 5eb42daa31..d44bbcf675 100644 --- a/extra/peg/javascript/parser/parser.factor +++ b/extra/peg/javascript/parser/parser.factor @@ -89,7 +89,8 @@ PrimExprHd = "(" Expr:e ")" => [[ e ]] | String => [[ ast-string boa ]] | RegExp => [[ ast-regexp boa ]] | "function" FuncRest:fr => [[ fr ]] - | "new" Name:n "(" Args:as ")" => [[ n as ast-new boa ]] + | "new" PrimExpr:n "(" Args:as ")" => [[ n as ast-new boa ]] + | "new" PrimExpr:n => [[ n f ast-new boa ]] | "[" Args:es "]" => [[ es ast-array boa ]] | Json JsonBindings = (JsonBinding ("," JsonBinding => [[ second ]])* => [[ first2 swap prefix ]])? From 75814c35f4697d2c7970170808059769bda5c741 Mon Sep 17 00:00:00 2001 From: Chris Double Date: Tue, 24 Jun 2008 10:50:41 +1200 Subject: [PATCH 47/47] Fix MulExpr to use Unary on rhs --- extra/peg/javascript/parser/parser.factor | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extra/peg/javascript/parser/parser.factor b/extra/peg/javascript/parser/parser.factor index d44bbcf675..b7df9908da 100644 --- a/extra/peg/javascript/parser/parser.factor +++ b/extra/peg/javascript/parser/parser.factor @@ -60,9 +60,9 @@ RelExpr = RelExpr:x ">" AddExpr:y => [[ x y ">" ast-binop AddExpr = AddExpr:x "+" MulExpr:y => [[ x y "+" ast-binop boa ]] | AddExpr:x "-" MulExpr:y => [[ x y "-" ast-binop boa ]] | MulExpr -MulExpr = MulExpr:x "*" MulExpr:y => [[ x y "*" ast-binop boa ]] - | MulExpr:x "/" MulExpr:y => [[ x y "/" ast-binop boa ]] - | MulExpr:x "%" MulExpr:y => [[ x y "%" ast-binop boa ]] +MulExpr = MulExpr:x "*" Unary:y => [[ x y "*" ast-binop boa ]] + | MulExpr:x "/" Unary:y => [[ x y "/" ast-binop boa ]] + | MulExpr:x "%" Unary:y => [[ x y "%" ast-binop boa ]] | Unary Unary = "-" Postfix:p => [[ p "-" ast-unop boa ]] | "+" Postfix:p => [[ p ]]