python: more correct reference counting, handles ucs2 and ucs4, stdlib module wrappers

extra/python/ffi/ffi.factor

@@ -11,7 +11,8 @@ USING:
 IN: python.ffi
 
 << "python" {
-    { unix { "3.0" "2.6" "2.7" } } { windows { "26" "27" "30" } }
+    { linux { "3.0" "2.6" "2.7" } }
+    { windows { "26" "27" "30" } }
 } os of [
     "python" prepend find-library
 ] map-find drop cdecl add-library >>
@@ -35,9 +36,11 @@ FUNCTION: long PyImport_GetMagicNumber ( ) ;
 FUNCTION: PyObject* PyImport_ImportModule ( c-string name ) ;
 
 ! Sys module
+! Borrowed reference
 FUNCTION: PyObject* PySys_GetObject ( c-string name ) ;
 
 ! Dicts
+! Borrowed reference
 FUNCTION: PyObject* PyDict_GetItemString ( PyObject* d, c-string key ) ;
 FUNCTION: PyObject* PyDict_New ( ) ;
 FUNCTION: int PyDict_Size ( PyObject* d ) ;
@@ -50,6 +53,7 @@ FUNCTION: PyObject* PyDict_Items ( PyObject *d ) ;
 ! Tuples
 FUNCTION: PyObject* PyTuple_GetItem ( PyObject* t, int pos ) ;
 FUNCTION: PyObject* PyTuple_New ( int len ) ;
+! Steals the reference
 FUNCTION: int PyTuple_SetItem ( PyObject* t, int pos, PyObject* o ) ;
 FUNCTION: int PyTuple_Size ( PyObject* t ) ;
 
@@ -104,6 +108,7 @@ DESTRUCTOR: Py_DecRef
 FUNCTION: c-string PyEval_GetFuncName ( PyObject* func ) ;
 
 ! Errors
+FUNCTION: void PyErr_Clear ( ) ;
 FUNCTION: void PyErr_Print ( ) ;
 FUNCTION: void PyErr_Fetch ( PyObject** ptype,
                              PyObject** pvalue,

extra/python/python-tests.factor

@@ -2,20 +2,23 @@ USING:
     accessors arrays assocs
     calendar
     continuations
+    destructors
     fry kernel
     math
     namespaces
-    python python.ffi
+    python python.ffi python.stdlib.sys
     sequences
     strings tools.test ;
 IN: python.tests
 
+py-initialize
+
 : py-test ( result quot -- )
-    '[ _ with-py ] unit-test ; inline
+    '[ _ with-destructors ] unit-test ; inline
 
 [ t ] [ Py_GetVersion string? ] unit-test
 
-[ "os" ] [ "os" PyImport_ImportModule PyModule_GetName ] py-test
+[ "os" ] [ "os" import PyModule_GetName ] py-test
 
 [ t ] [ "os" import "getpid" getattr { } py-call 0 > ] py-test
 
@@ -33,7 +36,7 @@ IN: python.tests
     { "year" "month" "day" } [ getattr >factor ] with map
     first3 0 0 0 instant <timestamp> ;
 
-! Datetimes
+! ! Datetimes
 [ t ] [
     [ py-date>factor ] "date" py-type-dispatch get set-at
     "datetime" import
@@ -111,3 +114,17 @@ SYMBOLS: year month day ;
 
 ! Modules
 [ t ] [ "os" import PyModule_GetDict py-dict-size 200 > ] py-test
+
+! Reference counting tests
+[ 2 ] [ 3 <py-tuple> getrefcount >factor ] py-test
+
+[ -2 ] [
+    H{ { "foo" 33 } { "bar" 44 } } >py
+    [ "foo" py-dict-get-item-string getrefcount >factor ]
+    [
+        '[
+            500 [ _ "foo" py-dict-get-item-string drop ] times
+        ] with-destructors
+    ]
+    [ "foo" py-dict-get-item-string getrefcount >factor ] tri -
+] py-test

extra/python/python.factor

@@ -3,12 +3,10 @@ USING:
     alien alien.c-types alien.data
     arrays
     assocs
-    destructors
     fry
     grouping
     hashtables
     kernel
-    memoize
     namespaces
     python.ffi
     sequences
@@ -17,6 +15,13 @@ USING:
 IN: python
 QUALIFIED: math
 
+! Initialization and finalization
+: py-initialize ( -- )
+    Py_IsInitialized [ Py_Initialize ] unless ;
+
+: py-finalize ( -- )
+    Py_IsInitialized [ Py_Finalize ] when ;
+
 ! Error handling
 ERROR: python-error type message ;
 
@@ -31,7 +36,7 @@ ERROR: python-error type message ;
     [ get-error throw-error f ] unless* ;
 
 : check-return ( value/f -- value' )
-    (check-return) ; ! &Py_DecRef ;
+    (check-return) &Py_DecRef ;
 
 : check-return-code ( return -- )
     0 = [ get-error throw-error ] unless ;
@@ -47,16 +52,12 @@ ERROR: python-error type message ;
 : call-object ( obj args -- value )
     PyObject_CallObject check-return ;
 
-! Context
-: with-py ( quot -- )
-    '[ Py_Initialize _ call Py_Finalize ] with-destructors ; inline
-
 ! Types
 : <py-tuple> ( length -- tuple )
     PyTuple_New check-return ;
 
 : py-tuple-set-item ( obj pos val -- )
-    PyTuple_SetItem check-return-code ;
+    dup Py_IncRef PyTuple_SetItem check-return-code ;
 
 : py-tuple-get-item ( obj pos -- val )
     PyTuple_GetItem check-return ;
@@ -64,6 +65,9 @@ ERROR: python-error type message ;
 : py-tuple-size ( obj -- len )
     PyTuple_Size ;
 
+: <1py-tuple> ( alien -- tuple )
+    1 <py-tuple> [ 0 rot py-tuple-set-item ] keep ;
+
 ! Dicts
 : <py-dict> ( -- dict )
     PyDict_New check-return ;
@@ -75,7 +79,7 @@ ERROR: python-error type message ;
     PyDict_SetItemString check-return-code ;
 
 : py-dict-get-item-string ( obj key -- val )
-    PyDict_GetItemString check-return ;
+    PyDict_GetItemString dup Py_IncRef check-return ;
 
 : py-dict-size ( obj -- len )
     PyDict_Size ;
@@ -88,16 +92,23 @@ ERROR: python-error type message ;
     PyList_GetItem check-return ;
 
 ! Unicodes
-: py-unicode>utf8 ( uni -- str )
-    PyUnicodeUCS2_AsUTF8String (check-return)
-    PyString_AsString (check-return)
+: py-ucs-size ( -- n )
+    "maxunicode" PySys_GetObject PyInt_AsLong 0xffff = 2 4 ? ;
 
-MEMO: py-ucs-size ( -- n )
-    "maxunicode" PySys_GetObject check-return PyInt_AsLong 0xffff = 2 4 ? ;
+: py-unicode>utf8 ( uni -- str )
+    py-ucs-size 4 =
+    [ PyUnicodeUCS4_AsUTF8String ]
+    [ PyUnicodeUCS2_AsUTF8String ] if (check-return)
+    PyString_AsString (check-return) ;
+
+: utf8>py-unicode ( str -- uni )
+    py-ucs-size 4 =
+    [ PyUnicodeUCS4_FromString ]
+    [ PyUnicodeUCS2_FromString ] if ;
 
 ! Data marshalling to Python
 GENERIC: (>py) ( obj -- obj' )
-M: string (>py) PyUnicodeUCS2_FromString ;
+M: string (>py) utf8>py-unicode ;
 M: math:fixnum (>py) PyLong_FromLong ;
 M: math:float (>py) PyFloat_FromDouble ;
 
@@ -110,11 +121,11 @@ M: hashtable (>py)
         swapd [ (>py) ] [ (>py) ] bi* py-dict-set-item
     ] with assoc-each ;
 
-! I'll make a fast-path for this
+! ! I'll make a fast-path for this
 M: word (>py) name>> (>py) ;
 
 : >py ( obj -- py-obj )
-    (>py) ; ! &Py_DecRef ;
+    (>py) &Py_DecRef ;
 
 ! Data marshalling to Factor
 SYMBOL: py-type-dispatch
@@ -126,7 +137,6 @@ DEFER: >factor
         { "NoneType" [ drop f ] }
         { "dict" [ PyDict_Items (check-return) >factor >hashtable ] }
         { "int" [ PyInt_AsLong ] }
-
         { "list" [
             dup py-list-size iota [ py-list-get-item >factor ] with map
         ] }
@@ -135,10 +145,7 @@ DEFER: >factor
         { "tuple" [
             dup py-tuple-size iota [ py-tuple-get-item >factor ] with map
         ] }
-        { "unicode" [
-            PyUnicodeUCS2_AsUTF8String (check-return)
-            PyString_AsString (check-return)
-        ] }
+        { "unicode" [ py-unicode>utf8 ] }
     } clone ;
 
 py-type-dispatch [ init-py-type-dispatch ] initialize

extra/python/stdlib/builtin/builtin.factor

@@ -0,0 +1,15 @@
+USING: alien arrays kernel namespaces python ;
+IN: python.stdlib.builtin
+
+py-initialize
+
+SYMBOL: builtin
+
+builtin [ "__builtin__" import ] initialize
+
+: repr ( alien/factor -- py-str )
+    dup alien? [ >py ] unless
+    <1py-tuple> builtin get "repr" getattr swap call-object ;
+
+: range ( n -- py-list )
+    builtin get "range" getattr swap 1array >py call-object ;

extra/python/stdlib/sys/sys.factor

@@ -0,0 +1,10 @@
+USING: kernel namespaces python ;
+IN: python.stdlib.sys
+
+py-initialize
+
+SYMBOL: sys
+sys [ "sys" import ] initialize
+
+: getrefcount ( alien -- py-int )
+    <1py-tuple> sys get "getrefcount" getattr swap call-object ;