forestdb: update to latest. fix tests to be better. if there's a .meta file (auto-compaction) then the old tests would fail.

extra/forestdb/ffi/ffi.factor

@@ -25,6 +25,7 @@ TYPEDEF: void* fdb_custom_cmp_variable
 TYPEDEF: void* fdb_file_handle
 TYPEDEF: void* fdb_kvs_handle
 TYPEDEF: void* fdb_iterator
+TYPEDEF: void* fdb_changes_callback_fn
 
 ENUM: fdb_open_flags < uint32_t
     { FDB_OPEN_FLAG_CREATE 1 }
@@ -57,7 +58,13 @@ ENUM: fdb_iterator_opt_t < uint16_t
     { FDB_ITR_NONE 0 }
     { FDB_ITR_NO_DELETES 2 }
     { FDB_ITR_SKIP_MIN_KEY 4 }
-    { FDB_ITR_SKIP_MAX_KEY 8 } ;
+    { FDB_ITR_SKIP_MAX_KEY 8 }
+    { FDB_ITR_NO_VALUES 0x10 } ; ! only keys and metadata for fdb_changes_since
+
+ENUM: fdb_changes_decision < int32_t
+    { FDB_CHANGES_PRESERVE 1 }
+    { FDB_CHANGES_CLEAN 0 }
+    { FDB_CHANGES_CANCEL -1 } ;
 
 ENUM: fdb_iterator_seek_opt_t < uint8_t
     { FDB_ITR_SEEK_HIGHER 0 }
@@ -182,8 +189,8 @@ ENUM: fdb_latency_stat_type < uint8_t
     { FDB_LATENCY_SETS 0 }
     { FDB_LATENCY_GETS 1 }
     { FDB_LATENCY_COMMITS 2 }
-    { FDB_LATENCY_SNAPSHOTS 3 }
+    { FDB_LATENCY_SNAP_INMEM 3 }
-    { FDB_LATENCY_SNAPSHOT_DUR 4 }
+    { FDB_LATENCY_SNAP_DUR 4 }
     { FDB_LATENCY_COMPACTS 5 }
     { FDB_LATENCY_ITR_INIT 6 }
     { FDB_LATENCY_ITR_SEQ_INIT 7 }
@@ -195,8 +202,15 @@ ENUM: fdb_latency_stat_type < uint8_t
     { FDB_LATENCY_ITR_SEEK_MAX 13 }
     { FDB_LATENCY_ITR_SEEK_MIN 14 }
     { FDB_LATENCY_ITR_CLOSE 15 }
-    { FDB_LATENCY_NUM_STATS 16 } ;
+    { FDB_LATENCY_OPEN 16 }
-
+    { FDB_LATENCY_KVS_OPEN 17 }
+    { FDB_LATENCY_SNAP_CLONE 18 }
+    { FDB_LATENCY_WAL_INS 19 }
+    { FDB_LATENCY_WAL_FIND 20 }
+    { FDB_LATENCY_WAL_COMMIT 21 }
+    { FDB_LATENCY_WAL_FLUSH 22 }
+    { FDB_LATENCY_WAL_RELEASE 23 }
+    { FDB_LATENCY_NUM_STATS 24 } ;
 
 STRUCT: fdb_latency_stat
     { lat_count uint64_t }
@@ -291,7 +305,9 @@ ENUM: fdb_status
     { FDB_RESULT_ELOOP -69 }
     { FDB_RESULT_ENAMETOOLONG -70 }
     { FDB_RESULT_EOVERFLOW -71 }
-    { FDB_RESULT_EAGAIN -72 } ;
+    { FDB_RESULT_EAGAIN -72 }
+    { FDB_RESULT_CANCELLED -73 }
+    { FDB_RESULT_LAST -73 } ; ! update this
 
 ! End fdb_errors.h
 
@@ -346,6 +362,11 @@ FUNCTION: fdb_status fdb_iterator_seek_to_min ( fdb_iterator* iterator )
 FUNCTION: fdb_status fdb_iterator_seek_to_max ( fdb_iterator* iterator )
 FUNCTION: fdb_status fdb_iterator_close ( fdb_iterator* iterator )
 
+FUNCTION: fdb_status fdb_changes_since ( fdb_kvs_handle *handle,
+                             fdb_seqnum_t since,
+                             fdb_iterator_opt_t opt,
+                             fdb_changes_callback_fn callback,
+                             void *ctx )
 FUNCTION: fdb_status fdb_compact ( fdb_file_handle* fhandle, c-string new_filename )
 FUNCTION: fdb_status fdb_compact_with_cow ( fdb_file_handle* fhandle, c-string new_filename )
 FUNCTION: fdb_status fdb_compact_upto ( fdb_file_handle* fhandle, c-string new_filename, fdb_snapshot_marker_t marker )
@@ -370,6 +391,10 @@ FUNCTION: fdb_status fdb_get_all_snap_markers (
     fdb_snapshot_info_t** markers,
     uint64_t* size )
 
+FUNCTION: fdb_seqnum_t fdb_get_available_rollback_seq (
+    fdb_kvs_handle* handle,
+    uint64_t request_seqno )
+
 FUNCTION: fdb_status fdb_free_snap_markers ( fdb_snapshot_info_t* markers, uint64_t size )
 FUNCTION: fdb_status fdb_free_kvs_name_list ( fdb_kvs_name_list* kvs_name_list )
 

extra/forestdb/lib/lib-tests.factor

@@ -9,28 +9,23 @@ kernel layouts libc make math math.parser math.ranges multiline
 namespaces sequences system tools.test ;
 IN: forestdb.lib
 
-{ } [ [ delete-test-db-0 ] ignore-errors ] unit-test
-{ } [ [ delete-test-db-1 ] ignore-errors ] unit-test
-
 ! Get/set by key/value
 { "val123" } [
-    delete-test-db-0
+    [
-    test-db-0 [
         "test123" [
             "key123" "val123" fdb-set-kv
             "key123" fdb-get-kv
         ] with-kvs
-    ] with-forestdb
+    ] with-forestdb-test-db
 ] unit-test
 
 { "val12345" } [
-    delete-test-db-0
+    [
-    test-db-0 [
         "test123" [
             "key123" "val12345" fdb-set-kv
             "key123" fdb-get-kv
         ] with-kvs
-    ] with-forestdb
+    ] with-forestdb-test-db
 ] unit-test
 
 ! Get
@@ -38,124 +33,117 @@ IN: forestdb.lib
 {
     { "key1" "val" }
 } [
-    delete-test-db-1 test-db-1 [
+    [
         5 set-kv-n
         fdb-commit-normal
         "key1" "meta" "val" [
             fdb_doc>doc [ key>> ] [ body>> ] bi 2array
         ] with-create-doc
-    ] with-forestdb-tester
+    ] with-forestdb-test-db
 ] unit-test
 
 
 {
     { "key1" f "val1" }
 } [
-    delete-test-db-1 test-db-1 [
+    [
         5 set-kv-n
         fdb-commit-normal
         "key1" "no meta" "going away" [
             fdb-get
             fdb_doc>doc [ key>> ] [ meta>> ] [ body>> ] tri 3array
         ] with-create-doc
-    ] with-forestdb-tester
+    ] with-forestdb-test-db
 ] unit-test
 
 
 {
     { "key2" f "val2" }
 } [
-    delete-test-db-1 test-db-1 [
+    [
         5 set-kv-n
         fdb-commit-normal
         2 <seqnum-doc> [
             fdb-get-byseq fdb_doc>doc
             [ key>> ] [ meta>> ] [ body>> ] tri 3array
         ] with-doc
-    ] with-forestdb-tester
+    ] with-forestdb-test-db
 ] unit-test
 
 {
     { "key2" f "val2" }
 } [
-    delete-test-db-1 test-db-1 [
+    [
         5 set-kv-n
         fdb-commit-normal
         2 <seqnum-doc> [
             fdb-get-byseq fdb_doc>doc
             [ key>> ] [ meta>> ] [ body>> ] tri 3array
         ] with-doc
-    ] with-forestdb-tester
+    ] with-forestdb-test-db
 ] unit-test
 
 ! Filename is only valid inside with-forestdb
 { f } [
-    delete-test-db-0
+    [
-    test-db-0 [
         fdb-get-info filename>> alien>native-string empty?
-    ] with-forestdb-tester
+    ] with-forestdb-test-db
 ] unit-test
 
 ! Test fdb_doc_create
 { 6 9 9 } [
-    delete-test-db-0
+    [
-    test-db-0 [
        "key123" "meta blah" "some body" [
             [ keylen>> ] [ metalen>> ] [ bodylen>> ] tri
         ] with-create-doc
-    ] with-forestdb-tester
+    ] with-forestdb-test-db
 ] unit-test
 
 { 7 8 15 } [
-    delete-test-db-0
+    [
-    test-db-0 [
        "key1234" "meta blah" "some body" [
             [ "new meta" "some other body" fdb-doc-update ]
             [ [ keylen>> ] [ metalen>> ] [ bodylen>> ] tri ] bi
         ] with-create-doc
-    ] with-forestdb-tester
+    ] with-forestdb-test-db
 ] unit-test
 
 { 1 1 } [
-    delete-test-db-1
+    [
-    test-db-1 [
         1 set-kv-n
         fdb-commit-normal
         fdb-get-kvs-info [ last_seqnum>> ] [ doc_count>> ] bi
-    ] with-forestdb-tester
+    ] with-forestdb-test-db
 ] unit-test
 
 { 6 5 } [
-    delete-test-db-1
+    [
-    test-db-1 [
         5 set-kv-n
         5 set-kv-nth
         fdb-commit-normal
         fdb-get-kvs-info [ last_seqnum>> ] [ doc_count>> ] bi
-    ] with-forestdb-tester
+    ] with-forestdb-test-db
 ] unit-test
 
 { 5 5 } [
-    delete-test-db-1
+    [
-    test-db-1 [
         5 set-kv-n
         fdb-commit-normal
         fdb-get-kvs-info [ last_seqnum>> ] [ doc_count>> ] bi
-    ] with-forestdb-tester
+    ] with-forestdb-test-db
 ] unit-test
 
 ! Snapshots
 
 /*
 { 5 5 } [
-    delete-test-db-1
+    [
-    test-db-1 [
         5 set-kv-n
         fdb-commit-normal
         FDB_SNAPSHOT_INMEM [
             fdb-get-kvs-info [ last_seqnum>> ] [ doc_count>> ] bi
         ] with-forestdb-snapshot
-    ] with-forestdb-tester
+    ] with-forestdb-test-db
 ] unit-test
 */
 
@@ -263,8 +251,7 @@ IN: forestdb.lib
 {
     { }
 } [
-    delete-test-db-1
+    [
-    test-db-1 [
         5 set-kv-n
         fdb-commit-normal
         [
@@ -272,7 +259,7 @@ IN: forestdb.lib
                   fdb_doc>doc [ seqnum>> ] [ key>> ] [ body>> ] tri 3array ,
             ] with-fdb-normal-iterator
         ] { } make
-    ] with-forestdb-tester
+    ] with-forestdb-test-db
 ] unit-test
 
 ! All the keys
@@ -285,8 +272,7 @@ IN: forestdb.lib
         { 5 "key5" "val5" }
     }
 } [
-    delete-test-db-1
+    [
-    test-db-1 [
         5 set-kv-n
         fdb-commit-normal
         [
@@ -294,7 +280,7 @@ IN: forestdb.lib
                   fdb_doc>doc [ seqnum>> ] [ key>> ] [ body>> ] tri 3array ,
             ] with-fdb-normal-iterator
         ] { } make
-    ] with-forestdb-tester
+    ] with-forestdb-test-db
 ] unit-test
 
 ! Test that keys at extremes get returned
@@ -303,8 +289,7 @@ IN: forestdb.lib
         { 1 "key1" "val1" }
     }
 } [
-    delete-test-db-1
+    [
-    test-db-1 [
         5 set-kv-n
         fdb-commit-normal
         [
@@ -312,7 +297,7 @@ IN: forestdb.lib
                   fdb_doc>doc [ seqnum>> ] [ key>> ] [ body>> ] tri 3array ,
             ] with-fdb-normal-iterator
         ] { } make
-    ] with-forestdb-tester
+    ] with-forestdb-test-db
 ] unit-test
 
 {
@@ -320,8 +305,7 @@ IN: forestdb.lib
         { 5 "key5" "val5" }
     }
 } [
-    delete-test-db-1
+    [
-    test-db-1 [
         5 set-kv-n
         fdb-commit-normal
         [
@@ -329,7 +313,7 @@ IN: forestdb.lib
                   fdb_doc>doc [ seqnum>> ] [ key>> ] [ body>> ] tri 3array ,
             ] with-fdb-normal-iterator
         ] { } make
-    ] with-forestdb-tester
+    ] with-forestdb-test-db
 ] unit-test
 
 
@@ -337,15 +321,14 @@ IN: forestdb.lib
 {
     V{ 1 2 3 4 5 }
 } [
-    delete-test-db-1
+    [
-    test-db-1 [
         5 set-kv-n
         fdb-commit-normal
         0 10 [
             fdb_doc>doc
         ] with-fdb-byseq-map
         [ seqnum>> ] map
-    ] with-forestdb-tester
+    ] with-forestdb-test-db
 ] unit-test
 
 ! XXX: Behavior changed here
@@ -354,8 +337,7 @@ IN: forestdb.lib
 ! {
     ! V{ { 6 t } { 7 t } { 8 t } { 9 t } { 10 t } }
 ! } [
-    ! delete-test-db-1
+    ! [
-    ! test-db-1 [
         ! 5 set-kv-n
         ! 5 del-kv-n
         ! fdb-commit-normal
@@ -363,14 +345,14 @@ IN: forestdb.lib
             ! fdb_doc>doc
         ! ] with-fdb-byseq-map
         ! [ [ seqnum>> ] [ deleted?>> ] bi 2array ] map
-    ! ] with-forestdb-tester
+    ! ] with-forestdb-test-db
 ! ] unit-test
 
 ! Test new behavior
 {
     V{ }
 } [
-    delete-test-db-1 test-db-1 [
+    [
         5 set-kv-n
         5 del-kv-n
         fdb-commit-normal
@@ -378,7 +360,7 @@ IN: forestdb.lib
             fdb_doc>doc
         ] with-fdb-byseq-map
         [ [ seqnum>> ] [ deleted?>> ] bi 2array ] map
-    ] with-forestdb-tester
+    ] with-forestdb-test-db
 ] unit-test
 
 {
@@ -391,7 +373,7 @@ IN: forestdb.lib
     }
 }
 [
-    delete-test-db-1 test-db-1 [
+    [
         5 set-kv-n
         fdb-commit-normal
         [
@@ -400,5 +382,5 @@ IN: forestdb.lib
                 [ [ key>> ] [ keylen>> ] bi alien/length>string ] bi 2array ,
             ] with-fdb-byseq-each
         ] { } make
-    ] with-forestdb-tester
+    ] with-forestdb-test-db
 ] unit-test

extra/forestdb/lib/lib.factor

@@ -148,11 +148,17 @@ M: byte-array encode-kv ;
 
 : fdb-commit-wal-flush ( -- ) FDB_COMMIT_MANUAL_WAL_FLUSH fdb-commit ;
 
-: fdb-compact ( new-path -- )
+: fdb-compact-to-path ( new-path -- )
     [ get-file-handle ] dip absolute-path
     fdb_compact fdb-check-error ;
 
-: fdb-compact-commit ( new-path -- )
+: fdb-compact ( -- )
+    get-file-handle f fdb_compact fdb-check-error ;
+
+: fdb-compact-commit-to-path ( path -- )
+    fdb-compact-to-path fdb-commit-wal-flush ;
+
+: fdb-compact-commit ( -- )
     fdb-compact fdb-commit-wal-flush ;
 
 
@@ -308,11 +314,15 @@ PRIVATE>
     [ fdb_open fdb-check-error ] 3keep
     2drop void* deref <fdb-file-handle> ;
 
+: fdb-default-config-auto-commit ( -- config )
+    fdb_get_default_config
+        FDB_SEQTREE_USE >>seqtree_opt ;
+        ! FDB_COMPACTION_AUTO >>compaction_mode
+        ! t >>auto_commit ;
+
 ! Make SEQTREES by default
 : fdb-open-default-config ( path -- file-handle )
-    fdb_get_default_config
+    fdb-default-config-auto-commit fdb-open ;
-        FDB_SEQTREE_USE >>seqtree_opt
-    fdb-open ;
 
 : fdb-kvs-open-config ( name config -- kvs-handle )
     [

extra/forestdb/utils/utils.factor

@@ -4,16 +4,19 @@ USING: assocs continuations forestdb.lib fry io.directories
 io.files.temp kernel math.parser math.ranges sequences ;
 IN: forestdb.utils
 
-: test-db-0 ( -- path ) "0.forestdb.0" temp-file ;
+: with-forestdb-test-db-kvs ( name quot -- )
-: test-db-1 ( -- path ) "1.forestdb.0" temp-file ;
-
-: with-forestdb-tester ( path quot -- )
     '[
-        "default" _ with-kvs
+        "forestdb-test" ".db" [
-    ] with-forestdb ; inline
+            _ _ with-forestdb-kvs 
+        ] cleanup-unique-file
+    ] with-temp-directory ; inline
 
-: delete-test-db-0 ( -- ) [ test-db-0 delete-file ] ignore-errors ;
+: with-forestdb-test-db ( quot -- )
-: delete-test-db-1 ( -- ) [ test-db-1 delete-file ] ignore-errors ;
+    '[
+        "forestdb-test" ".db" [
+            "default" _ with-forestdb-kvs 
+        ] cleanup-unique-file
+    ] with-temp-directory ; inline
 
 : make-kv-nth ( n -- key val )
     number>string [ "key" prepend ] [ "val" prepend ] bi ;