From 76b14d1bea8876f50905d6156477cab6dc0a9bb3 Mon Sep 17 00:00:00 2001 From: Doug Coleman Date: Thu, 6 Nov 2014 18:24:25 -0800 Subject: [PATCH] forestdb.ffi: Make fdb_iterator_opt_t its own type and remove comment. forestdb.lib: Implement iterators and some related stuff. Document possible bugs for now. forestdb.lib-tests: Test iterators. --- extra/forestdb/ffi/ffi.factor | 6 +- extra/forestdb/lib/lib-tests.factor | 77 ++++++++++++++- extra/forestdb/lib/lib.factor | 145 +++++++++++++++++++++++++--- 3 files changed, 215 insertions(+), 13 deletions(-) diff --git a/extra/forestdb/ffi/ffi.factor b/extra/forestdb/ffi/ffi.factor index 10e8ef9ebb..280e720440 100644 --- a/extra/forestdb/ffi/ffi.factor +++ b/extra/forestdb/ffi/ffi.factor @@ -16,7 +16,6 @@ CONSTANT: FDB_MAX_METALEN 65535 CONSTANT: FDB_MAX_BODYLEN 4294967295 ! TYPEDEF: uint32_t fdb_open_flags TYPEDEF: uint64_t fdb_seqnum_t -TYPEDEF: uint8_t fdb_iterator_opt_t CONSTANT: FDB_SNAPSHOT_INMEM -1 ! typedef int (*fdb_custom_cmp_fixed)(void *a, void *b); @@ -49,6 +48,11 @@ ENUM: fdb_compaction_mode_t { FDB_COMPACTION_MANUAL 0 } { FDB_COMPACTION_AUTO 1 } ; +ENUM: fdb_iterator_opt_t + { FDB_ITR_NONE 0 } + { FDB_ITR_METAONLY 1 } + { FDB_ITR_NO_DELETES 2 } ; + ENUM: fdb_isolation_level_t { FDB_ISOLATION_SERIALIZABLE 0 } { FDB_ISOLATION_REPEATABLE_READ 1 } diff --git a/extra/forestdb/lib/lib-tests.factor b/extra/forestdb/lib/lib-tests.factor index 2fdb55e41a..e38754fabd 100644 --- a/extra/forestdb/lib/lib-tests.factor +++ b/extra/forestdb/lib/lib-tests.factor @@ -2,7 +2,7 @@ ! See http://factorcode.org/license.txt for BSD license. USING: accessors alien.c-types alien.data alien.strings arrays assocs combinators continuations destructors forestdb.ffi fry -io.directories io.files.temp io.pathnames kernel libc +io.directories io.files.temp io.pathnames kernel libc make math.parser math.ranges multiline namespaces sequences tools.test ; IN: forestdb.lib @@ -208,3 +208,78 @@ IN: forestdb.lib ] with-forestdb-snapshot ] with-forestdb-path ] unit-test + + +! Iterators test +! No matching keys +{ + { } +} [ + delete-test-db-1 + test-db-1 [ + 5 set-kv-n + fdb-commit-normal + [ + "omg" "nada" [ + fdb_doc>doc [ seqnum>> ] [ key>> ] [ body>> ] tri 3array , + ] with-fdb-normal-iterator + ] { } make + ] with-forestdb-path +] unit-test + +! All the keys +{ + { + { 1 "key1" "val1" } + { 2 "key2" "val2" } + { 3 "key3" "val3" } + { 4 "key4" "val4" } + { 5 "key5" "val5" } + } +} [ + delete-test-db-1 + test-db-1 [ + 5 set-kv-n + fdb-commit-normal + [ + "key1" "key5" [ + fdb_doc>doc [ seqnum>> ] [ key>> ] [ body>> ] tri 3array , + ] with-fdb-normal-iterator + ] { } make + ] with-forestdb-path +] unit-test + +! Test that keys at extremes get returned +{ + { + { 1 "key1" "val1" } + } +} [ + delete-test-db-1 + test-db-1 [ + 5 set-kv-n + fdb-commit-normal + [ + "key0" "key1" [ + fdb_doc>doc [ seqnum>> ] [ key>> ] [ body>> ] tri 3array , + ] with-fdb-normal-iterator + ] { } make + ] with-forestdb-path +] unit-test + +{ + { + { 5 "key5" "val5" } + } +} [ + delete-test-db-1 + test-db-1 [ + 5 set-kv-n + fdb-commit-normal + [ + "key5" "key9" [ + fdb_doc>doc [ seqnum>> ] [ key>> ] [ body>> ] tri 3array , + ] with-fdb-normal-iterator + ] { } make + ] with-forestdb-path +] unit-test diff --git a/extra/forestdb/lib/lib.factor b/extra/forestdb/lib/lib.factor index 2bc93ceeae..441a368e98 100644 --- a/extra/forestdb/lib/lib.factor +++ b/extra/forestdb/lib/lib.factor @@ -1,12 +1,43 @@ ! Copyright (C) 2014 Doug Coleman. ! See http://factorcode.org/license.txt for BSD license. USING: accessors alien.c-types alien.data alien.strings -alien.syntax classes.struct combinators continuations -destructors forestdb.ffi fry generalizations io.encodings.string -io.encodings.utf8 io.pathnames kernel libc multiline namespaces -sequences ; +alien.syntax classes.struct combinators constructors +continuations destructors forestdb.ffi fry generalizations +io.encodings.string io.encodings.utf8 io.pathnames kernel libc +math multiline namespaces sequences ; IN: forestdb.lib +/* +! Possible bugs in foresdtb: + + ! why is meta set but metalen is 0? + ! also, there is no meta info in these docs + delete-test-db-1 + test-db-1 [ + 5 set-kv-n + fdb-commit-normal + "key1" "key5" [ + . + ] with-fdb-normal-meta-iterator + ] with-forestdb-path + + ... + S{ fdb_doc + { keylen 4 } + { metalen 0 } + { bodylen 4 } + { size_ondisk 0 } + { key ALIEN: 12721e3f0 } + { seqnum 5 } + { offset 4256 } + { meta ALIEN: 1272308d0 } + { body f } + { deleted f } + } + +! snapshot has doc_count of entire file, not in that snapshot +*/ + ERROR: fdb-error error ; : fdb-check-error ( ret -- ) @@ -26,6 +57,7 @@ M: fdb-handle dispose* TUPLE: fdb-doc < disposable doc ; + M: fdb-doc dispose* fdb_doc_free fdb-check-error ; @@ -115,16 +147,107 @@ FUNCTION: fdb_status fdb_rollback ( fdb_handle** handle_ptr, fdb_seqnum_t rollba [ fdb_rollback fdb-check-error ] 2keep drop void* deref fdb-current set ; + +TUPLE: fdb-iterator < disposable handle ; + +: ( handle -- obj ) + fdb-iterator new-disposable + swap >>handle ; inline + +M: fdb-iterator dispose* + handle>> fdb_iterator_close fdb-check-error ; + + +: fdb-iterator-init ( start-key end-key fdb_iterator_opt_t -- iterator ) + [ get-handle f void* ] 3dip + [ [ dup length ] bi@ ] dip + [ fdb_iterator_init fdb-check-error ] 7 nkeep 5 ndrop nip + void* deref ; + +: fdb-iterator-init-none ( start-key end-key -- iterator ) + FDB_ITR_NONE fdb-iterator-init ; + +: fdb-iterator-meta-only ( start-key end-key -- iterator ) + FDB_ITR_METAONLY fdb-iterator-init ; + +: fdb-iterator-no-deletes ( start-key end-key -- iterator ) + FDB_ITR_NO_DELETES fdb-iterator-init ; + +: check-iterate-result ( doc fdb_status -- doc/f ) + { + { FDB_RESULT_SUCCESS [ void* deref fdb_doc memory>struct ] } + { FDB_RESULT_ITERATOR_FAIL [ drop f ] } + [ throw ] + } case ; + +: fdb-iterate ( iterator word -- doc ) + '[ + fdb_doc fdb_doc + [ _ execute ] keep swap check-iterate-result + ] call ; inline + +! fdb_doc key, meta, body only valid inside with-forestdb +! so make a helper word to preserve them outside +TUPLE: doc seqnum key meta body deleted? offset size-ondisk ; + +CONSTRUCTOR: doc ( seqnum key meta body deleted? offset size-ondisk -- obj ) ; + /* -FUNCTION: fdb_status fdb_iterator_init ( fdb_handle* handle, fdb_iterator** iterator, c-string start_key, size_t start_keylen, c-string end_key, size_t end_keylen, fdb_iterator_opt_t opt ) ; -FUNCTION: fdb_status fdb_iterator_sequence_init ( fdb_handle* handle, fdb_iterator** iterator, fdb_seqnum_t start_seq, fdb_seqnum_t end_seq, fdb_iterator_opt_t opt ) ; -FUNCTION: fdb_status fdb_iterator_prev ( fdb_iterator* iterator, fdb_doc** doc ) ; -FUNCTION: fdb_status fdb_iterator_next ( fdb_iterator* iterator, fdb_doc** doc ) ; -FUNCTION: fdb_status fdb_iterator_next_metaonly ( fdb_iterator* iterator, fdb_doc** doc ) ; -FUNCTION: fdb_status fdb_iterator_seek ( fdb_iterator* iterator, c-string seek_key, size_t seek_keylen ) ; -FUNCTION: fdb_status fdb_iterator_close ( fdb_iterator* iterator ) ; +! Example fdb_doc and converted doc +S{ fdb_doc + { keylen 4 } + { metalen 0 } + { bodylen 4 } + { size_ondisk 0 } + { key ALIEN: 111e003b0 } + { seqnum 5 } + { offset 4256 } + { meta f } + { body ALIEN: 111d11740 } + { deleted f } +} +T{ doc + { seqnum 5 } + { key "key5" } + { body "val5" } + { offset 4256 } + { size-ondisk 0 } +} */ +: alien/length>string ( alien n -- string/f ) + [ drop f ] [ memory>byte-array utf8 decode ] if-zero ; + +: fdb_doc>doc ( fdb_doc -- doc ) + { + [ seqnum>> ] + [ [ key>> ] [ keylen>> ] bi alien/length>string ] + [ [ meta>> ] [ metalen>> ] bi alien/length>string ] + [ [ body>> ] [ bodylen>> ] bi alien/length>string ] + [ deleted>> >boolean ] + [ offset>> ] + [ size_ondisk>> ] + } cleave ; + +: fdb-iterator-prev ( iterator -- doc/f ) \ fdb_iterator_prev fdb-iterate ; +: fdb-iterator-next ( iterator -- doc/f ) \ fdb_iterator_next fdb-iterate ; +: fdb-iterator-next-meta-only ( iterator -- doc/f ) \ fdb_iterator_next_metaonly fdb-iterate ; +: fdb-iterator-seek ( iterator key -- ) + dup length fdb_iterator_seek fdb-check-error ; + +: with-fdb-iterator ( start-key end-key fdb_iterator_opt_t iterator-next quot: ( obj -- ) -- ) + [ fdb-iterator-init ] 2dip pick '[ + [ _ handle>> _ execute [ [ @ ] when* ] keep ] loop + _ &dispose drop + ] with-destructors ; inline + +: with-fdb-normal-iterator ( start-key end-key quot -- ) + [ FDB_ITR_NONE \ fdb-iterator-next ] dip with-fdb-iterator ; inline + +! XXX: broken? +: with-fdb-normal-meta-iterator ( start-key end-key quot -- ) + [ FDB_ITR_NONE \ fdb-iterator-next-meta-only ] dip with-fdb-iterator ; inline + ! Do not try to commit here, as it will fail with FDB_RESULT_RONLY_VIOLATION ! fdb-current is weird, it gets replaced if you call fdb-rollback ! Therefore, only clean up fdb-current once, and clean it up at the end