From 1f16d1933bdd868ce0a7df63beb9ce56b72d5469 Mon Sep 17 00:00:00 2001 From: Doug Coleman Date: Thu, 11 Dec 2014 14:42:19 -0800 Subject: [PATCH] forestdb: Update for recent changes. --- extra/forestdb/ffi/ffi.factor | 146 ++++++++++++++++------------ extra/forestdb/lib/lib-tests.factor | 22 ----- extra/forestdb/lib/lib.factor | 73 +++++++------- 3 files changed, 118 insertions(+), 123 deletions(-) diff --git a/extra/forestdb/ffi/ffi.factor b/extra/forestdb/ffi/ffi.factor index d42a07f246..dcd15746dd 100644 --- a/extra/forestdb/ffi/ffi.factor +++ b/extra/forestdb/ffi/ffi.factor @@ -11,12 +11,15 @@ IN: forestdb.ffi LIBRARY: forestdb +! Begin fdb_types.h + CONSTANT: FDB_MAX_KEYLEN 3840 CONSTANT: FDB_MAX_METALEN 65535 CONSTANT: FDB_MAX_BODYLEN 4294967295 CONSTANT: FDB_SNAPSHOT_INMEM -1 TYPEDEF: uint64_t fdb_seqnum_t +TYPEDEF: int64_t cs_off_t TYPEDEF: void* fdb_custom_cmp_fixed TYPEDEF: void* fdb_custom_cmp_variable @@ -25,6 +28,10 @@ TYPEDEF: void* fdb_file_handle TYPEDEF: void* fdb_kvs_handle TYPEDEF: void* fdb_iterator +ENUM: fdb_open_flags + { FDB_OPEN_FLAG_CREATE 1 } + { FDB_OPEN_FLAG_RDONLY 2 } ; + ENUM: fdb_commit_opt_t { FDB_COMMIT_NORMAL 0 } { FDB_COMMIT_MANUAL_WAL_FLUSH 1 } ; @@ -33,10 +40,6 @@ ENUM: fdb_seqtree_opt_t { FDB_SEQTREE_NOT_USE 0 } { FDB_SEQTREE_USE 1 } ; -ENUM: fdb_open_flags - { FDB_OPEN_FLAG_CREATE 1 } - { FDB_OPEN_FLAG_RDONLY 2 } ; - ENUM: fdb_durability_opt_t { FDB_DRB_NONE 0 } { FDB_DRB_ODIRECT 1 } @@ -47,55 +50,22 @@ ENUM: fdb_compaction_mode_t { FDB_COMPACTION_MANUAL 0 } { FDB_COMPACTION_AUTO 1 } ; -ENUM: fdb_iterator_opt_t - { FDB_ITR_NONE 0 } - { FDB_ITR_METAONLY 1 } - { FDB_ITR_NO_DELETES 2 } ; - ENUM: fdb_isolation_level_t { FDB_ISOLATION_SERIALIZABLE 0 } { FDB_ISOLATION_REPEATABLE_READ 1 } { FDB_ISOLATION_READ_COMMITTED 2 } { FDB_ISOLATION_READ_UNCOMMITTED 3 } ; -ENUM: fdb_status - { FDB_RESULT_SUCCESS 0 } - { FDB_RESULT_INVALID_ARGS -1 } - { FDB_RESULT_OPEN_FAIL -2 } - { FDB_RESULT_NO_SUCH_FILE -3 } - { FDB_RESULT_WRITE_FAIL -4 } - { FDB_RESULT_READ_FAIL -5 } - { FDB_RESULT_CLOSE_FAIL -6 } - { FDB_RESULT_COMMIT_FAIL -7 } - { FDB_RESULT_ALLOC_FAIL -8 } - { FDB_RESULT_KEY_NOT_FOUND -9 } - { FDB_RESULT_RONLY_VIOLATION -10 } - { FDB_RESULT_COMPACTION_FAIL -11 } - { FDB_RESULT_ITERATOR_FAIL -12 } - { FDB_RESULT_SEEK_FAIL -13 } - { FDB_RESULT_FSYNC_FAIL -14 } - { FDB_RESULT_CHECKSUM_ERROR -15 } - { FDB_RESULT_FILE_CORRUPTION -16 } - { FDB_RESULT_COMPRESSION_FAIL -17 } - { FDB_RESULT_NO_DB_INSTANCE -18 } - { FDB_RESULT_FAIL_BY_ROLLBACK -19 } - { FDB_RESULT_INVALID_CONFIG -20 } - { FDB_RESULT_MANUAL_COMPACTION_FAIL -21 } - { FDB_RESULT_INVALID_COMPACTION_MODE -22 } - { FDB_RESULT_FILE_IS_BUSY -23 } - { FDB_RESULT_FILE_REMOVE_FAIL -24 } - { FDB_RESULT_FILE_RENAME_FAIL -25 } - { FDB_RESULT_TRANSACTION_FAIL -26 } - { FDB_RESULT_FAIL_BY_TRANSACTION -27 } - { FDB_RESULT_FAIL_BY_COMPACTION -28 } - { FDB_RESULT_TOO_LONG_FILENAME -29 } - { FDB_RESULT_INVALID_HANDLE -30 } - { FDB_RESULT_KV_STORE_NOT_FOUND -31 } - { FDB_RESULT_KV_STORE_BUSY -32 } - { FDB_RESULT_INVALID_KV_INSTANCE_NAME -33 } - { FDB_RESULT_INVALID_CMP_FUNCTION -34 } - { FDB_RESULT_IN_USE_BY_COMPACTOR -35 } - { FDB_RESULT_FAIL -100 } ; +ENUM: fdb_iterator_opt_t + { FDB_ITR_NONE 0 } + { FDB_ITR_NO_DELETES 2 } + { FDB_ITR_SKIP_MIN_KEY 4 } + { FDB_ITR_SKIP_MAX_KEY 8 } ; + +ENUM: fdb_iterator_seek_opt_t + { FDB_ITR_SEEK_HIGHER 0 } + { FDB_ITR_SEEK_LOWER 1 } ; + ! cmp_fixed and cmp_variable have their own open() functions STRUCT: fdb_config @@ -115,7 +85,8 @@ STRUCT: fdb_config { compaction_threshold uint8_t } { compaction_minimum_filesize uint64_t } { compactor_sleep_duration uint64_t } - { multi_kv_instances bool } ; + { multi_kv_instances bool } + { prefetch_duration uint64_t } ; STRUCT: fdb_kvs_config { create_if_missing bool } @@ -154,6 +125,50 @@ STRUCT: fdb_kvs_name_list { num_kvs_names size_t } { kvs_names char** } ; +! end fdb_types.h + +! Begin fdb_errors.h +ENUM: fdb_status + { FDB_RESULT_SUCCESS 0 } + { FDB_RESULT_INVALID_ARGS -1 } + { FDB_RESULT_OPEN_FAIL -2 } + { FDB_RESULT_NO_SUCH_FILE -3 } + { FDB_RESULT_WRITE_FAIL -4 } + { FDB_RESULT_READ_FAIL -5 } + { FDB_RESULT_CLOSE_FAIL -6 } + { FDB_RESULT_COMMIT_FAIL -7 } + { FDB_RESULT_ALLOC_FAIL -8 } + { FDB_RESULT_KEY_NOT_FOUND -9 } + { FDB_RESULT_RONLY_VIOLATION -10 } + { FDB_RESULT_COMPACTION_FAIL -11 } + { FDB_RESULT_ITERATOR_FAIL -12 } + { FDB_RESULT_SEEK_FAIL -13 } + { FDB_RESULT_FSYNC_FAIL -14 } + { FDB_RESULT_CHECKSUM_ERROR -15 } + { FDB_RESULT_FILE_CORRUPTION -16 } + { FDB_RESULT_COMPRESSION_FAIL -17 } + { FDB_RESULT_NO_DB_INSTANCE -18 } + { FDB_RESULT_FAIL_BY_ROLLBACK -19 } + { FDB_RESULT_INVALID_CONFIG -20 } + { FDB_RESULT_MANUAL_COMPACTION_FAIL -21 } + { FDB_RESULT_INVALID_COMPACTION_MODE -22 } + { FDB_RESULT_FILE_IS_BUSY -23 } + { FDB_RESULT_FILE_REMOVE_FAIL -24 } + { FDB_RESULT_FILE_RENAME_FAIL -25 } + { FDB_RESULT_TRANSACTION_FAIL -26 } + { FDB_RESULT_FAIL_BY_TRANSACTION -27 } + { FDB_RESULT_FAIL_BY_COMPACTION -28 } + { FDB_RESULT_TOO_LONG_FILENAME -29 } + { FDB_RESULT_INVALID_HANDLE -30 } + { FDB_RESULT_KV_STORE_NOT_FOUND -31 } + { FDB_RESULT_KV_STORE_BUSY -32 } + { FDB_RESULT_INVALID_KV_INSTANCE_NAME -33 } + { FDB_RESULT_INVALID_CMP_FUNCTION -34 } + { FDB_RESULT_IN_USE_BY_COMPACTOR -35 } ; + +! End fdb_errors.h + +! Begin forestdb.h FUNCTION: fdb_status fdb_init ( fdb_config* config ) ; FUNCTION: fdb_config fdb_get_default_config ( ) ; FUNCTION: fdb_kvs_config fdb_get_default_kvs_config ( ) ; @@ -164,8 +179,8 @@ FUNCTION: fdb_status fdb_open_custom_cmp ( fdb_file_handle** ptr_fhandle, c-stri FUNCTION: fdb_status fdb_set_log_callback ( fdb_kvs_handle* handle, fdb_log_callback log_callback, void* ctx_data ) ; ! doc is calloc'd -FUNCTION: fdb_status fdb_doc_create ( fdb_doc** doc, c-string key, size_t keylen, c-string meta, size_t metalen, c-string body, size_t bodylen ) ; -FUNCTION: fdb_status fdb_doc_update ( fdb_doc** doc, c-string meta, size_t metalen, c-string body, size_t bodylen ) ; +FUNCTION: fdb_status fdb_doc_create ( fdb_doc** doc, void* key, size_t keylen, void* meta, size_t metalen, void* body, size_t bodylen ) ; +FUNCTION: fdb_status fdb_doc_update ( fdb_doc** doc, void* meta, size_t metalen, void* body, size_t bodylen ) ; FUNCTION: fdb_status fdb_doc_free ( fdb_doc* doc ) ; FUNCTION: fdb_status fdb_get ( fdb_kvs_handle* handle, fdb_doc* doc ) ; @@ -177,21 +192,26 @@ FUNCTION: fdb_status fdb_get_byoffset ( fdb_kvs_handle* handle, fdb_doc* doc ) ; FUNCTION: fdb_status fdb_set ( fdb_kvs_handle* handle, fdb_doc* doc ) ; FUNCTION: fdb_status fdb_del ( fdb_kvs_handle* handle, fdb_doc* doc ) ; -FUNCTION: fdb_status fdb_get_kv ( fdb_kvs_handle* handle, c-string key, size_t keylen, void** value_out, size_t* valuelen_out ) ; -FUNCTION: fdb_status fdb_set_kv ( fdb_kvs_handle* handle, c-string key, size_t keylen, c-string value, size_t valuelen ) ; -FUNCTION: fdb_status fdb_del_kv ( fdb_kvs_handle* handle, c-string key, size_t keylen ) ; +FUNCTION: fdb_status fdb_get_kv ( fdb_kvs_handle* handle, void* key, size_t keylen, void** value_out, size_t* valuelen_out ) ; +FUNCTION: fdb_status fdb_set_kv ( fdb_kvs_handle* handle, void* key, size_t keylen, void* value, size_t valuelen ) ; +FUNCTION: fdb_status fdb_del_kv ( fdb_kvs_handle* handle, void* key, size_t keylen ) ; FUNCTION: fdb_status fdb_commit ( fdb_file_handle* fhandle, fdb_commit_opt_t opt ) ; FUNCTION: fdb_status fdb_snapshot_open ( fdb_kvs_handle* handle_in, fdb_kvs_handle** handle_out, fdb_seqnum_t snapshot_seqnum ) ; ! Swaps out the handle for a new one FUNCTION: fdb_status fdb_rollback ( fdb_kvs_handle** handle_ptr, fdb_seqnum_t rollback_seqnum ) ; -FUNCTION: fdb_status fdb_iterator_init ( fdb_kvs_handle* handle, fdb_iterator** iterator, c-string start_key, size_t start_keylen, c-string end_key, size_t end_keylen, fdb_iterator_opt_t opt ) ; -FUNCTION: fdb_status fdb_iterator_sequence_init ( fdb_kvs_handle* handle, fdb_iterator** iterator, fdb_seqnum_t start_seq, fdb_seqnum_t end_seq, fdb_iterator_opt_t opt ) ; -FUNCTION: fdb_status fdb_iterator_prev ( fdb_iterator* iterator, fdb_doc** doc ) ; -FUNCTION: fdb_status fdb_iterator_next ( fdb_iterator* iterator, fdb_doc** doc ) ; -FUNCTION: fdb_status fdb_iterator_next_metaonly ( fdb_iterator* iterator, fdb_doc** doc ) ; -FUNCTION: fdb_status fdb_iterator_seek ( fdb_iterator* iterator, c-string seek_key, size_t seek_keylen ) ; +FUNCTION: fdb_status fdb_iterator_init ( fdb_kvs_handle* handle, fdb_iterator** iterator, void* min_key, size_t min_keylen, void* max_key, size_t max_keylen, fdb_iterator_opt_t opt ) ; +FUNCTION: fdb_status fdb_iterator_sequence_init ( fdb_kvs_handle* handle, fdb_iterator** iterator, fdb_seqnum_t min_seq, fdb_seqnum_t max_seq, fdb_iterator_opt_t opt ) ; + +FUNCTION: fdb_status fdb_iterator_prev ( fdb_iterator* iterator ) ; +FUNCTION: fdb_status fdb_iterator_next ( fdb_iterator* iterator ) ; +FUNCTION: fdb_status fdb_iterator_get ( fdb_iterator* iterator, fdb_doc **doc ) ; +FUNCTION: fdb_status fdb_iterator_get_metaonly ( fdb_iterator* iterator, fdb_doc **doc ) ; + +FUNCTION: fdb_status fdb_iterator_seek ( fdb_iterator* iterator, void* seek_key, size_t seek_keylen, fdb_iterator_seek_opt_t direction ) ; +FUNCTION: fdb_status fdb_iterator_seek_to_min ( fdb_iterator* iterator ) ; +FUNCTION: fdb_status fdb_iterator_seek_to_max ( fdb_iterator* iterator ) ; FUNCTION: fdb_status fdb_iterator_close ( fdb_iterator* iterator ) ; FUNCTION: fdb_status fdb_compact ( fdb_file_handle* handle, c-string new_filename ) ; @@ -199,12 +219,12 @@ FUNCTION: size_t fdb_estimate_space_used ( fdb_file_handle* fhandle ) ; FUNCTION: fdb_status fdb_get_file_info ( fdb_file_handle* handle, fdb_file_info* info ) ; FUNCTION: fdb_status fdb_get_kvs_info ( fdb_kvs_handle* handle, fdb_kvs_info* info ) ; FUNCTION: fdb_status fdb_get_kvs_seqnum ( fdb_kvs_handle* handle, fdb_seqnum_t* seqnum ) ; +FUNCTION: fdb_status fdb_get_kvs_name_list ( fdb_kvs_handle* handle, fdb_kvs_name_list* kvs_name_list ) ; +FUNCTION: fdb_status fdb_free_kvs_name_list ( fdb_kvs_name_list* kvs_name_list ) ; + FUNCTION: fdb_status fdb_switch_compaction_mode ( fdb_file_handle* fhandle, fdb_compaction_mode_t mode, size_t new_threshold ) ; - -FUNCTION: fdb_status fdb_get_kvs_name_list ( fdb_file_handle *fhandle, fdb_kvs_name_list *kvs_name_list ) ; -FUNCTION: fdb_status fdb_free_kvs_name_list ( fdb_kvs_name_list *kvs_name_list ) ; - FUNCTION: fdb_status fdb_close ( fdb_file_handle* fhandle ) ; + FUNCTION: fdb_status fdb_destroy ( c-string filename, fdb_config* fconfig ) ; FUNCTION: fdb_status fdb_shutdown ( ) ; diff --git a/extra/forestdb/lib/lib-tests.factor b/extra/forestdb/lib/lib-tests.factor index 2fa9def8f2..dbf934820d 100644 --- a/extra/forestdb/lib/lib-tests.factor +++ b/extra/forestdb/lib/lib-tests.factor @@ -321,28 +321,6 @@ IN: forestdb.lib ] unit-test -{ - { - { 1 "key1" } - { 2 "key2" } - { 3 "key3" } - { 4 "key4" } - { 5 "key5" } - } -} [ - delete-test-db-1 - test-db-1 [ - 5 set-kv-n - fdb-commit-normal - [ - 0 10 [ - [ seqnum>> ] - [ [ key>> ] [ keylen>> ] bi alien/length>string ] bi 2array , - ] with-fdb-byseq-each - ] { } make - ] with-forestdb-path -] unit-test - ! Test byseq mapping { V{ 1 2 3 4 5 } diff --git a/extra/forestdb/lib/lib.factor b/extra/forestdb/lib/lib.factor index 6e65b05c3d..8bc41c3b0d 100644 --- a/extra/forestdb/lib/lib.factor +++ b/extra/forestdb/lib/lib.factor @@ -10,9 +10,6 @@ IN: forestdb.lib /* ! Issues -! 2) build on macosx doesn't search /usr/local for libsnappy -! 3) build on macosx doesn't include -L/usr/local/lib when it finds snappy -! - link_directories(/usr/local/lib) or some other fix ! 4) byseq iteration doesn't have bodies, weird. ! 5) Get byseq ignores seqnum and uses key instead if key is set */ @@ -80,7 +77,7 @@ SYMBOL: current-fdb-kvs-handle : fdb-set-kv ( key value -- ) [ get-kvs-handle ] 2dip - [ dup length ] bi@ fdb_set_kv fdb-check-error ; + [ utf8 encode dup length ] bi@ fdb_set_kv fdb-check-error ; : ( key -- doc ) fdb_doc malloc-struct @@ -116,7 +113,7 @@ SYMBOL: current-fdb-kvs-handle : fdb-get-kv ( key -- value/f ) [ get-kvs-handle ] dip - dup length f void* 0 size_t + utf8 encode dup length f void* 0 size_t [ fdb_get_kv ] 2keep rot { { FDB_RESULT_SUCCESS [ ret>string ] } @@ -125,17 +122,18 @@ SYMBOL: current-fdb-kvs-handle } case ; : fdb-del-kv ( key -- ) - [ get-kvs-handle ] dip dup length fdb_del_kv fdb-check-error ; + [ get-kvs-handle ] dip + utf8 encode dup length fdb_del_kv fdb-check-error ; : fdb-doc-create ( key meta body -- doc ) [ f void* ] 3dip - [ dup length ] tri@ + [ utf8 encode dup length ] tri@ [ fdb_doc_create fdb-check-error ] 7 nkeep 6 ndrop void* deref fdb_doc memory>struct ; : fdb-doc-update ( doc meta body -- ) [ void* ] 2dip - [ dup length ] bi@ + [ utf8 encode dup length ] bi@ fdb_doc_update fdb-check-error ; : fdb-doc-free ( doc -- ) @@ -216,7 +214,7 @@ M: fdb-iterator dispose* : fdb-iterator-init ( start-key end-key fdb_iterator_opt_t -- iterator ) [ get-kvs-handle f void* ] 3dip - [ [ dup length ] bi@ ] dip + [ [ utf8 encode dup length ] bi@ ] dip [ fdb_iterator_init fdb-check-error ] 7 nkeep 5 ndrop nip void* deref ; @@ -228,25 +226,16 @@ M: fdb-iterator dispose* : fdb-iterator-init-none ( start-key end-key -- iterator ) FDB_ITR_NONE fdb-iterator-init ; -: fdb-iterator-meta-only ( start-key end-key -- iterator ) - FDB_ITR_METAONLY fdb-iterator-init ; - : fdb-iterator-no-deletes ( start-key end-key -- iterator ) FDB_ITR_NO_DELETES fdb-iterator-init ; -: check-iterate-result ( doc fdb_status -- doc/f ) +: check-iterate-result ( fdb_status -- ? ) { - { FDB_RESULT_SUCCESS [ void* deref fdb_doc memory>struct ] } - { FDB_RESULT_ITERATOR_FAIL [ drop f ] } + { FDB_RESULT_SUCCESS [ t ] } + { FDB_RESULT_ITERATOR_FAIL [ f ] } [ throw ] } case ; -: fdb-iterate ( iterator word -- doc ) - '[ - fdb_doc malloc-struct fdb_doc - [ _ execute ] keep swap check-iterate-result - ] call ; inline - ! fdb_doc key, meta, body only valid inside with-forestdb ! so make a helper word to preserve them outside TUPLE: fdb-doc seqnum keylen key metalen meta bodylen body deleted? offset size-ondisk ; @@ -269,8 +258,6 @@ T{ doc { metalen 0 } { bodylen 4 } { offset 4256 } { size-ondisk 0 } } - - */ : alien/length>string ( alien n -- string/f ) @@ -305,17 +292,28 @@ T{ doc [ file_size>> ] } cleave ; -: fdb-iterator-prev ( iterator -- doc/f ) \ fdb_iterator_prev fdb-iterate ; -: fdb-iterator-next ( iterator -- doc/f ) \ fdb_iterator_next fdb-iterate ; -: fdb-iterator-next-meta-only ( iterator -- doc/f ) \ fdb_iterator_next_metaonly fdb-iterate ; -: fdb-iterator-seek ( iterator key -- ) - dup length fdb_iterator_seek fdb-check-error ; +: fdb-iterator-get ( iterator -- doc/f ) + fdb_doc malloc-struct fdb_doc + [ fdb_iterator_get check-iterate-result ] keep swap + [ void* deref fdb_doc memory>struct ] [ void* deref free f ] if ; -: with-fdb-iterator ( start-key end-key fdb_iterator_opt_t iterator-init iterator-next quot: ( obj -- ) -- ) +: fdb-iterator-seek ( iterator key seek-opt -- ) + [ dup length ] dip fdb_iterator_seek fdb-check-error ; + +: fdb-iterator-seek-lower ( iterator key -- ) + FDB_ITR_SEEK_LOWER fdb-iterator-seek ; + +: fdb-iterator-seek-higher ( iterator key -- ) + FDB_ITR_SEEK_HIGHER fdb-iterator-seek ; + +: with-fdb-iterator ( start-key end-key fdb_iterator_opt_t iterator-init iterator-advance quot: ( obj -- ) -- ) [ execute ] 2dip + swap '[ _ &dispose handle>> [ - _ execute [ _ with-doc t ] [ f ] if* + [ fdb-iterator-get ] keep swap + [ _ with-doc _ execute check-iterate-result ] + [ drop f ] if* ] curry loop ] with-destructors ; inline @@ -331,26 +329,25 @@ PRIVATE> : with-fdb-map ( start-key end-key fdb_iterator_opt_t iterator-init iterator-next quot: ( obj -- ) -- ) [ execute ] 2dip + swap '[ _ &dispose handle>> [ - _ execute [ _ with-doc t ] [ f f ] if* swap + [ fdb-iterator-get ] keep swap + [ _ with-doc swap _ execute check-iterate-result ] + [ drop f ] if* swap ] curry collector-when [ loop ] dip ] with-destructors ; inline : with-fdb-normal-iterator ( start-key end-key quot -- ) - [ FDB_ITR_NONE \ fdb-iterator-init \ fdb-iterator-next ] dip - with-fdb-iterator ; inline - -: with-fdb-normal-meta-iterator ( start-key end-key quot -- ) - [ FDB_ITR_NONE \ fdb-iterator-init \ fdb-iterator-next-meta-only ] dip + [ FDB_ITR_NONE \ fdb-iterator-init \ fdb_iterator_next ] dip with-fdb-iterator ; inline : with-fdb-byseq-each ( start-seq end-seq quot -- ) - [ FDB_ITR_NONE \ fdb-iterator-byseq-init \ fdb-iterator-next-meta-only ] dip + [ FDB_ITR_NONE \ fdb-iterator-byseq-init \ fdb_iterator_next ] dip with-fdb-iterator ; inline : with-fdb-byseq-map ( start-seq end-seq quot -- ) - [ FDB_ITR_NONE \ fdb-iterator-byseq-init \ fdb-iterator-next-meta-only ] dip + [ FDB_ITR_NONE \ fdb-iterator-byseq-init \ fdb_iterator_next ] dip with-fdb-map ; inline ! Do not try to commit here, as it will fail with FDB_RESULT_RONLY_VIOLATION