forestdb: Support snapshots and rollbacks, change how with-forestdb works. Unit test things.

db4
Doug Coleman 2014-11-06 17:01:58 -08:00
parent 2c3dd8c3c8
commit 046e815a8c
3 changed files with 267 additions and 45 deletions

View File

@ -161,6 +161,7 @@ FUNCTION: fdb_status fdb_del_kv ( fdb_handle* handle, c-string key, size_t keyle
FUNCTION: fdb_status fdb_commit ( fdb_handle* handle, fdb_commit_opt_t opt ) ;
FUNCTION: fdb_status fdb_snapshot_open ( fdb_handle* handle_in, fdb_handle** handle_out, fdb_seqnum_t snapshot_seqnum ) ;
! Swaps out the handle for a new one
FUNCTION: fdb_status fdb_rollback ( fdb_handle** handle_ptr, fdb_seqnum_t rollback_seqnum ) ;
FUNCTION: fdb_status fdb_iterator_init ( fdb_handle* handle, fdb_iterator** iterator, c-string start_key, size_t start_keylen, c-string end_key, size_t end_keylen, fdb_iterator_opt_t opt ) ;

View File

@ -1,46 +1,210 @@
! Copyright (C) 2014 Doug Coleman.
! See http://factorcode.org/license.txt for BSD license.
USING: accessors alien.c-types alien.data alien.strings
combinators destructors forestdb.ffi fry io.files.temp
io.pathnames kernel libc namespaces sequences tools.test ;
USING: accessors alien.c-types alien.data alien.strings arrays
assocs combinators continuations destructors forestdb.ffi fry
io.directories io.files.temp io.pathnames kernel libc
math.parser math.ranges multiline namespaces sequences
tools.test ;
IN: forestdb.lib
: forestdb-test-path ( -- path )
"forestdb-test.fdb" temp-file ;
: test-db-0 ( -- path ) "0.forestdb.0" temp-file ;
: test-db-1 ( -- path ) "1.forestdb.0" temp-file ;
: delete-test-db-0 ( -- ) [ test-db-0 delete-file ] ignore-errors ;
: delete-test-db-1 ( -- ) [ test-db-1 delete-file ] ignore-errors ;
: make-kv-nth ( n -- key val )
number>string [ "key" prepend ] [ "val" prepend ] bi ;
: make-kv-n ( n -- seq )
[1,b] [ make-kv-nth ] { } map>assoc ;
: make-kv-range ( a b -- seq )
[a,b] [ make-kv-nth ] { } map>assoc ;
: set-kv-n ( n -- )
make-kv-n [ fdb-set-kv ] assoc-each ;
: set-kv-nth ( n -- )
make-kv-nth fdb-set-kv ;
: set-kv-range ( a b -- )
make-kv-range [ fdb-set-kv ] assoc-each ;
{ } [ [ delete-test-db-0 ] ignore-errors ] unit-test
{ } [ [ delete-test-db-1 ] ignore-errors ] unit-test
{ "val123" } [
forestdb-test-path [
delete-test-db-0
test-db-0 [
"key123" "val123" fdb-set-kv
"key123" fdb-get-kv
] with-forestdb
] with-forestdb-path
] unit-test
{ "val12345" } [
forestdb-test-path [
delete-test-db-0
test-db-0 [
"key123" "val12345" fdb-set-kv
"key123" fdb-get-kv
] with-forestdb
] with-forestdb-path
] unit-test
{ f } [
! Filename is only valid inside with-forestdb
forestdb-test-path [
get-current-db-info filename>> alien>native-string empty?
] with-forestdb
delete-test-db-0
test-db-0 [
fdb-info filename>> alien>native-string empty?
] with-forestdb-path
] unit-test
{ 6 9 9 } [
forestdb-test-path [
delete-test-db-0
test-db-0 [
"key123" "meta blah" "some body" fdb-doc-create
[ keylen>> ] [ metalen>> ] [ bodylen>> ] tri
] with-forestdb
] with-forestdb-path
] unit-test
{ 7 8 15 } [
forestdb-test-path [
delete-test-db-0
test-db-0 [
"key1234" "meta blah" "some body" fdb-doc-create
dup "new meta" "some other body" fdb-doc-update
[ keylen>> ] [ metalen>> ] [ bodylen>> ] tri
] with-forestdb
] with-forestdb-path
] unit-test
! Snapshots
{ 1 1 } [
delete-test-db-1
test-db-1 [
1 set-kv-n
fdb-commit-normal
fdb-info [ last_seqnum>> ] [ doc_count>> ] bi
] with-forestdb-path
] unit-test
{ 6 5 } [
delete-test-db-1
test-db-1 [
5 set-kv-n
5 set-kv-nth
fdb-commit-normal
fdb-info [ last_seqnum>> ] [ doc_count>> ] bi
] with-forestdb-path
] unit-test
{ 5 5 } [
delete-test-db-1
test-db-1 [
5 set-kv-n
fdb-commit-normal
fdb-info [ last_seqnum>> ] [ doc_count>> ] bi
] with-forestdb-path
] unit-test
{ 5 5 } [
delete-test-db-1
test-db-1 [
5 set-kv-n
fdb-commit-normal
5 fdb-open-snapshot [
fdb-info [ last_seqnum>> ] [ doc_count>> ] bi
] with-forestdb-snapshot
] with-forestdb-path
] unit-test
! Snapshots can only occur on commits. If you commit five keys at once,
! and then try to open a snapshot on the second key, it should fail.
[
delete-test-db-1
test-db-1 [
5 set-kv-n
fdb-commit-normal
2 fdb-open-snapshot [
fdb-info [ last_seqnum>> ] [ doc_count>> ] bi
] with-forestdb-snapshot
] with-forestdb-path
] [
T{ fdb-error { error FDB_RESULT_NO_DB_INSTANCE } } =
] must-fail-with
! Test that we take two snapshots and their seqnums/doc counts are right.
! XXX: We test this to make sure the forestdb doesn't change.
! Bug in forestdb? doc_count>> should be 5 at snapshot 5
{
{ 5 7 }
{ 7 7 }
} [
delete-test-db-1
test-db-1 [
5 set-kv-n
fdb-commit-normal
6 7 set-kv-range
fdb-commit-normal
5 fdb-open-snapshot [
fdb-info [ last_seqnum>> ] [ doc_count>> ] bi 2array
] with-forestdb-snapshot
7 fdb-open-snapshot [
fdb-info [ last_seqnum>> ] [ doc_count>> ] bi 2array
] with-forestdb-snapshot
] with-forestdb-path
] unit-test
! Same test as above, but with buggy behavior for now so it passes
{
5
7
} [
delete-test-db-1
test-db-1 [
5 set-kv-n
fdb-commit-normal
6 7 set-kv-range
fdb-commit-normal
5 fdb-open-snapshot [
fdb-info last_seqnum>>
] with-forestdb-snapshot
7 fdb-open-snapshot [
fdb-info last_seqnum>>
] with-forestdb-snapshot
] with-forestdb-path
] unit-test
! Rollback test
! Make sure the doc_count is correct after a rollback
{
7
{ 5 5 }
} [
delete-test-db-1
test-db-1 [
5 set-kv-n
fdb-commit-normal
6 7 set-kv-range
fdb-commit-normal
7 fdb-open-snapshot [
fdb-info last_seqnum>>
] with-forestdb-snapshot
5 fdb-rollback
5 fdb-open-snapshot [
fdb-info [ last_seqnum>> ] [ doc_count>> ] bi 2array
] with-forestdb-snapshot
] with-forestdb-path
] unit-test

View File

@ -1,14 +1,15 @@
! Copyright (C) 2014 Doug Coleman.
! See http://factorcode.org/license.txt for BSD license.
USING: accessors alien.c-types alien.data alien.strings
classes.struct combinators destructors forestdb.ffi fry
generalizations io.encodings.string io.encodings.utf8
io.pathnames kernel libc namespaces sequences ;
alien.syntax classes.struct combinators continuations
destructors forestdb.ffi fry generalizations io.encodings.string
io.encodings.utf8 io.pathnames kernel libc multiline namespaces
sequences ;
IN: forestdb.lib
ERROR: fdb-error error ;
: check-forestdb-error ( ret -- )
: fdb-check-error ( ret -- )
dup FDB_RESULT_SUCCESS = [
drop
] [
@ -21,37 +22,38 @@ TUPLE: fdb-handle < disposable handle ;
swap >>handle ; inline
M: fdb-handle dispose*
handle>> fdb_close check-forestdb-error ;
handle>> fdb_close fdb-check-error ;
TUPLE: fdb-doc < disposable doc ;
M: fdb-doc dispose*
fdb_doc_free check-forestdb-error ;
fdb_doc_free fdb-check-error ;
: open-default-forestdb ( path -- handle )
: fdb-open ( path -- handle )
[ f void* <ref> ] dip
absolute-path f
[ fdb_open check-forestdb-error ] 3keep 2drop void* deref <fdb-handle> ;
[ fdb_open fdb-check-error ] 3keep
2drop void* deref <fdb-handle> ;
: ret>string ( void** len -- string )
[ void* deref ] [ size_t deref ] bi*
[ memory>byte-array utf8 decode ] [ drop (free) ] 2bi ;
SYMBOL: current-forestdb
SYMBOL: fdb-current
: get-handle ( -- handle )
current-forestdb get handle>> ;
fdb-current get handle>> ;
: fdb-set-kv ( key value -- )
[ get-handle ] 2dip
[ dup length ] bi@ fdb_set_kv check-forestdb-error ;
[ dup length ] bi@ fdb_set_kv fdb-check-error ;
: fdb-set ( doc -- )
[ get-handle ] dip fdb_set check-forestdb-error ;
[ get-handle ] dip fdb_set fdb-check-error ;
: fdb-del ( doc -- )
[ get-handle ] dip fdb_del check-forestdb-error ;
[ get-handle ] dip fdb_del fdb-check-error ;
: fdb-get-kv ( key -- value/f )
[ get-handle ] dip
@ -64,36 +66,91 @@ SYMBOL: current-forestdb
} case ;
: fdb-del-kv ( key -- )
[ get-handle ] dip dup length fdb_del_kv check-forestdb-error ;
[ get-handle ] dip dup length fdb_del_kv fdb-check-error ;
: fdb-doc-create ( key meta body -- doc )
[ f void* <ref> ] 3dip
[ dup length ] tri@
[ fdb_doc_create check-forestdb-error ] 7 nkeep 6 ndrop
[ fdb_doc_create fdb-check-error ] 7 nkeep 6 ndrop
void* deref fdb_doc memory>struct ;
: fdb-doc-update ( doc meta body -- )
[ void* <ref> ] 2dip
[ dup length ] bi@
fdb_doc_update check-forestdb-error ;
fdb_doc_update fdb-check-error ;
: fdb-doc-free ( doc -- )
fdb_doc_free check-forestdb-error ;
fdb_doc_free fdb-check-error ;
: get-current-db-info ( -- info )
: fdb-info ( -- info )
get-handle
fdb_info <struct> [ fdb_get_dbinfo check-forestdb-error ] keep ;
fdb_info <struct> [ fdb_get_dbinfo fdb-check-error ] keep ;
: commit-forestdb ( -- )
get-handle FDB_COMMIT_NORMAL fdb_commit check-forestdb-error ;
: fdb-commit ( fdb_commit_opt_t -- )
[ get-handle ] dip fdb_commit fdb-check-error ;
: with-forestdb ( path quot -- )
[ absolute-path open-default-forestdb ] dip
dupd '[
_ current-forestdb [
_ &dispose drop
@
commit-forestdb
: fdb-maybe-commit ( fdb_commit_opt_t/f -- )
[ fdb-commit ] when* ;
: fdb-commit-normal ( -- ) FDB_COMMIT_NORMAL fdb-commit ;
: fdb-commit-wal-flush ( -- ) FDB_COMMIT_MANUAL_WAL_FLUSH fdb-commit ;
FUNCTION: fdb_status fdb_rollback ( fdb_handle** handle_ptr, fdb_seqnum_t rollback_seqnum ) ;
! Call from within with-foresdb
: fdb-open-snapshot ( seqnum -- handle )
[
get-handle
f void* <ref>
] dip [
fdb_snapshot_open fdb-check-error
] 2keep drop void* deref <fdb-handle> ;
! fdb_rollback returns a new handle, so we
! have to replace our current handle with that one
! XXX: can't call dispose on old handle, library handles that
: fdb-rollback ( seqnum -- )
[ get-handle void* <ref> ] dip
[ fdb_rollback fdb-check-error ] 2keep drop
void* deref <fdb-handle> fdb-current set ;
/*
FUNCTION: fdb_status fdb_iterator_init ( fdb_handle* handle, fdb_iterator** iterator, c-string start_key, size_t start_keylen, c-string end_key, size_t end_keylen, fdb_iterator_opt_t opt ) ;
FUNCTION: fdb_status fdb_iterator_sequence_init ( fdb_handle* handle, fdb_iterator** iterator, fdb_seqnum_t start_seq, fdb_seqnum_t end_seq, fdb_iterator_opt_t opt ) ;
FUNCTION: fdb_status fdb_iterator_prev ( fdb_iterator* iterator, fdb_doc** doc ) ;
FUNCTION: fdb_status fdb_iterator_next ( fdb_iterator* iterator, fdb_doc** doc ) ;
FUNCTION: fdb_status fdb_iterator_next_metaonly ( fdb_iterator* iterator, fdb_doc** doc ) ;
FUNCTION: fdb_status fdb_iterator_seek ( fdb_iterator* iterator, c-string seek_key, size_t seek_keylen ) ;
FUNCTION: fdb_status fdb_iterator_close ( fdb_iterator* iterator ) ;
*/
! Do not try to commit here, as it will fail with FDB_RESULT_RONLY_VIOLATION
! fdb-current is weird, it gets replaced if you call fdb-rollback
! Therefore, only clean up fdb-current once, and clean it up at the end
: with-forestdb-handle ( handle quot fdb_commit_opt_t/f -- )
'[
_ fdb-current [
[
@
_ fdb-maybe-commit
fdb-current get &dispose drop
] [
fdb-current get &dispose drop
rethrow
] recover
] with-variable
] with-destructors ; inline
! Commit normal at the end
: with-forestdb-handle-commit-normal ( handle quot commit -- )
FDB_COMMIT_NORMAL with-forestdb-handle ; inline
: with-forestdb-handle-commit-wal ( handle quot commit -- )
FDB_COMMIT_MANUAL_WAL_FLUSH with-forestdb-handle ; inline
: with-forestdb-snapshot ( handle quot commit -- )
f with-forestdb-handle ; inline
: with-forestdb-path ( path quot -- )
[ absolute-path fdb-open ] dip with-forestdb-handle-commit-normal ; inline