diff --git a/vm/code_block_visitor.hpp b/vm/code_block_visitor.hpp new file mode 100644 index 0000000000..12146a940e --- /dev/null +++ b/vm/code_block_visitor.hpp @@ -0,0 +1,79 @@ +namespace factor +{ + +template struct call_frame_code_block_visitor { + Visitor visitor; + + explicit call_frame_code_block_visitor(Visitor visitor_) : visitor(visitor_) {} + + void operator()(stack_frame *frame) + { + cell offset = (cell)FRAME_RETURN_ADDRESS(frame,parent) - (cell)frame->xt; + + code_block *new_block = visitor.visit_code_block(parent->frame_code(frame)); + frame->xt = new_block->xt(); + + FRAME_RETURN_ADDRESS(frame,parent) = (void *)((cell)frame->xt + offset); + } +}; + +template void factor_vm::visit_object_code_block(object *obj, Visitor visitor) +{ + switch(obj->h.hi_tag()) + { + case WORD_TYPE: + { + word *w = (word *)obj; + if(w->code) + w->code = visitor.visit_code_block(w->code); + if(w->profiling) + w->code = visitor.visit_code_block(w->profiling); + + update_word_xt(obj); + break; + } + case QUOTATION_TYPE: + { + quotation *q = (quotation *)obj; + if(q->code) + set_quot_xt(visitor.visit_code_block(q->code)); + break; + } + case CALLSTACK_TYPE: + { + callstack *stack = (callstack *)obj; + call_frame_code_block_visitor call_frame_visitor(visitor); + iterate_callstack_object(stack,call_frame_visitor); + break; + } + } +} + +template void factor_vm::visit_context_code_blocks(Visitor visitor) +{ + callstack *stack = (callstack *)obj; + call_frame_code_block_visitor call_frame_visitor(visitor); + iterate_active_frames(call_frame_visitor); +} + +template struct callback_code_block_visitor { + callback_heap *callbacks; + Visitor visitor; + + explicit callback_code_block_visitor(callback_heap *callbacks_, Visitor visitor_) : + callbacks(callbacks_), visitor(visitor_) {} + + void operator()(callback *stub) + { + stub->compiled = visitor.visit_code_block(stub->compiled); + callbacks->update(stub); + } +}; + +template void factor_vm::visit_callback_code_blocks(Visitor visitor) +{ + callback_code_block_visitor callback_visitor(callbacks,visitor); + callbacks->iterate(callback_visitor); +} + +} diff --git a/vm/code_heap.cpp b/vm/code_heap.cpp index 5ae55cb760..c837ec7615 100755 --- a/vm/code_heap.cpp +++ b/vm/code_heap.cpp @@ -188,7 +188,7 @@ void factor_vm::primitive_modify_code_heap() break; } - update_word_xt(word.value()); + update_word_xt(word.untagged()); } update_code_heap_words(); @@ -205,99 +205,42 @@ void factor_vm::primitive_code_room() dpush(tag_fixnum(max_free / 1024)); } -code_block *code_heap::forward_code_block(code_block *compiled) -{ - return (code_block *)allocator->state.forward_block(compiled); -} +struct code_block_forwarder { + mark_bits *forwarding_map; -struct callframe_forwarder { - factor_vm *parent; + explicit code_block_forwarder(mark_bits *forwarding_map_) : + forwarding_map(forwarding_map_) {} - explicit callframe_forwarder(factor_vm *parent_) : parent(parent_) {} - - void operator()(stack_frame *frame) + code_block *operator()(code_block *compiled) { - cell offset = (cell)FRAME_RETURN_ADDRESS(frame,parent) - (cell)frame->xt; - - code_block *forwarded = parent->code->forward_code_block(parent->frame_code(frame)); - frame->xt = forwarded->xt(); - - FRAME_RETURN_ADDRESS(frame,parent) = (void *)((cell)frame->xt + offset); + return (code_block *)forwarding_map->forward_block(compiled); } }; void factor_vm::forward_object_xts() { + code_block_forwarder forwarder(&code->allocator->state); + begin_scan(); cell obj; while(to_boolean(obj = next_object())) - { - switch(tagged(obj).type()) - { - case WORD_TYPE: - { - word *w = untag(obj); - - if(w->code) - w->code = code->forward_code_block(w->code); - if(w->profiling) - w->profiling = code->forward_code_block(w->profiling); - - update_word_xt(obj); - } - break; - case QUOTATION_TYPE: - { - quotation *quot = untag(obj); - - if(quot->code) - { - quot->code = code->forward_code_block(quot->code); - set_quot_xt(quot,quot->code); - } - } - break; - case CALLSTACK_TYPE: - { - callstack *stack = untag(obj); - callframe_forwarder forwarder(this); - iterate_callstack_object(stack,forwarder); - } - break; - default: - break; - } - } + visit_object_code_block(untag(obj),forwarder); end_scan(); } void factor_vm::forward_context_xts() { - callframe_forwarder forwarder(this); - iterate_active_frames(forwarder); + code_block_forwarder forwarder(&code->allocator->state); + visit_context_code_blocks(forwarder); } -struct callback_forwarder { - code_heap *code; - callback_heap *callbacks; - - callback_forwarder(code_heap *code_, callback_heap *callbacks_) : - code(code_), callbacks(callbacks_) {} - - void operator()(callback *stub) - { - stub->compiled = code->forward_code_block(stub->compiled); - callbacks->update(stub); - } -}; - void factor_vm::forward_callback_xts() { - callback_forwarder forwarder(code,callbacks); - callbacks->iterate(forwarder); + code_block_forwarder forwarder(&code->allocator->state); + visit_callback_code_blocks(forwarder); } /* Move all free space to the end of the code heap. Live blocks must be marked diff --git a/vm/code_heap.hpp b/vm/code_heap.hpp index 2d9961c03a..5548892d3f 100755 --- a/vm/code_heap.hpp +++ b/vm/code_heap.hpp @@ -26,7 +26,6 @@ struct code_heap { void set_marked_p(code_block *compiled); void clear_mark_bits(); void code_heap_free(code_block *compiled); - code_block *forward_code_block(code_block *compiled); }; } diff --git a/vm/collector.hpp b/vm/collector.hpp index a1a7dc5695..6f6a5e31cd 100644 --- a/vm/collector.hpp +++ b/vm/collector.hpp @@ -1,20 +1,14 @@ namespace factor { -template struct collector { +template struct collector_workhorse { factor_vm *parent; - data_heap *data; - code_heap *code; - gc_state *current_gc; generation_statistics *stats; TargetGeneration *target; Policy policy; - explicit collector(factor_vm *parent_, generation_statistics *stats_, TargetGeneration *target_, Policy policy_) : + explicit collector_workhorse(factor_vm *parent_, generation_statistics *stats_, TargetGeneration *target_, Policy policy_) : parent(parent_), - data(parent_->data), - code(parent_->code), - current_gc(parent_->current_gc), stats(stats_), target(target_), policy(policy_) {} @@ -32,52 +26,12 @@ template struct collector { return untagged; } - void trace_handle(cell *handle) - { - cell pointer = *handle; - - if(immediate_p(pointer)) return; - - object *untagged = untag(pointer); - if(!policy.should_copy_p(untagged)) - { - policy.visited_object(untagged); - return; - } - - object *forwarding = resolve_forwarding(untagged); - - if(forwarding == untagged) - untagged = promote_object(untagged); - else if(policy.should_copy_p(forwarding)) - untagged = promote_object(forwarding); - else - { - untagged = forwarding; - policy.visited_object(untagged); - } - - *handle = RETAG(untagged,TAG(pointer)); - } - - void trace_slots(object *ptr) - { - cell *slot = (cell *)ptr; - cell *end = (cell *)((cell)ptr + parent->binary_payload_start(ptr)); - - if(slot != end) - { - slot++; - for(; slot < end; slot++) trace_handle(slot); - } - } - object *promote_object(object *untagged) { cell size = untagged->size(); object *newpointer = target->allot(size); /* XXX not exception-safe */ - if(!newpointer) longjmp(current_gc->gc_unwind,1); + if(!newpointer) longjmp(parent->current_gc->gc_unwind,1); memcpy(newpointer,untagged,size); untagged->h.forward_to(newpointer); @@ -90,68 +44,90 @@ template struct collector { return newpointer; } - void trace_stack_elements(segment *region, cell *top) + object *visit_handle(object *obj) { - for(cell *ptr = (cell *)region->start; ptr <= top; ptr++) - trace_handle(ptr); - } - - void trace_registered_locals() - { - std::vector::const_iterator iter = parent->gc_locals.begin(); - std::vector::const_iterator end = parent->gc_locals.end(); - - for(; iter < end; iter++) - trace_handle((cell *)(*iter)); - } - - void trace_registered_bignums() - { - std::vector::const_iterator iter = parent->gc_bignums.begin(); - std::vector::const_iterator end = parent->gc_bignums.end(); - - for(; iter < end; iter++) + if(!policy.should_copy_p(obj)) { - cell *handle = (cell *)(*iter); + policy.visited_object(obj); + return obj; + } - if(*handle) - { - *handle |= BIGNUM_TYPE; - trace_handle(handle); - *handle &= ~BIGNUM_TYPE; - } + object *forwarding = resolve_forwarding(obj); + + if(forwarding == obj) + return promote_object(obj); + else if(policy.should_copy_p(forwarding)) + return promote_object(forwarding); + else + { + policy.visited_object(forwarding); + return forwarding; } } +}; + +template +inline static slot_visitor > make_collector_workhorse( + factor_vm *parent, + generation_statistics *stats, + TargetGeneration *target, + Policy policy) +{ + return slot_visitor >(parent, + collector_workhorse(parent,stats,target,policy)); +} + +template struct collector { + factor_vm *parent; + data_heap *data; + code_heap *code; + generation_statistics *stats; + TargetGeneration *target; + slot_visitor > workhorse; + + explicit collector(factor_vm *parent_, generation_statistics *stats_, TargetGeneration *target_, Policy policy_) : + parent(parent_), + data(parent_->data), + code(parent_->code), + stats(stats_), + target(target_), + workhorse(make_collector_workhorse(parent_,stats_,target_,policy_)) {} + + void trace_handle(cell *handle) + { + workhorse.visit_handle(handle); + } + + void trace_slots(object *ptr) + { + workhorse.visit_slots(ptr); + } - /* Copy roots over at the start of GC, namely various constants, stacks, - the user environment and extra roots registered by local_roots.hpp */ void trace_roots() { - trace_handle(&parent->true_object); - trace_handle(&parent->bignum_zero); - trace_handle(&parent->bignum_pos_one); - trace_handle(&parent->bignum_neg_one); - - trace_registered_locals(); - trace_registered_bignums(); - - for(cell i = 0; i < special_object_count; i++) - trace_handle(&parent->special_objects[i]); + workhorse.visit_roots(); } void trace_contexts() { - context *ctx = parent->ctx; + workhorse.visit_contexts(); + } - while(ctx) + /* Trace all literals referenced from a code block. Only for aging and nursery collections */ + void trace_literal_references(code_block *compiled) + { + workhorse.visit_literal_references(compiled); + } + + void trace_code_heap_roots(std::set *remembered_set) + { + std::set::const_iterator iter = remembered_set->begin(); + std::set::const_iterator end = remembered_set->end(); + + for(; iter != end; iter++) { - trace_stack_elements(ctx->datastack_region,(cell *)ctx->datastack); - trace_stack_elements(ctx->retainstack_region,(cell *)ctx->retainstack); - - trace_handle(&ctx->catchstack_save); - trace_handle(&ctx->current_callback_save); - - ctx = ctx->next; + trace_literal_references(*iter); + parent->gc_stats.code_blocks_scanned++; } } @@ -167,17 +143,17 @@ template struct collector { inline cell card_deck_for_address(cell a) { - return addr_to_deck(a - this->data->start); + return addr_to_deck(a - data->start); } inline cell card_start_address(cell card) { - return (card << card_bits) + this->data->start; + return (card << card_bits) + data->start; } inline cell card_end_address(cell card) { - return ((card + 1) << card_bits) + this->data->start; + return ((card + 1) << card_bits) + data->start; } void trace_partial_objects(cell start, cell end, cell card_start, cell card_end) @@ -195,7 +171,7 @@ template struct collector { if(slot_ptr != end_ptr) { for(; slot_ptr < end_ptr; slot_ptr++) - this->trace_handle(slot_ptr); + workhorse.visit_handle(slot_ptr); } } } @@ -205,10 +181,10 @@ template struct collector { { u64 start_time = current_micros(); - card_deck *decks = this->data->decks; - card_deck *cards = this->data->cards; + card_deck *decks = data->decks; + card_deck *cards = data->cards; - cell gen_start_card = addr_to_card(gen->start - this->data->start); + cell gen_start_card = addr_to_card(gen->start - data->start); cell first_deck = card_deck_for_address(gen->start); cell last_deck = card_deck_for_address(gen->end); @@ -219,7 +195,7 @@ template struct collector { { if(decks[deck_index] & mask) { - this->parent->gc_stats.decks_scanned++; + parent->gc_stats.decks_scanned++; cell first_card = first_card_in_deck(deck_index); cell last_card = last_card_in_deck(deck_index); @@ -228,17 +204,17 @@ template struct collector { { if(cards[card_index] & mask) { - this->parent->gc_stats.cards_scanned++; + parent->gc_stats.cards_scanned++; if(end < card_start_address(card_index)) { start = gen->starts.find_object_containing_card(card_index - gen_start_card); - binary_start = start + this->parent->binary_payload_start((object *)start); + binary_start = start + parent->binary_payload_start((object *)start); end = start + ((object *)start)->size(); } #ifdef FACTOR_DEBUG - assert(addr_to_card(start - this->data->start) <= card_index); + assert(addr_to_card(start - data->start) <= card_index); assert(start < card_end_address(card_index)); #endif @@ -253,7 +229,7 @@ scan_next_object: { start = gen->next_object_after(start); if(start) { - binary_start = start + this->parent->binary_payload_start((object *)start); + binary_start = start + parent->binary_payload_start((object *)start); end = start + ((object *)start)->size(); goto scan_next_object; } @@ -270,24 +246,7 @@ scan_next_object: { } } -end: this->parent->gc_stats.card_scan_time += (current_micros() - start_time); - } - - /* Trace all literals referenced from a code block. Only for aging and nursery collections */ - void trace_literal_references(code_block *compiled) - { - this->trace_handle(&compiled->owner); - this->trace_handle(&compiled->literals); - this->trace_handle(&compiled->relocation); - this->parent->gc_stats.code_blocks_scanned++; - } - - void trace_code_heap_roots(std::set *remembered_set) - { - std::set::const_iterator iter = remembered_set->begin(); - std::set::const_iterator end = remembered_set->end(); - - for(; iter != end; iter++) trace_literal_references(*iter); +end: parent->gc_stats.card_scan_time += (current_micros() - start_time); } }; diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index 817908ece5..fbffe5b33e 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -10,101 +10,36 @@ full_collector::full_collector(factor_vm *parent_) : parent_->data->tenured, full_policy(parent_)) {} -struct stack_frame_marker { - factor_vm *parent; +struct code_block_marker { + code_heap *code; full_collector *collector; - explicit stack_frame_marker(full_collector *collector_) : - parent(collector_->parent), collector(collector_) {} + explicit code_block_marker(code_heap *code_, full_collector *collector_) : + code(code_), collector(collector_) {} - void operator()(stack_frame *frame) + code_block *operator()(code_block *compiled) { - collector->mark_code_block(parent->frame_code(frame)); + if(!code->marked_p(compiled)) + { + code->set_marked_p(compiled); + collector->trace_literal_references(compiled); + } + + return compiled; } }; -/* Mark code blocks executing in currently active stack frames. */ -void full_collector::mark_active_blocks() -{ - stack_frame_marker marker(this); - parent->iterate_active_frames(marker); -} - -void full_collector::mark_object_code_block(object *obj) -{ - switch(obj->h.hi_tag()) - { - case WORD_TYPE: - { - word *w = (word *)obj; - if(w->code) - mark_code_block(w->code); - if(w->profiling) - mark_code_block(w->profiling); - break; - } - case QUOTATION_TYPE: - { - quotation *q = (quotation *)obj; - if(q->code) - mark_code_block(q->code); - break; - } - case CALLSTACK_TYPE: - { - callstack *stack = (callstack *)obj; - stack_frame_marker marker(this); - parent->iterate_callstack_object(stack,marker); - break; - } - } -} - -struct callback_tracer { - full_collector *collector; - - callback_tracer(full_collector *collector_) : collector(collector_) {} - - void operator()(callback *stub) - { - collector->mark_code_block(stub->compiled); - } -}; - -void full_collector::trace_callbacks() -{ - callback_tracer tracer(this); - parent->callbacks->iterate(tracer); -} - -/* Trace all literals referenced from a code block. Only for aging and nursery collections */ -void full_collector::trace_literal_references(code_block *compiled) -{ - this->trace_handle(&compiled->owner); - this->trace_handle(&compiled->literals); - this->trace_handle(&compiled->relocation); -} - -/* Mark all literals referenced from a word XT. Only for tenured -collections */ -void full_collector::mark_code_block(code_block *compiled) -{ - if(!this->code->marked_p(compiled)) - { - this->code->set_marked_p(compiled); - trace_literal_references(compiled); - } -} - void full_collector::mark_reachable_objects() { + code_block_marker marker(code,this); std::vector *mark_stack = &this->target->mark_stack; + while(!mark_stack->empty()) { object *obj = mark_stack->back(); mark_stack->pop_back(); this->trace_slots(obj); - this->mark_object_code_block(obj); + parent->visit_object_code_block(obj,marker); } } @@ -131,8 +66,9 @@ void factor_vm::collect_full_impl(bool trace_contexts_p) if(trace_contexts_p) { collector.trace_contexts(); - collector.mark_active_blocks(); - collector.trace_callbacks(); + code_block_marker marker(code,&collector); + visit_context_code_blocks(marker); + visit_callback_code_blocks(marker); } collector.mark_reachable_objects(); diff --git a/vm/full_collector.hpp b/vm/full_collector.hpp index 9aef352b4b..f613558997 100644 --- a/vm/full_collector.hpp +++ b/vm/full_collector.hpp @@ -28,11 +28,6 @@ struct full_collector : collector { bool trace_contexts_p; full_collector(factor_vm *parent_); - void mark_active_blocks(); - void mark_object_code_block(object *object); - void trace_callbacks(); - void trace_literal_references(code_block *compiled); - void mark_code_block(code_block *compiled); void mark_reachable_objects(); }; diff --git a/vm/master.hpp b/vm/master.hpp index 9168cecce4..dc01429c30 100755 --- a/vm/master.hpp +++ b/vm/master.hpp @@ -72,11 +72,13 @@ namespace factor #include "vm.hpp" #include "tagged.hpp" #include "local_roots.hpp" +#include "slot_visitor.hpp" #include "collector.hpp" #include "copying_collector.hpp" #include "nursery_collector.hpp" #include "aging_collector.hpp" #include "to_tenured_collector.hpp" +#include "code_block_visitor.hpp" #include "full_collector.hpp" #include "callstack.hpp" #include "generic_arrays.hpp" diff --git a/vm/profiler.cpp b/vm/profiler.cpp index 5113b55cf7..2f5fc6fcf4 100755 --- a/vm/profiler.cpp +++ b/vm/profiler.cpp @@ -40,7 +40,7 @@ void factor_vm::set_profiling(bool profiling) tagged word(array_nth(words.untagged(),i)); if(profiling) word->counter = tag_fixnum(0); - update_word_xt(word.value()); + update_word_xt(word.untagged()); } update_code_heap_words(); diff --git a/vm/quotations.cpp b/vm/quotations.cpp index c65c0fe909..17b7c4328b 100755 --- a/vm/quotations.cpp +++ b/vm/quotations.cpp @@ -338,7 +338,7 @@ void factor_vm::compile_all_words() if(!word->code || !word->code->optimized_p()) jit_compile_word(word.value(),word->def,false); - update_word_xt(word.value()); + update_word_xt(word.untagged()); } diff --git a/vm/slot_visitor.hpp b/vm/slot_visitor.hpp new file mode 100644 index 0000000000..a62fdd87ec --- /dev/null +++ b/vm/slot_visitor.hpp @@ -0,0 +1,101 @@ +namespace factor +{ + +template struct slot_visitor { + factor_vm *parent; + Visitor visitor; + + slot_visitor(factor_vm *parent_, Visitor visitor_) : + parent(parent_), visitor(visitor_) {} + + void visit_handle(cell *handle) + { + cell pointer = *handle; + + if(immediate_p(pointer)) return; + + object *untagged = untag(pointer); + untagged = visitor.visit_handle(untagged); + *handle = RETAG(untagged,TAG(pointer)); + } + + void visit_slots(object *ptr) + { + cell *slot = (cell *)ptr; + cell *end = (cell *)((cell)ptr + parent->binary_payload_start(ptr)); + + if(slot != end) + { + slot++; + for(; slot < end; slot++) visit_handle(slot); + } + } + + void visit_stack_elements(segment *region, cell *top) + { + for(cell *ptr = (cell *)region->start; ptr <= top; ptr++) + visit_handle(ptr); + } + + void visit_registered_locals() + { + std::vector::const_iterator iter = parent->gc_locals.begin(); + std::vector::const_iterator end = parent->gc_locals.end(); + + for(; iter < end; iter++) + visit_handle((cell *)(*iter)); + } + + void visit_registered_bignums() + { + std::vector::const_iterator iter = parent->gc_bignums.begin(); + std::vector::const_iterator end = parent->gc_bignums.end(); + + for(; iter < end; iter++) + { + cell *handle = (cell *)(*iter); + + if(*handle) + *handle = (cell)visitor.visit_handle(*(object **)handle); + } + } + + void visit_roots() + { + visit_handle(&parent->true_object); + visit_handle(&parent->bignum_zero); + visit_handle(&parent->bignum_pos_one); + visit_handle(&parent->bignum_neg_one); + + visit_registered_locals(); + visit_registered_bignums(); + + for(cell i = 0; i < special_object_count; i++) + visit_handle(&parent->special_objects[i]); + } + + void visit_contexts() + { + context *ctx = parent->ctx; + + while(ctx) + { + visit_stack_elements(ctx->datastack_region,(cell *)ctx->datastack); + visit_stack_elements(ctx->retainstack_region,(cell *)ctx->retainstack); + + visit_handle(&ctx->catchstack_save); + visit_handle(&ctx->current_callback_save); + + ctx = ctx->next; + } + } + + void visit_literal_references(code_block *compiled) + { + visit_handle(&compiled->owner); + visit_handle(&compiled->literals); + visit_handle(&compiled->relocation); + } +}; + +} diff --git a/vm/vm.hpp b/vm/vm.hpp index e22e45f22e..674f934bba 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -353,7 +353,7 @@ struct factor_vm word *allot_word(cell name_, cell vocab_, cell hashcode_); void primitive_word(); void primitive_word_xt(); - void update_word_xt(cell w_); + void update_word_xt(word *w_); void primitive_optimized_p(); void primitive_wrapper(); @@ -485,6 +485,11 @@ struct factor_vm code_block *allot_code_block(cell size, code_block_type type); code_block *add_code_block(code_block_type type, cell code_, cell labels_, cell owner_, cell relocation_, cell literals_); + //code_block_visitor + template void visit_object_code_block(object *obj, Visitor visitor); + template void visit_context_code_blocks(Visitor visitor); + template void visit_callback_code_blocks(Visitor visitor); + //code heap inline void check_code_pointer(cell ptr) { diff --git a/vm/words.cpp b/vm/words.cpp index 37a3821069..4248c14b7d 100644 --- a/vm/words.cpp +++ b/vm/words.cpp @@ -23,7 +23,7 @@ word *factor_vm::allot_word(cell name_, cell vocab_, cell hashcode_) new_word->code = NULL; jit_compile_word(new_word.value(),new_word->def,true); - update_word_xt(new_word.value()); + update_word_xt(new_word.untagged()); if(profiling_p) relocate_code_block(new_word->profiling); @@ -59,7 +59,7 @@ void factor_vm::primitive_word_xt() } /* Allocates memory */ -void factor_vm::update_word_xt(cell w_) +void factor_vm::update_word_xt(word *w_) { gc_root w(w_,this);