diff --git a/vm/collector.hpp b/vm/collector.hpp index 31d830256d..a52d5f97b1 100644 --- a/vm/collector.hpp +++ b/vm/collector.hpp @@ -39,7 +39,7 @@ template struct collector_workhorse return newpointer; } - object *visit_object(object *obj) + object *operator()(object *obj) { if(!policy.should_copy_p(obj)) { diff --git a/vm/compaction.cpp b/vm/compaction.cpp index dee922d2f4..eb11ec9f3d 100644 --- a/vm/compaction.cpp +++ b/vm/compaction.cpp @@ -2,27 +2,15 @@ namespace factor { -struct object_slot_forwarder { - mark_bits *forwarding_map; +template struct forwarder { + mark_bits *forwarding_map; - explicit object_slot_forwarder(mark_bits *forwarding_map_) : + explicit forwarder(mark_bits *forwarding_map_) : forwarding_map(forwarding_map_) {} - object *visit_object(object *obj) + Block *operator()(Block *block) { - return forwarding_map->forward_block(obj); - } -}; - -struct code_block_forwarder { - mark_bits *forwarding_map; - - explicit code_block_forwarder(mark_bits *forwarding_map_) : - forwarding_map(forwarding_map_) {} - - code_block *operator()(code_block *compiled) - { - return forwarding_map->forward_block(compiled); + return forwarding_map->forward_block(block); } }; @@ -56,7 +44,7 @@ struct compaction_sizer { cell operator()(object *obj) { if(!forwarding_map->marked_p(obj)) - return forwarding_map->unmarked_space_starting_at(obj); + return forwarding_map->unmarked_block_size(obj); else if(obj->h.hi_tag() == TUPLE_TYPE) return align(tuple_size_with_forwarding(forwarding_map,obj),data_alignment); else @@ -66,14 +54,14 @@ struct compaction_sizer { struct object_compaction_updater { factor_vm *parent; - slot_visitor slot_forwarder; - code_block_visitor code_forwarder; + slot_visitor > slot_forwarder; + code_block_visitor > code_forwarder; mark_bits *data_forwarding_map; object_start_map *starts; explicit object_compaction_updater(factor_vm *parent_, - slot_visitor slot_forwarder_, - code_block_visitor code_forwarder_, + slot_visitor > slot_forwarder_, + code_block_visitor > code_forwarder_, mark_bits *data_forwarding_map_) : parent(parent_), slot_forwarder(slot_forwarder_), @@ -125,9 +113,8 @@ void factor_vm::collect_compact_impl(bool trace_contexts_p) data_forwarding_map->compute_forwarding(); code_forwarding_map->compute_forwarding(); - /* Update root pointers */ - slot_visitor slot_forwarder(this,object_slot_forwarder(data_forwarding_map)); - code_block_visitor code_forwarder(this,code_block_forwarder(code_forwarding_map)); + slot_visitor > slot_forwarder(this,forwarder(data_forwarding_map)); + code_block_visitor > code_forwarder(this,forwarder(code_forwarding_map)); /* Object start offsets get recomputed by the object_compaction_updater */ data->tenured->starts.clear_object_start_offsets(); @@ -140,7 +127,7 @@ void factor_vm::collect_compact_impl(bool trace_contexts_p) /* Slide everything in the code heap up, and update data and code heap pointers inside code blocks. */ - code_block_compaction_updater > code_block_updater(this,slot_forwarder); + code_block_compaction_updater > > code_block_updater(this,slot_forwarder); standard_sizer code_block_sizer; code->allocator->compact(code_block_updater,code_block_sizer); @@ -156,14 +143,14 @@ void factor_vm::collect_compact_impl(bool trace_contexts_p) } struct object_code_block_updater { - code_block_visitor *forwarder; + code_block_visitor > *visitor; - explicit object_code_block_updater(code_block_visitor *forwarder_) : - forwarder(forwarder_) {} + explicit object_code_block_updater(code_block_visitor > *visitor_) : + visitor(visitor_) {} void operator()(cell obj) { - forwarder->visit_object_code_block(tagged(obj).untagged()); + visitor->visit_object_code_block(tagged(obj).untagged()); } }; @@ -180,7 +167,7 @@ void factor_vm::collect_compact_code_impl() code_forwarding_map->compute_forwarding(); /* Update root pointers */ - code_block_visitor code_forwarder(this,code_block_forwarder(code_forwarding_map)); + code_block_visitor > code_forwarder(this,forwarder(code_forwarding_map)); /* Slide everything in the code heap up, and update code heap pointers inside code blocks. */ diff --git a/vm/data_heap.cpp b/vm/data_heap.cpp index 2ddc520668..21008af5cb 100755 --- a/vm/data_heap.cpp +++ b/vm/data_heap.cpp @@ -11,8 +11,7 @@ void factor_vm::init_card_decks() data_heap::data_heap(cell young_size_, cell aging_size_, - cell tenured_size_, - cell promotion_threshold_) + cell tenured_size_) { young_size_ = align(young_size_,deck_size); aging_size_ = align(aging_size_,deck_size); @@ -21,7 +20,6 @@ data_heap::data_heap(cell young_size_, young_size = young_size_; aging_size = aging_size_; tenured_size = tenured_size_; - promotion_threshold = promotion_threshold_; cell total_size = young_size + 2 * aging_size + tenured_size + deck_size; seg = new segment(total_size,false); @@ -64,8 +62,7 @@ data_heap *data_heap::grow(cell requested_bytes) cell new_tenured_size = (tenured_size * 2) + requested_bytes; return new data_heap(young_size, aging_size, - new_tenured_size, - promotion_threshold * 1.25); + new_tenured_size); } template void data_heap::clear_cards(Generation *gen) @@ -108,9 +105,9 @@ void factor_vm::set_data_heap(data_heap *data_) init_card_decks(); } -void factor_vm::init_data_heap(cell young_size, cell aging_size, cell tenured_size, cell promotion_threshold) +void factor_vm::init_data_heap(cell young_size, cell aging_size, cell tenured_size) { - set_data_heap(new data_heap(young_size,aging_size,tenured_size,promotion_threshold)); + set_data_heap(new data_heap(young_size,aging_size,tenured_size)); } /* Size of the object pointed to by a tagged pointer */ diff --git a/vm/data_heap.hpp b/vm/data_heap.hpp index 0cb9b9254b..a434024873 100755 --- a/vm/data_heap.hpp +++ b/vm/data_heap.hpp @@ -8,8 +8,6 @@ struct data_heap { cell aging_size; cell tenured_size; - cell promotion_threshold; - segment *seg; nursery_space *nursery; @@ -23,7 +21,7 @@ struct data_heap { card_deck *decks; card_deck *decks_end; - explicit data_heap(cell young_size, cell aging_size, cell tenured_size, cell promotion_threshold); + explicit data_heap(cell young_size, cell aging_size, cell tenured_size); ~data_heap(); data_heap *grow(cell requested_size); template void clear_cards(Generation *gen); diff --git a/vm/factor.cpp b/vm/factor.cpp index 50a3070922..d382745da8 100755 --- a/vm/factor.cpp +++ b/vm/factor.cpp @@ -22,7 +22,6 @@ void factor_vm::default_parameters(vm_parameters *p) p->young_size = sizeof(cell) / 4; p->aging_size = sizeof(cell) / 2; p->tenured_size = 24 * sizeof(cell); - p->promotion_threshold = 8 * sizeof(cell); p->max_pic_size = 3; @@ -70,7 +69,6 @@ void factor_vm::init_parameters_from_args(vm_parameters *p, int argc, vm_char ** else if(factor_arg(arg,STRING_LITERAL("-young=%d"),&p->young_size)); else if(factor_arg(arg,STRING_LITERAL("-aging=%d"),&p->aging_size)); else if(factor_arg(arg,STRING_LITERAL("-tenured=%d"),&p->tenured_size)); - else if(factor_arg(arg,STRING_LITERAL("-promote=%d"),&p->promotion_threshold)); else if(factor_arg(arg,STRING_LITERAL("-codeheap=%d"),&p->code_size)); else if(factor_arg(arg,STRING_LITERAL("-pic=%d"),&p->max_pic_size)); else if(factor_arg(arg,STRING_LITERAL("-callbacks=%d"),&p->callback_size)); @@ -104,7 +102,6 @@ void factor_vm::init_factor(vm_parameters *p) p->young_size <<= 20; p->aging_size <<= 20; p->tenured_size <<= 20; - p->promotion_threshold <<= 20; p->code_size <<= 20; /* Disable GC during init as a sanity check */ diff --git a/vm/free_list_allocator.hpp b/vm/free_list_allocator.hpp index 6515078100..a4801daa72 100644 --- a/vm/free_list_allocator.hpp +++ b/vm/free_list_allocator.hpp @@ -5,7 +5,6 @@ template struct free_list_allocator { cell size; cell start; cell end; - cell high_water_mark; free_list free_blocks; mark_bits state; @@ -43,7 +42,6 @@ free_list_allocator::free_list_allocator(cell size_, cell start_) : template void free_list_allocator::initial_free_list(cell occupied) { free_blocks.initial_free_list(start,end,occupied); - high_water_mark = free_blocks.free_space; } template bool free_list_allocator::contains_p(Block *block) @@ -131,52 +129,27 @@ void free_list_allocator::sweep() { free_blocks.clear_free_list(); - Block *prev = NULL; - Block *scan = this->first_block(); + Block *start = this->first_block(); Block *end = this->last_block(); - while(scan != end) + while(start != end) { - cell size = scan->size(); + /* find next unmarked block */ + start = state.next_unmarked_block_after(start); + + if(start != end) + { + /* find size */ + cell size = state.unmarked_block_size(start); + assert(size > 0); - if(scan->free_p()) - { - if(prev && prev->free_p()) - { - free_heap_block *free_prev = (free_heap_block *)prev; - free_prev->make_free(free_prev->size() + size); - } - else - prev = scan; - } - else if(this->state.marked_p(scan)) - { - if(prev && prev->free_p()) - free_blocks.add_to_free_list((free_heap_block *)prev); - prev = scan; - } - else - { - if(prev && prev->free_p()) - { - free_heap_block *free_prev = (free_heap_block *)prev; - free_prev->make_free(free_prev->size() + size); - } - else - { - free_heap_block *free_block = (free_heap_block *)scan; - free_block->make_free(size); - prev = scan; - } - } + free_heap_block *free_block = (free_heap_block *)start; + free_block->make_free(size); + free_blocks.add_to_free_list(free_block); - scan = (Block *)((cell)scan + size); + start = (Block *)((char *)start + size); + } } - - if(prev && prev->free_p()) - free_blocks.add_to_free_list((free_heap_block *)prev); - - high_water_mark = free_blocks.free_space; } template @@ -230,8 +203,6 @@ void free_list_allocator::sweep(Iterator &iter) if(prev && prev->free_p()) free_blocks.add_to_free_list((free_heap_block *)prev); - - high_water_mark = free_blocks.free_space; } template struct heap_compactor { diff --git a/vm/image.cpp b/vm/image.cpp index 152a39bdf3..fce730df5a 100755 --- a/vm/image.cpp +++ b/vm/image.cpp @@ -20,8 +20,7 @@ void factor_vm::load_data_heap(FILE *file, image_header *h, vm_parameters *p) init_data_heap(p->young_size, p->aging_size, - p->tenured_size, - p->promotion_threshold); + p->tenured_size); fixnum bytes_read = fread((void*)data->tenured->start,1,h->data_size,file); diff --git a/vm/image.hpp b/vm/image.hpp index 753d11987d..cca0e9e378 100755 --- a/vm/image.hpp +++ b/vm/image.hpp @@ -33,7 +33,6 @@ struct vm_parameters { const vm_char *executable_path; cell ds_size, rs_size; cell young_size, aging_size, tenured_size; - cell promotion_threshold; cell code_size; bool fep; bool console; diff --git a/vm/mark_bits.hpp b/vm/mark_bits.hpp index 1b8fa913a6..97d4bd4fa3 100644 --- a/vm/mark_bits.hpp +++ b/vm/mark_bits.hpp @@ -60,8 +60,8 @@ template struct mark_bits { bool bitmap_elt(u64 *bits, Block *address) { - std::pair pair = bitmap_deref(address); - return (bits[pair.first] & ((u64)1 << pair.second)) != 0; + std::pair position = bitmap_deref(address); + return (bits[position.first] & ((u64)1 << position.second)) != 0; } Block *next_block_after(Block *block) @@ -142,12 +142,12 @@ template struct mark_bits { #ifdef FACTOR_DEBUG assert(marked_p(original)); #endif - std::pair pair = bitmap_deref(original); + std::pair position = bitmap_deref(original); - cell approx_popcount = forwarding[pair.first]; - u64 mask = ((u64)1 << pair.second) - 1; + cell approx_popcount = forwarding[position.first]; + u64 mask = ((u64)1 << position.second) - 1; - cell new_line_number = approx_popcount + popcount(marked[pair.first] & mask); + cell new_line_number = approx_popcount + popcount(marked[position.first] & mask); Block *new_block = line_block(new_line_number); #ifdef FACTOR_DEBUG assert(new_block <= original); @@ -155,18 +155,78 @@ template struct mark_bits { return new_block; } - /* Find the next allocated block without calling size() on unmarked - objects. */ - cell unmarked_space_starting_at(Block *original) + cell rightmost_clear_bit(u64 x) { - char *start = (char *)original; - char *scan = start; + cell n = 0; + while(x & 1) + { + n++; + x >>= 1; + } + return n; + } + + Block *next_unmarked_block_after_slow(Block *original) + { + char *scan = (char *)original; + char *end = (char *)(this->start + this->size); + + while(scan != end && marked_p((Block *)scan)) + scan += block_granularity; + + return (Block *)scan; + } + + Block *next_unmarked_block_after_fast(Block *original) + { + std::pair position = bitmap_deref(original); + cell bit_index = position.second; + + for(cell index = position.first; index < bits_size; index++) + { + u64 mask = ((s64)marked[index] >> bit_index); + if(~mask) + { + /* Found an unmarked block on this page. + Stop, it's hammer time */ + cell clear_bit = rightmost_clear_bit(mask); + return line_block(index * 64 + bit_index + clear_bit); + } + else + { + /* No unmarked blocks on this page. + Keep looking */ + bit_index = 0; + } + } + + /* No unmarked blocks were found */ + return (Block *)(this->start + this->size); + } + + Block *next_unmarked_block_after(Block *original) + { + Block *first_result = next_unmarked_block_after_slow(original); + Block *second_result = next_unmarked_block_after_fast(original); + assert(first_result == second_result); + return second_result; + } + + Block *next_marked_block_after(Block *original) + { + char *scan = (char *)original; char *end = (char *)(this->start + this->size); while(scan != end && !marked_p((Block *)scan)) scan += block_granularity; - return scan - start; + return (Block *)scan; + } + + cell unmarked_block_size(Block *original) + { + Block *next_marked = next_marked_block_after(original); + return ((char *)next_marked - (char *)original); } }; diff --git a/vm/slot_visitor.hpp b/vm/slot_visitor.hpp index 48fb0c1af6..38d0081c0a 100644 --- a/vm/slot_visitor.hpp +++ b/vm/slot_visitor.hpp @@ -15,7 +15,7 @@ template struct slot_visitor { if(immediate_p(pointer)) return; object *untagged = untag(pointer); - untagged = visitor.visit_object(untagged); + untagged = visitor(untagged); *handle = RETAG(untagged,TAG(pointer)); } @@ -61,7 +61,7 @@ template struct slot_visitor { cell *handle = (cell *)(*iter); if(*handle) - *handle = (cell)visitor.visit_object(*(object **)handle); + *handle = (cell)visitor(*(object **)handle); } } diff --git a/vm/vm.hpp b/vm/vm.hpp index f04030761b..139c13bb41 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -218,7 +218,7 @@ struct factor_vm //data heap void init_card_decks(); void set_data_heap(data_heap *data_); - void init_data_heap(cell young_size, cell aging_size, cell tenured_size, cell promotion_threshold); + void init_data_heap(cell young_size, cell aging_size, cell tenured_size); void primitive_size(); data_heap_room data_room(); void primitive_data_room(); @@ -351,8 +351,8 @@ struct factor_vm void primitive_uninitialized_byte_array(); void primitive_resize_byte_array(); - template byte_array *byte_array_from_value(T *value); - template byte_array *byte_array_from_values(T *values, cell len); + template byte_array *byte_array_from_value(Type *value); + template byte_array *byte_array_from_values(Type *values, cell len); //tuples void primitive_tuple();