From 464aac14cfbedab901591ae6ee3d666c64691fba Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Mon, 19 Oct 2009 00:07:43 -0500 Subject: [PATCH 01/74] vm: working on new compaction implementation --- vm/code_heap.cpp | 6 +- vm/debug.cpp | 5 +- vm/heap.cpp | 19 ++++--- vm/heap.hpp | 14 +---- vm/layouts.hpp | 5 ++ vm/mark_bits.hpp | 139 ++++++++++++++++++++++++++++++++++++----------- vm/vm.hpp | 5 +- 7 files changed, 138 insertions(+), 55 deletions(-) diff --git a/vm/code_heap.cpp b/vm/code_heap.cpp index 288c2221f2..b058248bee 100755 --- a/vm/code_heap.cpp +++ b/vm/code_heap.cpp @@ -139,7 +139,11 @@ void factor_vm::primitive_code_room() code_block *code_heap::forward_code_block(code_block *compiled) { - return (code_block *)forwarding[compiled]; + code_block *block1 = (code_block *)state->forward_block(compiled); + code_block *block2 = (code_block *)forwarding[compiled]; + printf("%lx %lx\n",block1,block2); + assert(block1 == block2); + return block2; } struct callframe_forwarder { diff --git a/vm/debug.cpp b/vm/debug.cpp index 4b47e24221..bcd9e6d4d6 100755 --- a/vm/debug.cpp +++ b/vm/debug.cpp @@ -290,8 +290,9 @@ void factor_vm::dump_code_heap() cell reloc_size = 0, literal_size = 0; heap_block *scan = code->first_block(); + heap_block *end = code->last_block(); - while(scan) + while(scan != end) { const char *status; if(scan->type() == FREE_BLOCK_TYPE) @@ -313,7 +314,7 @@ void factor_vm::dump_code_heap() print_cell_hex(scan->size()); print_string(" "); print_string(status); print_string("\n"); - scan = code->next_block(scan); + scan = scan->next(); } print_cell(reloc_size); print_string(" bytes of relocation data\n"); diff --git a/vm/heap.cpp b/vm/heap.cpp index 71aac62704..8cbf914130 100644 --- a/vm/heap.cpp +++ b/vm/heap.cpp @@ -141,7 +141,7 @@ void heap::heap_free(heap_block *block) void heap::mark_block(heap_block *block) { - state->set_marked_p(block,true); + state->set_marked_p(block); } /* Compute total sum of sizes of free blocks, and size of largest free block */ @@ -152,8 +152,9 @@ void heap::heap_usage(cell *used, cell *total_free, cell *max_free) *max_free = 0; heap_block *scan = first_block(); + heap_block *end = last_block(); - while(scan) + while(scan != end) { cell size = scan->size(); @@ -166,7 +167,7 @@ void heap::heap_usage(cell *used, cell *total_free, cell *max_free) else *used += size; - scan = next_block(scan); + scan = scan->next(); } } @@ -174,11 +175,12 @@ void heap::heap_usage(cell *used, cell *total_free, cell *max_free) cell heap::heap_size() { heap_block *scan = first_block(); + heap_block *end = last_block(); - while(scan) + while(scan != end) { if(scan->type() == FREE_BLOCK_TYPE) break; - else scan = next_block(scan); + else scan = scan->next(); } assert(scan->type() == FREE_BLOCK_TYPE); @@ -190,14 +192,17 @@ cell heap::heap_size() void heap::compact_heap() { forwarding.clear(); + state->compute_forwarding(); heap_block *scan = first_block(); + heap_block *end = last_block(); + char *address = (char *)scan; /* Slide blocks up while building the forwarding hashtable. */ - while(scan) + while(scan != end) { - heap_block *next = next_block(scan); + heap_block *next = scan->next(); if(state->is_marked_p(scan)) { diff --git a/vm/heap.hpp b/vm/heap.hpp index bc8851bab1..8575dac2a1 100644 --- a/vm/heap.hpp +++ b/vm/heap.hpp @@ -18,15 +18,6 @@ struct heap { explicit heap(bool secure_gc_, cell size, bool executable_p); ~heap(); - - inline heap_block *next_block(heap_block *block) - { - cell next = ((cell)block + block->size()); - if(next == seg->end) - return NULL; - else - return (heap_block *)next; - } inline heap_block *first_block() { @@ -62,8 +53,9 @@ struct heap { heap_block *prev = NULL; heap_block *scan = first_block(); + heap_block *end = last_block(); - while(scan) + while(scan != end) { if(scan->type() == FREE_BLOCK_TYPE) { @@ -82,7 +74,7 @@ struct heap { else prev = free_allocated(prev,scan); - scan = next_block(scan); + scan = scan->next(); } if(prev && prev->type() == FREE_BLOCK_TYPE) diff --git a/vm/layouts.hpp b/vm/layouts.hpp index 34dbe163f9..5b94ddfaf5 100644 --- a/vm/layouts.hpp +++ b/vm/layouts.hpp @@ -212,6 +212,11 @@ struct heap_block { header = (header & 0x2f) | (size << 6); } + + inline heap_block *next() + { + return (heap_block *)((cell)this + size()); + } }; struct free_heap_block : public heap_block diff --git a/vm/mark_bits.hpp b/vm/mark_bits.hpp index 7945be1850..b13f6889bd 100644 --- a/vm/mark_bits.hpp +++ b/vm/mark_bits.hpp @@ -1,43 +1,41 @@ namespace factor { -const int forwarding_granularity = 128; +const int forwarding_granularity = 64; template struct mark_bits { cell start; cell size; cell bits_size; - unsigned int *marked; - unsigned int *freed; - cell forwarding_size; + u64 *marked; + u64 *allocated; cell *forwarding; void clear_mark_bits() { - memset(marked,0,bits_size * sizeof(unsigned int)); + memset(marked,0,bits_size * sizeof(u64)); } - void clear_free_bits() + void clear_allocated_bits() { - memset(freed,0,bits_size * sizeof(unsigned int)); + memset(allocated,0,bits_size * sizeof(u64)); } void clear_forwarding() { - memset(forwarding,0,forwarding_size * sizeof(cell)); + memset(forwarding,0,bits_size * sizeof(cell)); } explicit mark_bits(cell start_, cell size_) : start(start_), size(size_), - bits_size(size / Granularity / 32), - marked(new unsigned int[bits_size]), - freed(new unsigned int[bits_size]), - forwarding_size(size / Granularity / forwarding_granularity), - forwarding(new cell[forwarding_size]) + bits_size(size / Granularity / forwarding_granularity), + marked(new u64[bits_size]), + allocated(new u64[bits_size]), + forwarding(new cell[bits_size]) { clear_mark_bits(); - clear_free_bits(); + clear_allocated_bits(); clear_forwarding(); } @@ -45,17 +43,27 @@ template struct mark_bits { { delete[] marked; marked = NULL; - delete[] freed; - freed = NULL; + delete[] allocated; + allocated = NULL; delete[] forwarding; forwarding = NULL; } + cell block_line(Block *address) + { + return (((cell)address - start) / Granularity); + } + + Block *line_block(cell line) + { + return (Block *)(line * Granularity + start); + } + std::pair bitmap_deref(Block *address) { - cell word_number = (((cell)address - start) / Granularity); - cell word_index = (word_number >> 5); - cell word_shift = (word_number & 31); + cell line_number = block_line(address); + cell word_index = (line_number >> 6); + cell word_shift = (line_number & 63); #ifdef FACTOR_DEBUG assert(word_index < bits_size); @@ -64,19 +72,34 @@ template struct mark_bits { return std::make_pair(word_index,word_shift); } - bool bitmap_elt(unsigned int *bits, Block *address) + bool bitmap_elt(u64 *bits, Block *address) { std::pair pair = bitmap_deref(address); - return (bits[pair.first] & (1 << pair.second)) != 0; + return (bits[pair.first] & ((u64)1 << pair.second)) != 0; } - void set_bitmap_elt(unsigned int *bits, Block *address, bool flag) + void set_bitmap_range(u64 *bits, Block *address) { - std::pair pair = bitmap_deref(address); - if(flag) - bits[pair.first] |= (1 << pair.second); + std::pair start = bitmap_deref(address); + std::pair end = bitmap_deref(address->next()); + + u64 start_mask = ((u64)1 << start.second) - 1; + u64 end_mask = ((u64)1 << end.second) - 1; + + if(start.first == end.first) + bits[start.first] |= start_mask ^ end_mask; else - bits[pair.first] &= ~(1 << pair.second); + { + bits[start.first] |= ~start_mask; + + if(end.first != 0) + { + for(cell index = start.first + 1; index < end.first - 1; index++) + bits[index] = (u64)-1; + } + + bits[end.first] |= end_mask; + } } bool is_marked_p(Block *address) @@ -84,19 +107,71 @@ template struct mark_bits { return bitmap_elt(marked,address); } - void set_marked_p(Block *address, bool marked_p) + void set_marked_p(Block *address) { - set_bitmap_elt(marked,address,marked_p); + set_bitmap_range(marked,address); } - bool is_free_p(Block *address) + bool is_allocated_p(Block *address) { - return bitmap_elt(freed,address); + return bitmap_elt(allocated,address); } - void set_free_p(Block *address, bool free_p) + void set_allocated_p(Block *address) { - set_bitmap_elt(freed,address,free_p); + set_bitmap_range(allocated,address); + } + + cell popcount1(u64 x) + { + cell accum = 0; + while(x > 0) + { + accum += (x & 1); + x >>= 1; + } + return accum; + } + + /* From http://chessprogramming.wikispaces.com/Population+Count */ + cell popcount(u64 x) + { + cell old = x; + u64 k1 = 0x5555555555555555ll; + u64 k2 = 0x3333333333333333ll; + u64 k4 = 0x0f0f0f0f0f0f0f0fll; + u64 kf = 0x0101010101010101ll; + x = x - ((x >> 1) & k1); // put count of each 2 bits into those 2 bits + x = (x & k2) + ((x >> 2) & k2); // put count of each 4 bits into those 4 bits + x = (x + (x >> 4)) & k4 ; // put count of each 8 bits into those 8 bits + x = (x * kf) >> 56; // returns 8 most significant bits of x + (x<<8) + (x<<16) + (x<<24) + ... + + assert(x == popcount1(old)); + return (cell)x; + } + + /* The eventual destination of a block after compaction is just the number + of marked blocks before it. */ + void compute_forwarding() + { + cell accum = 0; + for(cell index = 0; index < bits_size; index++) + { + forwarding[index] = accum; + accum += popcount(marked[index]); + } + } + + /* We have the popcount for every 64 entries; look up and compute the rest */ + Block *forward_block(Block *original) + { + std::pair pair = bitmap_deref(original); + + cell approx_popcount = forwarding[pair.first]; + u64 mask = ((u64)1 << pair.second) - 1; + + cell new_line_number = approx_popcount + popcount(marked[pair.first] & mask); + return line_block(new_line_number); } }; diff --git a/vm/vm.hpp b/vm/vm.hpp index 202996ce26..a5cb2562d1 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -525,12 +525,13 @@ struct factor_vm template void iterate_code_heap(Iterator &iter) { heap_block *scan = code->first_block(); + heap_block *end = code->last_block(); - while(scan) + while(scan != end) { if(scan->type() != FREE_BLOCK_TYPE) iter((code_block *)scan); - scan = code->next_block(scan); + scan = scan->next(); } } From ac25b8ebbbd7d2c40882480a38bda89b6e531deb Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Mon, 19 Oct 2009 02:21:11 -0500 Subject: [PATCH 02/74] vm: new mark_bits data structure replaces hashtable when compacting code heap --- vm/code_block.cpp | 2 +- vm/code_heap.cpp | 47 ++++++++++++++++++++++++++++++++++++++----- vm/factor.cpp | 2 +- vm/full_collector.cpp | 44 ++-------------------------------------- vm/heap.cpp | 2 -- vm/heap.hpp | 7 +++---- vm/mach_signal.cpp | 2 +- vm/mark_bits.hpp | 24 ++++------------------ vm/master.hpp | 19 +---------------- vm/vm.hpp | 6 ++++-- 10 files changed, 59 insertions(+), 96 deletions(-) diff --git a/vm/code_block.cpp b/vm/code_block.cpp index 1f77148b5c..d2337d71de 100755 --- a/vm/code_block.cpp +++ b/vm/code_block.cpp @@ -379,7 +379,7 @@ struct literal_and_word_references_updater { } }; -void factor_vm::update_code_block_for_full_gc(code_block *compiled) +void factor_vm::update_code_block_words_and_literals(code_block *compiled) { if(code->needs_fixup_p(compiled)) relocate_code_block(compiled); diff --git a/vm/code_heap.cpp b/vm/code_heap.cpp index b058248bee..020c8c2ba8 100755 --- a/vm/code_heap.cpp +++ b/vm/code_heap.cpp @@ -73,6 +73,44 @@ void factor_vm::update_code_heap_words() iterate_code_heap(updater); } +/* After a full GC that did not grow the heap, we have to update references +to literals and other words. */ +struct word_and_literal_code_heap_updater { + factor_vm *parent; + + word_and_literal_code_heap_updater(factor_vm *parent_) : parent(parent_) {} + + void operator()(heap_block *block) + { + parent->update_code_block_words_and_literals((code_block *)block); + } +}; + +void factor_vm::update_code_heap_words_and_literals() +{ + word_and_literal_code_heap_updater updater(this); + code->sweep_heap(updater); +} + +/* After growing the heap, we have to perform a full relocation to update +references to card and deck arrays. */ +struct code_heap_relocator { + factor_vm *parent; + + code_heap_relocator(factor_vm *parent_) : parent(parent_) {} + + void operator()(heap_block *block) + { + parent->relocate_code_block((code_block *)block); + } +}; + +void factor_vm::relocate_code_heap() +{ + code_heap_relocator relocator(this); + code->sweep_heap(relocator); +} + void factor_vm::primitive_modify_code_heap() { gc_root alist(dpop(),this); @@ -139,11 +177,7 @@ void factor_vm::primitive_code_room() code_block *code_heap::forward_code_block(code_block *compiled) { - code_block *block1 = (code_block *)state->forward_block(compiled); - code_block *block2 = (code_block *)forwarding[compiled]; - printf("%lx %lx\n",block1,block2); - assert(block1 == block2); - return block2; + return (code_block *)state->forward_block(compiled); } struct callframe_forwarder { @@ -248,6 +282,9 @@ void factor_vm::compact_code_heap(bool trace_contexts_p) forward_context_xts(); forward_callback_xts(); } + + code_heap_relocator relocator(this); + iterate_code_heap(relocator); } struct stack_trace_stripper { diff --git a/vm/factor.cpp b/vm/factor.cpp index 5548ebd610..f2b0d4c92a 100755 --- a/vm/factor.cpp +++ b/vm/factor.cpp @@ -4,7 +4,7 @@ namespace factor { factor_vm *vm; -unordered_map thread_vms; +std::map thread_vms; void init_globals() { diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index f9db1c8653..61827fba41 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -104,32 +104,6 @@ void full_collector::cheneys_algorithm() } } -/* After growing the heap, we have to perform a full relocation to update -references to card and deck arrays. */ -struct big_code_heap_updater { - factor_vm *parent; - - big_code_heap_updater(factor_vm *parent_) : parent(parent_) {} - - void operator()(heap_block *block) - { - parent->relocate_code_block((code_block *)block); - } -}; - -/* After a full GC that did not grow the heap, we have to update references -to literals and other words. */ -struct small_code_heap_updater { - factor_vm *parent; - - small_code_heap_updater(factor_vm *parent_) : parent(parent_) {} - - void operator()(heap_block *block) - { - parent->update_code_block_for_full_gc((code_block *)block); - } -}; - void factor_vm::collect_full_impl(bool trace_contexts_p) { full_collector collector(this); @@ -161,16 +135,9 @@ void factor_vm::collect_growing_heap(cell requested_bytes, delete old; if(compact_code_heap_p) - { compact_code_heap(trace_contexts_p); - big_code_heap_updater updater(this); - iterate_code_heap(updater); - } else - { - big_code_heap_updater updater(this); - code->free_unmarked(updater); - } + relocate_code_heap(); code->clear_remembered_set(); } @@ -183,16 +150,9 @@ void factor_vm::collect_full(bool trace_contexts_p, bool compact_code_heap_p) collect_full_impl(trace_contexts_p); if(compact_code_heap_p) - { compact_code_heap(trace_contexts_p); - big_code_heap_updater updater(this); - iterate_code_heap(updater); - } else - { - small_code_heap_updater updater(this); - code->free_unmarked(updater); - } + update_code_heap_words_and_literals(); code->clear_remembered_set(); } diff --git a/vm/heap.cpp b/vm/heap.cpp index 8cbf914130..c2a44e42a4 100644 --- a/vm/heap.cpp +++ b/vm/heap.cpp @@ -191,7 +191,6 @@ cell heap::heap_size() void heap::compact_heap() { - forwarding.clear(); state->compute_forwarding(); heap_block *scan = first_block(); @@ -208,7 +207,6 @@ void heap::compact_heap() { cell size = scan->size(); memmove(address,scan,size); - forwarding[scan] = address; address += size; } diff --git a/vm/heap.hpp b/vm/heap.hpp index 8575dac2a1..ba00b9ba6c 100644 --- a/vm/heap.hpp +++ b/vm/heap.hpp @@ -14,7 +14,6 @@ struct heap { segment *seg; heap_free_list free; mark_bits *state; - unordered_map forwarding; explicit heap(bool secure_gc_, cell size, bool executable_p); ~heap(); @@ -45,9 +44,9 @@ struct heap { heap_block *free_allocated(heap_block *prev, heap_block *scan); - /* After code GC, all referenced code blocks have status set to B_MARKED, so any - which are allocated and not marked can be reclaimed. */ - template void free_unmarked(Iterator &iter) + /* After code GC, all live code blocks are marked, so any + which are not marked can be reclaimed. */ + template void sweep_heap(Iterator &iter) { clear_free_list(); diff --git a/vm/mach_signal.cpp b/vm/mach_signal.cpp index 2d76b12c38..d733e6b3bc 100644 --- a/vm/mach_signal.cpp +++ b/vm/mach_signal.cpp @@ -78,7 +78,7 @@ static void call_fault_handler( { THREADHANDLE thread_id = pthread_from_mach_thread_np(thread); assert(thread_id); - unordered_map::const_iterator vm = thread_vms.find(thread_id); + std::map::const_iterator vm = thread_vms.find(thread_id); if (vm != thread_vms.end()) vm->second->call_fault_handler(exception,code,exc_state,thread_state,float_state); } diff --git a/vm/mark_bits.hpp b/vm/mark_bits.hpp index b13f6889bd..f123701816 100644 --- a/vm/mark_bits.hpp +++ b/vm/mark_bits.hpp @@ -92,11 +92,8 @@ template struct mark_bits { { bits[start.first] |= ~start_mask; - if(end.first != 0) - { - for(cell index = start.first + 1; index < end.first - 1; index++) - bits[index] = (u64)-1; - } + for(cell index = start.first + 1; index < end.first; index++) + bits[index] = (u64)-1; bits[end.first] |= end_mask; } @@ -122,21 +119,9 @@ template struct mark_bits { set_bitmap_range(allocated,address); } - cell popcount1(u64 x) - { - cell accum = 0; - while(x > 0) - { - accum += (x & 1); - x >>= 1; - } - return accum; - } - /* From http://chessprogramming.wikispaces.com/Population+Count */ cell popcount(u64 x) { - cell old = x; u64 k1 = 0x5555555555555555ll; u64 k2 = 0x3333333333333333ll; u64 k4 = 0x0f0f0f0f0f0f0f0fll; @@ -145,13 +130,12 @@ template struct mark_bits { x = (x & k2) + ((x >> 2) & k2); // put count of each 4 bits into those 4 bits x = (x + (x >> 4)) & k4 ; // put count of each 8 bits into those 8 bits x = (x * kf) >> 56; // returns 8 most significant bits of x + (x<<8) + (x<<16) + (x<<24) + ... - - assert(x == popcount1(old)); + return (cell)x; } /* The eventual destination of a block after compaction is just the number - of marked blocks before it. */ + of marked blocks before it. Live blocks must be marked on entry. */ void compute_forwarding() { cell accum = 0; diff --git a/vm/master.hpp b/vm/master.hpp index 847980fac6..b0e73a4b29 100755 --- a/vm/master.hpp +++ b/vm/master.hpp @@ -25,27 +25,10 @@ /* C++ headers */ #include +#include #include #include -#if __GNUC__ == 4 - #include - - namespace factor - { - using std::tr1::unordered_map; - } -#elif __GNUC__ == 3 - #include - - namespace factor - { - using boost::unordered_map; - } -#else - #error Factor requires GCC 3.x or later -#endif - /* Forward-declare this since it comes up in function prototypes */ namespace factor { diff --git a/vm/vm.hpp b/vm/vm.hpp index a5cb2562d1..7742ea8d60 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -494,7 +494,7 @@ struct factor_vm void update_literal_references(code_block *compiled); void relocate_code_block_step(relocation_entry rel, cell index, code_block *compiled); void update_word_references(code_block *compiled); - void update_code_block_for_full_gc(code_block *compiled); + void update_code_block_words_and_literals(code_block *compiled); void check_code_address(cell address); void relocate_code_block(code_block *compiled); void fixup_labels(array *labels, code_block *compiled); @@ -513,6 +513,8 @@ struct factor_vm bool in_code_heap_p(cell ptr); void jit_compile_word(cell word_, cell def_, bool relocate); void update_code_heap_words(); + void update_code_heap_words_and_literals(); + void relocate_code_heap(); void primitive_modify_code_heap(); void primitive_code_room(); void forward_object_xts(); @@ -711,6 +713,6 @@ struct factor_vm }; -extern unordered_map thread_vms; +extern std::map thread_vms; } From 4ddd63d83ef6c920cf7b31fe46142771c516b146 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Mon, 19 Oct 2009 03:05:20 -0500 Subject: [PATCH 03/74] vm: move compaction algorithm to mark_bits.hpp since it doesn't rely on properties of heaps per se --- vm/code_heap.cpp | 23 ++++++++----- vm/heap.cpp | 29 ---------------- vm/heap.hpp | 85 +++++++++++++++++++++++++++++++---------------- vm/image.cpp | 2 +- vm/mark_bits.hpp | 19 +++++++++++ vm/quotations.cpp | 1 - vm/vm.hpp | 20 ++++++----- 7 files changed, 102 insertions(+), 77 deletions(-) diff --git a/vm/code_heap.cpp b/vm/code_heap.cpp index 020c8c2ba8..756dfdbff6 100755 --- a/vm/code_heap.cpp +++ b/vm/code_heap.cpp @@ -59,7 +59,7 @@ struct word_updater { factor_vm *parent; explicit word_updater(factor_vm *parent_) : parent(parent_) {} - void operator()(code_block *compiled) + void operator()(code_block *compiled, cell size) { parent->update_word_references(compiled); } @@ -80,7 +80,7 @@ struct word_and_literal_code_heap_updater { word_and_literal_code_heap_updater(factor_vm *parent_) : parent(parent_) {} - void operator()(heap_block *block) + void operator()(heap_block *block, cell size) { parent->update_code_block_words_and_literals((code_block *)block); } @@ -99,16 +99,17 @@ struct code_heap_relocator { code_heap_relocator(factor_vm *parent_) : parent(parent_) {} - void operator()(heap_block *block) + void operator()(code_block *block, cell size) { - parent->relocate_code_block((code_block *)block); + parent->relocate_code_block(block); } }; void factor_vm::relocate_code_heap() { code_heap_relocator relocator(this); - code->sweep_heap(relocator); + code_heap_iterator iter(relocator); + code->sweep_heap(iter); } void factor_vm::primitive_modify_code_heap() @@ -275,7 +276,10 @@ on entry to this function. XTs in code blocks must be updated after this function returns. */ void factor_vm::compact_code_heap(bool trace_contexts_p) { - code->compact_heap(); + /* Figure out where blocks are going to go */ + code->state->compute_forwarding(); + + /* Update references to the code heap from the data heap */ forward_object_xts(); if(trace_contexts_p) { @@ -283,14 +287,17 @@ void factor_vm::compact_code_heap(bool trace_contexts_p) forward_callback_xts(); } + /* Move code blocks and update references amongst them (this requires + that the data heap is up to date since relocation looks up object XTs) */ code_heap_relocator relocator(this); - iterate_code_heap(relocator); + code_heap_iterator iter(relocator); + code->compact_heap(iter); } struct stack_trace_stripper { explicit stack_trace_stripper() {} - void operator()(code_block *compiled) + void operator()(code_block *compiled, cell size) { compiled->owner = false_object; } diff --git a/vm/heap.cpp b/vm/heap.cpp index c2a44e42a4..2132ba1a20 100644 --- a/vm/heap.cpp +++ b/vm/heap.cpp @@ -189,35 +189,6 @@ cell heap::heap_size() return (cell)scan - (cell)first_block(); } -void heap::compact_heap() -{ - state->compute_forwarding(); - - heap_block *scan = first_block(); - heap_block *end = last_block(); - - char *address = (char *)scan; - - /* Slide blocks up while building the forwarding hashtable. */ - while(scan != end) - { - heap_block *next = scan->next(); - - if(state->is_marked_p(scan)) - { - cell size = scan->size(); - memmove(address,scan,size); - address += size; - } - - scan = next; - } - - /* Now update the free list; there will be a single free block at - the end */ - build_free_list((cell)address - seg->start); -} - heap_block *heap::free_allocated(heap_block *prev, heap_block *scan) { if(secure_gc) diff --git a/vm/heap.hpp b/vm/heap.hpp index ba00b9ba6c..1cdca5180c 100644 --- a/vm/heap.hpp +++ b/vm/heap.hpp @@ -29,7 +29,6 @@ struct heap { } void clear_free_list(); - void new_heap(cell size); void add_to_free_list(free_heap_block *block); void build_free_list(cell size); void assert_free_block(free_heap_block *block); @@ -44,41 +43,69 @@ struct heap { heap_block *free_allocated(heap_block *prev, heap_block *scan); - /* After code GC, all live code blocks are marked, so any - which are not marked can be reclaimed. */ - template void sweep_heap(Iterator &iter) + template void sweep_heap(Iterator &iter); + template void compact_heap(Iterator &iter); + + template void iterate_heap(Iterator &iter) { - clear_free_list(); - - heap_block *prev = NULL; heap_block *scan = first_block(); heap_block *end = last_block(); - + while(scan != end) { - if(scan->type() == FREE_BLOCK_TYPE) - { - if(prev && prev->type() == FREE_BLOCK_TYPE) - prev->set_size(prev->size() + scan->size()); - else - prev = scan; - } - else if(state->is_marked_p(scan)) - { - if(prev && prev->type() == FREE_BLOCK_TYPE) - add_to_free_list((free_heap_block *)prev); - prev = scan; - iter(scan); - } - else - prev = free_allocated(prev,scan); - - scan = scan->next(); + heap_block *next = scan->next(); + if(scan->type() != FREE_BLOCK_TYPE) iter(scan,scan->size()); + scan = next; } - - if(prev && prev->type() == FREE_BLOCK_TYPE) - add_to_free_list((free_heap_block *)prev); } }; +/* After code GC, all live code blocks are marked, so any +which are not marked can be reclaimed. */ +template void heap::sweep_heap(Iterator &iter) +{ + this->clear_free_list(); + + heap_block *prev = NULL; + heap_block *scan = this->first_block(); + heap_block *end = this->last_block(); + + while(scan != end) + { + if(scan->type() == FREE_BLOCK_TYPE) + { + if(prev && prev->type() == FREE_BLOCK_TYPE) + prev->set_size(prev->size() + scan->size()); + else + prev = scan; + } + else if(this->state->is_marked_p(scan)) + { + if(prev && prev->type() == FREE_BLOCK_TYPE) + this->add_to_free_list((free_heap_block *)prev); + prev = scan; + iter(scan,scan->size()); + } + else + prev = this->free_allocated(prev,scan); + + scan = scan->next(); + } + + if(prev && prev->type() == FREE_BLOCK_TYPE) + this->add_to_free_list((free_heap_block *)prev); +} + +/* The forwarding map must be computed first by calling +state->compute_forwarding(). */ +template void heap::compact_heap(Iterator &iter) +{ + heap_compacter compacter(state,first_block(),iter); + this->iterate_heap(compacter); + + /* Now update the free list; there will be a single free block at + the end */ + this->build_free_list((cell)compacter.address - this->seg->start); +} + } diff --git a/vm/image.cpp b/vm/image.cpp index c6d1ad7aca..c96da6b703 100755 --- a/vm/image.cpp +++ b/vm/image.cpp @@ -225,7 +225,7 @@ struct code_block_fixupper { code_block_fixupper(factor_vm *parent_, cell data_relocation_base_) : parent(parent_), data_relocation_base(data_relocation_base_) { } - void operator()(code_block *compiled) + void operator()(code_block *compiled, cell size) { parent->fixup_code_block(compiled,data_relocation_base); } diff --git a/vm/mark_bits.hpp b/vm/mark_bits.hpp index f123701816..ad3eda89df 100644 --- a/vm/mark_bits.hpp +++ b/vm/mark_bits.hpp @@ -159,4 +159,23 @@ template struct mark_bits { } }; +template struct heap_compacter { + mark_bits *state; + char *address; + Iterator &iter; + + explicit heap_compacter(mark_bits *state_, Block *address_, Iterator &iter_) : + state(state_), address((char *)address_), iter(iter_) {} + + void operator()(Block *block, cell size) + { + if(this->state->is_marked_p(block)) + { + memmove(address,block,size); + iter((Block *)address,size); + address += size; + } + } +}; + } diff --git a/vm/quotations.cpp b/vm/quotations.cpp index 9c2c85215d..d75d1c680c 100755 --- a/vm/quotations.cpp +++ b/vm/quotations.cpp @@ -283,7 +283,6 @@ void quotation_jit::iterate_quotation() void factor_vm::set_quot_xt(quotation *quot, code_block *code) { - assert(code->type() == QUOTATION_TYPE); quot->code = code; quot->xt = code->xt(); } diff --git a/vm/vm.hpp b/vm/vm.hpp index 7742ea8d60..05a918c5e9 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -524,17 +524,19 @@ struct factor_vm void primitive_strip_stack_traces(); /* Apply a function to every code block */ - template void iterate_code_heap(Iterator &iter) - { - heap_block *scan = code->first_block(); - heap_block *end = code->last_block(); - - while(scan != end) + template struct code_heap_iterator { + Iterator &iter; + explicit code_heap_iterator(Iterator &iter_) : iter(iter_) {} + void operator()(heap_block *block, cell size) { - if(scan->type() != FREE_BLOCK_TYPE) - iter((code_block *)scan); - scan = scan->next(); + iter((code_block *)block,size); } + }; + + template void iterate_code_heap(Iterator &iter_) + { + code_heap_iterator iter(iter_); + code->iterate_heap(iter); } //callbacks From 838a44e90165534be4907547ec2a58bf7b403d27 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Tue, 20 Oct 2009 09:37:24 -0500 Subject: [PATCH 04/74] vm: change code heap layout somewhat, remove unused allocation bitmap from mark_bits --- vm/code_block.cpp | 14 ++++++----- vm/code_heap.cpp | 2 +- vm/debug.cpp | 2 +- vm/heap.cpp | 51 +++++++++++++++------------------------ vm/heap.hpp | 30 ++++++++++++++++------- vm/inline_cache.cpp | 12 +++------- vm/jit.cpp | 2 +- vm/jit.hpp | 4 ++-- vm/layouts.hpp | 58 ++++++++++++++++++++++++++++++++++++++------- vm/mark_bits.hpp | 20 ---------------- vm/profiler.cpp | 2 +- vm/quotations.cpp | 2 +- vm/quotations.hpp | 2 +- vm/tagged.hpp | 19 +++++++++++---- vm/vm.hpp | 4 ++-- vm/words.cpp | 3 ++- vm/words.hpp | 5 ---- 17 files changed, 128 insertions(+), 104 deletions(-) diff --git a/vm/code_block.cpp b/vm/code_block.cpp index d2337d71de..7e6892202a 100755 --- a/vm/code_block.cpp +++ b/vm/code_block.cpp @@ -346,7 +346,7 @@ void factor_vm::update_word_references(code_block *compiled) are referenced after this is done. So instead of polluting the code heap with dead PICs that will be freed on the next GC, we add them to the free list immediately. */ - else if(compiled->type() == PIC_TYPE) + else if(compiled->pic_p()) code->code_heap_free(compiled); else { @@ -437,9 +437,9 @@ void factor_vm::fixup_labels(array *labels, code_block *compiled) } /* Might GC */ -code_block *factor_vm::allot_code_block(cell size, cell type) +code_block *factor_vm::allot_code_block(cell size, code_block_type type) { - heap_block *block = code->heap_allot(size + sizeof(code_block),type); + heap_block *block = code->heap_allot(size + sizeof(code_block)); /* If allocation failed, do a full GC and compact the code heap. A full GC that occurs as a result of the data heap filling up does not @@ -449,7 +449,7 @@ code_block *factor_vm::allot_code_block(cell size, cell type) if(block == NULL) { primitive_compact_gc(); - block = code->heap_allot(size + sizeof(code_block),type); + block = code->heap_allot(size + sizeof(code_block)); /* Insufficient room even after code GC, give up */ if(block == NULL) @@ -465,11 +465,13 @@ code_block *factor_vm::allot_code_block(cell size, cell type) } } - return (code_block *)block; + code_block *compiled = (code_block *)block; + compiled->set_type(type); + return compiled; } /* Might GC */ -code_block *factor_vm::add_code_block(cell type, cell code_, cell labels_, cell owner_, cell relocation_, cell literals_) +code_block *factor_vm::add_code_block(code_block_type type, cell code_, cell labels_, cell owner_, cell relocation_, cell literals_) { gc_root code(code_,this); gc_root labels(labels_,this); diff --git a/vm/code_heap.cpp b/vm/code_heap.cpp index 756dfdbff6..19c9c87395 100755 --- a/vm/code_heap.cpp +++ b/vm/code_heap.cpp @@ -144,7 +144,7 @@ void factor_vm::primitive_modify_code_heap() cell code = array_nth(compiled_data,4); code_block *compiled = add_code_block( - WORD_TYPE, + code_block_optimized, code, labels, owner, diff --git a/vm/debug.cpp b/vm/debug.cpp index bcd9e6d4d6..a777c5f970 100755 --- a/vm/debug.cpp +++ b/vm/debug.cpp @@ -295,7 +295,7 @@ void factor_vm::dump_code_heap() while(scan != end) { const char *status; - if(scan->type() == FREE_BLOCK_TYPE) + if(scan->free_p()) status = "free"; else if(code->state->is_marked_p(scan)) { diff --git a/vm/heap.cpp b/vm/heap.cpp index 2132ba1a20..e13001ff4f 100644 --- a/vm/heap.cpp +++ b/vm/heap.cpp @@ -49,15 +49,16 @@ void heap::build_free_list(cell size) { clear_free_list(); free_heap_block *end = (free_heap_block *)(seg->start + size); - end->set_type(FREE_BLOCK_TYPE); + end->set_free(); end->set_size(seg->end - (cell)end); add_to_free_list(end); } void heap::assert_free_block(free_heap_block *block) { - if(block->type() != FREE_BLOCK_TYPE) - critical_error("Invalid block in free list",(cell)block); +#ifdef FACTOR_DEBUG + assert(block->free_p()); +#endif } free_heap_block *heap::find_free_block(cell size) @@ -102,11 +103,11 @@ free_heap_block *heap::find_free_block(cell size) free_heap_block *heap::split_free_block(free_heap_block *block, cell size) { - if(block->size() != size ) + if(block->size() != size) { /* split the block in two */ free_heap_block *split = (free_heap_block *)((cell)block + size); - split->set_type(FREE_BLOCK_TYPE); + split->set_free(); split->set_size(block->size() - size); split->next_free = block->next_free; block->set_size(size); @@ -116,27 +117,25 @@ free_heap_block *heap::split_free_block(free_heap_block *block, cell size) return block; } -/* Allocate a block of memory from the mark and sweep GC heap */ -heap_block *heap::heap_allot(cell size, cell type) +heap_block *heap::heap_allot(cell size) { - size = (size + block_size_increment - 1) & ~(block_size_increment - 1); + size = align(size,block_size_increment); free_heap_block *block = find_free_block(size); if(block) { block = split_free_block(block,size); - block->set_type(type); return block; } else return NULL; } -/* Deallocates a block manually */ void heap::heap_free(heap_block *block) { - block->set_type(FREE_BLOCK_TYPE); - add_to_free_list((free_heap_block *)block); + free_heap_block *free_block = (free_heap_block *)block; + free_block->set_free(); + add_to_free_list(free_block); } void heap::mark_block(heap_block *block) @@ -158,7 +157,7 @@ void heap::heap_usage(cell *used, cell *total_free, cell *max_free) { cell size = scan->size(); - if(scan->type() == FREE_BLOCK_TYPE) + if(scan->free_p()) { *total_free += size; if(size > *max_free) @@ -179,31 +178,19 @@ cell heap::heap_size() while(scan != end) { - if(scan->type() == FREE_BLOCK_TYPE) break; + if(scan->free_p()) break; else scan = scan->next(); } - assert(scan->type() == FREE_BLOCK_TYPE); - assert((cell)scan + scan->size() == seg->end); - - return (cell)scan - (cell)first_block(); -} - -heap_block *heap::free_allocated(heap_block *prev, heap_block *scan) -{ - if(secure_gc) - memset(scan + 1,0,scan->size() - sizeof(heap_block)); - - if(prev && prev->type() == FREE_BLOCK_TYPE) + if(scan != end) { - prev->set_size(prev->size() + scan->size()); - return prev; + assert(scan->free_p()); + assert((cell)scan + scan->size() == seg->end); + + return (cell)scan - (cell)first_block(); } else - { - scan->set_type(FREE_BLOCK_TYPE); - return scan; - } + return seg->size; } } diff --git a/vm/heap.hpp b/vm/heap.hpp index 70a4324798..7c3dca1eaf 100644 --- a/vm/heap.hpp +++ b/vm/heap.hpp @@ -34,15 +34,13 @@ struct heap { void assert_free_block(free_heap_block *block); free_heap_block *find_free_block(cell size); free_heap_block *split_free_block(free_heap_block *block, cell size); - heap_block *heap_allot(cell size, cell type); + heap_block *heap_allot(cell size); void heap_free(heap_block *block); void mark_block(heap_block *block); void heap_usage(cell *used, cell *total_free, cell *max_free); cell heap_size(); void compact_heap(); - heap_block *free_allocated(heap_block *prev, heap_block *scan); - template void sweep_heap(Iterator &iter); template void compact_heap(Iterator &iter); @@ -54,7 +52,7 @@ struct heap { while(scan != end) { heap_block *next = scan->next(); - if(scan->type() != FREE_BLOCK_TYPE) iter(scan,scan->size()); + if(!scan->free_p()) iter(scan,scan->size()); scan = next; } } @@ -72,27 +70,41 @@ template void heap::sweep_heap(Iterator &iter) while(scan != end) { - if(scan->type() == FREE_BLOCK_TYPE) + if(scan->free_p()) { - if(prev && prev->type() == FREE_BLOCK_TYPE) + if(prev && prev->free_p()) prev->set_size(prev->size() + scan->size()); else prev = scan; } else if(this->state->is_marked_p(scan)) { - if(prev && prev->type() == FREE_BLOCK_TYPE) + if(prev && prev->free_p()) this->add_to_free_list((free_heap_block *)prev); prev = scan; iter(scan,scan->size()); } else - prev = this->free_allocated(prev,scan); + { + if(secure_gc) + memset(scan + 1,0,scan->size() - sizeof(heap_block)); + + if(prev && prev->free_p()) + { + free_heap_block *free_prev = (free_heap_block *)prev; + free_prev->set_size(free_prev->size() + scan->size()); + } + else + { + scan->set_free(); + prev = scan; + } + } scan = scan->next(); } - if(prev && prev->type() == FREE_BLOCK_TYPE) + if(prev && prev->free_p()) this->add_to_free_list((free_heap_block *)prev); } diff --git a/vm/inline_cache.cpp b/vm/inline_cache.cpp index f6e756f758..772631d1ce 100755 --- a/vm/inline_cache.cpp +++ b/vm/inline_cache.cpp @@ -19,15 +19,9 @@ void factor_vm::deallocate_inline_cache(cell return_address) check_code_pointer((cell)old_xt); code_block *old_block = (code_block *)old_xt - 1; - cell old_type = old_block->type(); -#ifdef FACTOR_DEBUG - /* The call target was either another PIC, - or a compiled quotation (megamorphic stub) */ - assert(old_type == PIC_TYPE || old_type == QUOTATION_TYPE); -#endif - - if(old_type == PIC_TYPE) + /* Free the old PIC since we know its unreachable */ + if(old_block->pic_p()) code->code_heap_free(old_block); } @@ -78,7 +72,7 @@ void factor_vm::update_pic_count(cell type) struct inline_cache_jit : public jit { fixnum index; - explicit inline_cache_jit(cell generic_word_,factor_vm *vm) : jit(PIC_TYPE,generic_word_,vm) {}; + explicit inline_cache_jit(cell generic_word_,factor_vm *vm) : jit(code_block_pic,generic_word_,vm) {}; void emit_check(cell klass); void compile_inline_cache(fixnum index, diff --git a/vm/jit.cpp b/vm/jit.cpp index ced487e659..98212d2efe 100644 --- a/vm/jit.cpp +++ b/vm/jit.cpp @@ -10,7 +10,7 @@ namespace factor - polymorphic inline caches (inline_cache.cpp) */ /* Allocates memory */ -jit::jit(cell type_, cell owner_, factor_vm *vm) +jit::jit(code_block_type type_, cell owner_, factor_vm *vm) : type(type_), owner(owner_,vm), code(vm), diff --git a/vm/jit.hpp b/vm/jit.hpp index d69f44d05d..4928962fc6 100644 --- a/vm/jit.hpp +++ b/vm/jit.hpp @@ -2,7 +2,7 @@ namespace factor { struct jit { - cell type; + code_block_type type; gc_root owner; growable_byte_array code; growable_byte_array relocation; @@ -12,7 +12,7 @@ struct jit { cell offset; factor_vm *parent; - explicit jit(cell jit_type, cell owner, factor_vm *vm); + explicit jit(code_block_type type, cell owner, factor_vm *parent); void compute_position(cell offset); void emit_relocation(cell code_template); diff --git a/vm/layouts.hpp b/vm/layouts.hpp index 5b94ddfaf5..3249aac946 100644 --- a/vm/layouts.hpp +++ b/vm/layouts.hpp @@ -62,8 +62,14 @@ inline static cell align8(cell a) #define TYPE_COUNT 15 /* Not real types, but code_block's type can be set to this */ -#define PIC_TYPE 16 -#define FREE_BLOCK_TYPE 17 + +enum code_block_type +{ + code_block_unoptimized, + code_block_optimized, + code_block_profiling, + code_block_pic +}; /* Constants used when floating-point trap exceptions are thrown */ enum @@ -201,16 +207,29 @@ struct heap_block { cell header; - cell type() { return (header >> 1) & 0x1f; } - void set_type(cell type) + bool free_p() { - header = ((header & ~(0x1f << 1)) | (type << 1)); + return header & 1 == 1; + } + + void set_free() + { + header |= 1; + } + + void clear_free() + { + header &= ~1; + } + + cell size() + { + return header >> 3; } - cell size() { return (header >> 6); } void set_size(cell size) { - header = (header & 0x2f) | (size << 6); + header = (header & 0x7) | (size << 3); } inline heap_block *next() @@ -230,7 +249,30 @@ struct code_block : public heap_block cell literals; /* tagged pointer to array or f */ cell relocation; /* tagged pointer to byte-array or f */ - void *xt() { return (void *)(this + 1); } + void *xt() + { + return (void *)(this + 1); + } + + cell type() + { + return (header >> 1) & 0x3; + } + + void set_type(code_block_type type) + { + header = ((header & ~0x7) | (type << 1)); + } + + bool pic_p() + { + return type() == code_block_pic; + } + + bool optimized_p() + { + return type() == code_block_optimized; + } }; /* Assembly code makes assumptions about the layout of this struct */ diff --git a/vm/mark_bits.hpp b/vm/mark_bits.hpp index ad3eda89df..a4dc715c50 100644 --- a/vm/mark_bits.hpp +++ b/vm/mark_bits.hpp @@ -8,7 +8,6 @@ template struct mark_bits { cell size; cell bits_size; u64 *marked; - u64 *allocated; cell *forwarding; void clear_mark_bits() @@ -16,11 +15,6 @@ template struct mark_bits { memset(marked,0,bits_size * sizeof(u64)); } - void clear_allocated_bits() - { - memset(allocated,0,bits_size * sizeof(u64)); - } - void clear_forwarding() { memset(forwarding,0,bits_size * sizeof(cell)); @@ -31,11 +25,9 @@ template struct mark_bits { size(size_), bits_size(size / Granularity / forwarding_granularity), marked(new u64[bits_size]), - allocated(new u64[bits_size]), forwarding(new cell[bits_size]) { clear_mark_bits(); - clear_allocated_bits(); clear_forwarding(); } @@ -43,8 +35,6 @@ template struct mark_bits { { delete[] marked; marked = NULL; - delete[] allocated; - allocated = NULL; delete[] forwarding; forwarding = NULL; } @@ -109,16 +99,6 @@ template struct mark_bits { set_bitmap_range(marked,address); } - bool is_allocated_p(Block *address) - { - return bitmap_elt(allocated,address); - } - - void set_allocated_p(Block *address) - { - set_bitmap_range(allocated,address); - } - /* From http://chessprogramming.wikispaces.com/Population+Count */ cell popcount(u64 x) { diff --git a/vm/profiler.cpp b/vm/profiler.cpp index 4674b726b1..df9d9ee67b 100755 --- a/vm/profiler.cpp +++ b/vm/profiler.cpp @@ -13,7 +13,7 @@ code_block *factor_vm::compile_profiling_stub(cell word_) { gc_root word(word_,this); - jit jit(WORD_TYPE,word.value(),this); + jit jit(code_block_profiling,word.value(),this); jit.emit_with(userenv[JIT_PROFILING],word.value()); return jit.to_code_block(); diff --git a/vm/quotations.cpp b/vm/quotations.cpp index d75d1c680c..e06b5c23d5 100755 --- a/vm/quotations.cpp +++ b/vm/quotations.cpp @@ -335,7 +335,7 @@ void factor_vm::compile_all_words() { gc_root word(array_nth(words.untagged(),i),this); - if(!word->code || !word_optimized_p(word.untagged())) + if(!word->code || !word->code->optimized_p()) jit_compile_word(word.value(),word->def,false); update_word_xt(word.value()); diff --git a/vm/quotations.hpp b/vm/quotations.hpp index feb2af1ce4..e6e6afcd0b 100755 --- a/vm/quotations.hpp +++ b/vm/quotations.hpp @@ -6,7 +6,7 @@ struct quotation_jit : public jit { bool compiling, relocate; explicit quotation_jit(cell quot, bool compiling_, bool relocate_, factor_vm *vm) - : jit(QUOTATION_TYPE,quot,vm), + : jit(code_block_unoptimized,quot,vm), elements(owner.as().untagged()->array,vm), compiling(compiling_), relocate(relocate_){}; diff --git a/vm/tagged.hpp b/vm/tagged.hpp index a61c599aeb..02fcdee26c 100755 --- a/vm/tagged.hpp +++ b/vm/tagged.hpp @@ -27,23 +27,34 @@ struct tagged return tag; } - bool type_p(cell type_) const { return type() == type_; } + bool type_p(cell type_) const + { + return type() == type_; + } + + bool type_p() const + { + if(Type::type_number == TYPE_COUNT) + return true; + else + return type_p(Type::type_number); + } Type *untag_check(factor_vm *parent) const { - if(Type::type_number != TYPE_COUNT && !type_p(Type::type_number)) + if(!type_p()) parent->type_error(Type::type_number,value_); return untagged(); } explicit tagged(cell tagged) : value_(tagged) { #ifdef FACTOR_DEBUG - untag_check(tls_vm()); + assert(type_p()); #endif } explicit tagged(Type *untagged) : value_(factor::tag(untagged)) { #ifdef FACTOR_DEBUG - untag_check(tls_vm()); + assert(type_p()); #endif } diff --git a/vm/vm.hpp b/vm/vm.hpp index 05a918c5e9..921e829bda 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -498,8 +498,8 @@ struct factor_vm void check_code_address(cell address); void relocate_code_block(code_block *compiled); void fixup_labels(array *labels, code_block *compiled); - code_block *allot_code_block(cell size, cell type); - code_block *add_code_block(cell type, cell code_, cell labels_, cell owner_, cell relocation_, cell literals_); + code_block *allot_code_block(cell size, code_block_type type); + code_block *add_code_block(code_block_type type, cell code_, cell labels_, cell owner_, cell relocation_, cell literals_); //code heap inline void check_code_pointer(cell ptr) diff --git a/vm/words.cpp b/vm/words.cpp index 6193a5c93c..9d3ccff3c3 100644 --- a/vm/words.cpp +++ b/vm/words.cpp @@ -82,7 +82,8 @@ void factor_vm::update_word_xt(cell w_) void factor_vm::primitive_optimized_p() { - drepl(tag_boolean(word_optimized_p(untag_check(dpeek())))); + word *w = untag_check(dpeek()); + drepl(tag_boolean(w->code->optimized_p())); } void factor_vm::primitive_wrapper() diff --git a/vm/words.hpp b/vm/words.hpp index 1701def6dc..412ef35bb4 100644 --- a/vm/words.hpp +++ b/vm/words.hpp @@ -1,9 +1,4 @@ namespace factor { -inline bool word_optimized_p(word *word) -{ - return word->code->type() == WORD_TYPE; -} - } From acdcb181e04e077462723c56e3058a3716317b67 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Tue, 20 Oct 2009 10:22:06 -0500 Subject: [PATCH 05/74] vm: working on making heap more generic --- Makefile | 1 - vm/code_heap.cpp | 3 +- vm/code_heap.hpp | 14 ++- vm/debug.cpp | 39 +++---- vm/heap.cpp | 196 ---------------------------------- vm/heap.hpp | 269 +++++++++++++++++++++++++++++++++++++++++------ vm/layouts.hpp | 5 - vm/mark_bits.hpp | 18 ++-- 8 files changed, 285 insertions(+), 260 deletions(-) delete mode 100644 vm/heap.cpp diff --git a/Makefile b/Makefile index 35cf7a05c4..5a44333d42 100755 --- a/Makefile +++ b/Makefile @@ -49,7 +49,6 @@ DLL_OBJS = $(PLAF_DLL_OBJS) \ vm/factor.o \ vm/full_collector.o \ vm/gc.o \ - vm/heap.o \ vm/image.o \ vm/inline_cache.o \ vm/io.o \ diff --git a/vm/code_heap.cpp b/vm/code_heap.cpp index 19c9c87395..df557074af 100755 --- a/vm/code_heap.cpp +++ b/vm/code_heap.cpp @@ -3,7 +3,8 @@ namespace factor { -code_heap::code_heap(bool secure_gc, cell size) : heap(secure_gc,size,true) {} +code_heap::code_heap(bool secure_gc, cell size) : + heap(secure_gc,size,true) {} void code_heap::write_barrier(code_block *compiled) { diff --git a/vm/code_heap.hpp b/vm/code_heap.hpp index 0a96a0b27b..e98d966afe 100755 --- a/vm/code_heap.hpp +++ b/vm/code_heap.hpp @@ -1,7 +1,19 @@ namespace factor { -struct code_heap : heap { +struct code_heap_layout { + cell block_size(heap_block *block) + { + return block->size(); + } + + heap_block *next_block_after(heap_block *block) + { + return (heap_block *)((cell)block + block_size(block)); + } +}; + +struct code_heap : heap { /* Set of blocks which need full relocation. */ std::set needs_fixup; diff --git a/vm/debug.cpp b/vm/debug.cpp index a777c5f970..ddf4877eab 100755 --- a/vm/debug.cpp +++ b/vm/debug.cpp @@ -284,41 +284,44 @@ void factor_vm::find_data_references(cell look_for) end_scan(); } -/* Dump all code blocks for debugging */ -void factor_vm::dump_code_heap() -{ - cell reloc_size = 0, literal_size = 0; +struct code_block_printer { + factor_vm *parent; + cell reloc_size, literal_size; - heap_block *scan = code->first_block(); - heap_block *end = code->last_block(); + code_block_printer(factor_vm *parent_) : + parent(parent_), reloc_size(0), literal_size(0) {} - while(scan != end) + void operator()(heap_block *scan, cell size) { const char *status; if(scan->free_p()) status = "free"; - else if(code->state->is_marked_p(scan)) + else if(parent->code->state->is_marked_p(scan)) { - reloc_size += object_size(((code_block *)scan)->relocation); - literal_size += object_size(((code_block *)scan)->literals); + reloc_size += parent->object_size(((code_block *)scan)->relocation); + literal_size += parent->object_size(((code_block *)scan)->literals); status = "marked"; } else { - reloc_size += object_size(((code_block *)scan)->relocation); - literal_size += object_size(((code_block *)scan)->literals); + reloc_size += parent->object_size(((code_block *)scan)->relocation); + literal_size += parent->object_size(((code_block *)scan)->literals); status = "allocated"; } print_cell_hex((cell)scan); print_string(" "); - print_cell_hex(scan->size()); print_string(" "); + print_cell_hex(size); print_string(" "); print_string(status); print_string("\n"); - - scan = scan->next(); } - - print_cell(reloc_size); print_string(" bytes of relocation data\n"); - print_cell(literal_size); print_string(" bytes of literal data\n"); +}; + +/* Dump all code blocks for debugging */ +void factor_vm::dump_code_heap() +{ + code_block_printer printer(this); + code->iterate_heap(printer); + print_cell(printer.reloc_size); print_string(" bytes of relocation data\n"); + print_cell(printer.literal_size); print_string(" bytes of literal data\n"); } void factor_vm::factorbug() diff --git a/vm/heap.cpp b/vm/heap.cpp deleted file mode 100644 index e13001ff4f..0000000000 --- a/vm/heap.cpp +++ /dev/null @@ -1,196 +0,0 @@ -#include "master.hpp" - -/* This malloc-style heap code is reasonably generic. Maybe in the future, it -will be used for the data heap too, if we ever get mark/sweep/compact GC. */ - -namespace factor -{ - -void heap::clear_free_list() -{ - memset(&free,0,sizeof(heap_free_list)); -} - -heap::heap(bool secure_gc_, cell size, bool executable_p) : secure_gc(secure_gc_) -{ - if(size > (1L << (sizeof(cell) * 8 - 6))) fatal_error("Heap too large",size); - seg = new segment(align_page(size),executable_p); - if(!seg) fatal_error("Out of memory in heap allocator",size); - state = new mark_bits(seg->start,size); - clear_free_list(); -} - -heap::~heap() -{ - delete seg; - seg = NULL; - delete state; - state = NULL; -} - -void heap::add_to_free_list(free_heap_block *block) -{ - if(block->size() < free_list_count * block_size_increment) - { - int index = block->size() / block_size_increment; - block->next_free = free.small_blocks[index]; - free.small_blocks[index] = block; - } - else - { - block->next_free = free.large_blocks; - free.large_blocks = block; - } -} - -/* Called after reading the code heap from the image file, and after code heap -compaction. Makes a free list consisting of one free block, at the very end. */ -void heap::build_free_list(cell size) -{ - clear_free_list(); - free_heap_block *end = (free_heap_block *)(seg->start + size); - end->set_free(); - end->set_size(seg->end - (cell)end); - add_to_free_list(end); -} - -void heap::assert_free_block(free_heap_block *block) -{ -#ifdef FACTOR_DEBUG - assert(block->free_p()); -#endif -} - -free_heap_block *heap::find_free_block(cell size) -{ - cell attempt = size; - - while(attempt < free_list_count * block_size_increment) - { - int index = attempt / block_size_increment; - free_heap_block *block = free.small_blocks[index]; - if(block) - { - assert_free_block(block); - free.small_blocks[index] = block->next_free; - return block; - } - - attempt *= 2; - } - - free_heap_block *prev = NULL; - free_heap_block *block = free.large_blocks; - - while(block) - { - assert_free_block(block); - if(block->size() >= size) - { - if(prev) - prev->next_free = block->next_free; - else - free.large_blocks = block->next_free; - return block; - } - - prev = block; - block = block->next_free; - } - - return NULL; -} - -free_heap_block *heap::split_free_block(free_heap_block *block, cell size) -{ - if(block->size() != size) - { - /* split the block in two */ - free_heap_block *split = (free_heap_block *)((cell)block + size); - split->set_free(); - split->set_size(block->size() - size); - split->next_free = block->next_free; - block->set_size(size); - add_to_free_list(split); - } - - return block; -} - -heap_block *heap::heap_allot(cell size) -{ - size = align(size,block_size_increment); - - free_heap_block *block = find_free_block(size); - if(block) - { - block = split_free_block(block,size); - return block; - } - else - return NULL; -} - -void heap::heap_free(heap_block *block) -{ - free_heap_block *free_block = (free_heap_block *)block; - free_block->set_free(); - add_to_free_list(free_block); -} - -void heap::mark_block(heap_block *block) -{ - state->set_marked_p(block); -} - -/* Compute total sum of sizes of free blocks, and size of largest free block */ -void heap::heap_usage(cell *used, cell *total_free, cell *max_free) -{ - *used = 0; - *total_free = 0; - *max_free = 0; - - heap_block *scan = first_block(); - heap_block *end = last_block(); - - while(scan != end) - { - cell size = scan->size(); - - if(scan->free_p()) - { - *total_free += size; - if(size > *max_free) - *max_free = size; - } - else - *used += size; - - scan = scan->next(); - } -} - -/* The size of the heap after compaction */ -cell heap::heap_size() -{ - heap_block *scan = first_block(); - heap_block *end = last_block(); - - while(scan != end) - { - if(scan->free_p()) break; - else scan = scan->next(); - } - - if(scan != end) - { - assert(scan->free_p()); - assert((cell)scan + scan->size() == seg->end); - - return (cell)scan - (cell)first_block(); - } - else - return seg->size; -} - -} diff --git a/vm/heap.hpp b/vm/heap.hpp index 7c3dca1eaf..653ac2d93d 100644 --- a/vm/heap.hpp +++ b/vm/heap.hpp @@ -2,30 +2,30 @@ namespace factor { static const cell free_list_count = 32; -static const cell block_size_increment = 16; struct heap_free_list { free_heap_block *small_blocks[free_list_count]; free_heap_block *large_blocks; }; -struct heap { +template struct heap { bool secure_gc; segment *seg; heap_free_list free; - mark_bits *state; + mark_bits *state; + HeapLayout layout; explicit heap(bool secure_gc_, cell size, bool executable_p); ~heap(); - - inline heap_block *first_block() + + inline Block *first_block() { - return (heap_block *)seg->start; + return (Block *)seg->start; } - - inline heap_block *last_block() + + inline Block *last_block() { - return (heap_block *)seg->end; + return (Block *)seg->end; } void clear_free_list(); @@ -34,46 +34,253 @@ struct heap { void assert_free_block(free_heap_block *block); free_heap_block *find_free_block(cell size); free_heap_block *split_free_block(free_heap_block *block, cell size); - heap_block *heap_allot(cell size); - void heap_free(heap_block *block); - void mark_block(heap_block *block); + Block *heap_allot(cell size); + void heap_free(Block *block); + void mark_block(Block *block); void heap_usage(cell *used, cell *total_free, cell *max_free); cell heap_size(); - void compact_heap(); template void sweep_heap(Iterator &iter); template void compact_heap(Iterator &iter); template void iterate_heap(Iterator &iter) { - heap_block *scan = first_block(); - heap_block *end = last_block(); + Block *scan = first_block(); + Block *end = last_block(); while(scan != end) { - heap_block *next = scan->next(); - if(!scan->free_p()) iter(scan,scan->size()); + Block *next = layout.next_block_after(scan); + if(!scan->free_p()) iter(scan,layout.block_size(scan)); scan = next; } } }; +template +void heap::clear_free_list() +{ + memset(&free,0,sizeof(heap_free_list)); +} + +template +heap::heap(bool secure_gc_, cell size, bool executable_p) : secure_gc(secure_gc_) +{ + if(size > (1L << (sizeof(cell) * 8 - 6))) fatal_error("Heap too large",size); + seg = new segment(align_page(size),executable_p); + if(!seg) fatal_error("Out of memory in heap allocator",size); + state = new mark_bits(seg->start,size); + clear_free_list(); +} + +template +heap::~heap() +{ + delete seg; + seg = NULL; + delete state; + state = NULL; +} + +template +void heap::add_to_free_list(free_heap_block *block) +{ + if(block->size() < free_list_count * block_granularity) + { + int index = block->size() / block_granularity; + block->next_free = free.small_blocks[index]; + free.small_blocks[index] = block; + } + else + { + block->next_free = free.large_blocks; + free.large_blocks = block; + } +} + +/* Called after reading the code heap from the image file, and after code heap +compaction. Makes a free list consisting of one free block, at the very end. */ +template +void heap::build_free_list(cell size) +{ + clear_free_list(); + free_heap_block *end = (free_heap_block *)(seg->start + size); + end->set_free(); + end->set_size(seg->end - (cell)end); + add_to_free_list(end); +} + +template +void heap::assert_free_block(free_heap_block *block) +{ +#ifdef FACTOR_DEBUG + assert(block->free_p()); +#endif +} + +template +free_heap_block *heap::find_free_block(cell size) +{ + cell attempt = size; + + while(attempt < free_list_count * block_granularity) + { + int index = attempt / block_granularity; + free_heap_block *block = free.small_blocks[index]; + if(block) + { + assert_free_block(block); + free.small_blocks[index] = block->next_free; + return block; + } + + attempt *= 2; + } + + free_heap_block *prev = NULL; + free_heap_block *block = free.large_blocks; + + while(block) + { + assert_free_block(block); + if(block->size() >= size) + { + if(prev) + prev->next_free = block->next_free; + else + free.large_blocks = block->next_free; + return block; + } + + prev = block; + block = block->next_free; + } + + return NULL; +} + +template +free_heap_block *heap::split_free_block(free_heap_block *block, cell size) +{ + if(block->size() != size) + { + /* split the block in two */ + free_heap_block *split = (free_heap_block *)((cell)block + size); + split->set_free(); + split->set_size(block->size() - size); + split->next_free = block->next_free; + block->set_size(size); + add_to_free_list(split); + } + + return block; +} + +template +Block *heap::heap_allot(cell size) +{ + size = align(size,block_granularity); + + free_heap_block *block = find_free_block(size); + if(block) + { + block = split_free_block(block,size); + return (Block *)block; + } + else + return NULL; +} + +template +void heap::heap_free(Block *block) +{ + free_heap_block *free_block = (free_heap_block *)block; + free_block->set_free(); + add_to_free_list(free_block); +} + +template +void heap::mark_block(Block *block) +{ + state->set_marked_p(block); +} + +/* Compute total sum of sizes of free blocks, and size of largest free block */ +template +void heap::heap_usage(cell *used, cell *total_free, cell *max_free) +{ + *used = 0; + *total_free = 0; + *max_free = 0; + + Block *scan = first_block(); + Block *end = last_block(); + + while(scan != end) + { + cell size = layout.block_size(scan); + + if(scan->free_p()) + { + *total_free += size; + if(size > *max_free) + *max_free = size; + } + else + *used += size; + + scan = layout.next_block_after(scan); + } +} + +/* The size of the heap after compaction */ +template +cell heap::heap_size() +{ + Block *scan = first_block(); + Block *end = last_block(); + + while(scan != end) + { + if(scan->free_p()) break; + else scan = layout.next_block_after(scan); + } + + if(scan != end) + { + free_heap_block *free_block = (free_heap_block *)scan; + assert(free_block->free_p()); + assert((cell)scan + scan->size() == seg->end); + + return (cell)scan - (cell)first_block(); + } + else + return seg->size; +} + /* After code GC, all live code blocks are marked, so any which are not marked can be reclaimed. */ -template void heap::sweep_heap(Iterator &iter) +template +template +void heap::sweep_heap(Iterator &iter) { this->clear_free_list(); - heap_block *prev = NULL; - heap_block *scan = this->first_block(); - heap_block *end = this->last_block(); + Block *prev = NULL; + Block *scan = this->first_block(); + Block *end = this->last_block(); while(scan != end) { if(scan->free_p()) { + free_heap_block *free_scan = (free_heap_block *)scan; + if(prev && prev->free_p()) - prev->set_size(prev->size() + scan->size()); + { + free_heap_block *free_prev = (free_heap_block *)prev; + free_prev->set_size(free_prev->size() + free_scan->size()); + } else prev = scan; } @@ -82,17 +289,17 @@ template void heap::sweep_heap(Iterator &iter) if(prev && prev->free_p()) this->add_to_free_list((free_heap_block *)prev); prev = scan; - iter(scan,scan->size()); + iter(scan,layout.block_size(scan)); } else { if(secure_gc) - memset(scan + 1,0,scan->size() - sizeof(heap_block)); + memset(scan + 1,0,layout.block_size(scan) - sizeof(heap_block)); if(prev && prev->free_p()) { free_heap_block *free_prev = (free_heap_block *)prev; - free_prev->set_size(free_prev->size() + scan->size()); + free_prev->set_size(free_prev->size() + layout.block_size(scan)); } else { @@ -101,7 +308,7 @@ template void heap::sweep_heap(Iterator &iter) } } - scan = scan->next(); + scan = layout.next_block_after(scan); } if(prev && prev->free_p()) @@ -110,14 +317,16 @@ template void heap::sweep_heap(Iterator &iter) /* The forwarding map must be computed first by calling state->compute_forwarding(). */ -template void heap::compact_heap(Iterator &iter) +template +template +void heap::compact_heap(Iterator &iter) { - heap_compacter compacter(state,first_block(),iter); - this->iterate_heap(compacter); + heap_compactor compactor(state,first_block(),iter); + this->iterate_heap(compactor); /* Now update the free list; there will be a single free block at the end */ - this->build_free_list((cell)compacter.address - this->seg->start); + this->build_free_list((cell)compactor.address - this->seg->start); } } diff --git a/vm/layouts.hpp b/vm/layouts.hpp index 3249aac946..6e8c89ceb3 100644 --- a/vm/layouts.hpp +++ b/vm/layouts.hpp @@ -231,11 +231,6 @@ struct heap_block { header = (header & 0x7) | (size << 3); } - - inline heap_block *next() - { - return (heap_block *)((cell)this + size()); - } }; struct free_heap_block : public heap_block diff --git a/vm/mark_bits.hpp b/vm/mark_bits.hpp index a4dc715c50..d9c9534edb 100644 --- a/vm/mark_bits.hpp +++ b/vm/mark_bits.hpp @@ -1,9 +1,11 @@ namespace factor { +const int block_granularity = 16; const int forwarding_granularity = 64; -template struct mark_bits { +template struct mark_bits { + HeapLayout layout; cell start; cell size; cell bits_size; @@ -23,7 +25,7 @@ template struct mark_bits { explicit mark_bits(cell start_, cell size_) : start(start_), size(size_), - bits_size(size / Granularity / forwarding_granularity), + bits_size(size / block_granularity / forwarding_granularity), marked(new u64[bits_size]), forwarding(new cell[bits_size]) { @@ -41,12 +43,12 @@ template struct mark_bits { cell block_line(Block *address) { - return (((cell)address - start) / Granularity); + return (((cell)address - start) / block_granularity); } Block *line_block(cell line) { - return (Block *)(line * Granularity + start); + return (Block *)(line * block_granularity + start); } std::pair bitmap_deref(Block *address) @@ -71,7 +73,7 @@ template struct mark_bits { void set_bitmap_range(u64 *bits, Block *address) { std::pair start = bitmap_deref(address); - std::pair end = bitmap_deref(address->next()); + std::pair end = bitmap_deref(layout.next_block_after(address)); u64 start_mask = ((u64)1 << start.second) - 1; u64 end_mask = ((u64)1 << end.second) - 1; @@ -139,12 +141,12 @@ template struct mark_bits { } }; -template struct heap_compacter { - mark_bits *state; +template struct heap_compactor { + mark_bits *state; char *address; Iterator &iter; - explicit heap_compacter(mark_bits *state_, Block *address_, Iterator &iter_) : + explicit heap_compactor(mark_bits *state_, Block *address_, Iterator &iter_) : state(state_), address((char *)address_), iter(iter_) {} void operator()(Block *block, cell size) From 50f9bf67a78a36e0bdb65c45cde8dba6f38eb1ba Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Tue, 20 Oct 2009 12:19:02 -0500 Subject: [PATCH 06/74] vm: fix crash when converting a callstack to an array --- vm/callstack.cpp | 6 +++--- vm/code_block.cpp | 2 +- vm/vm.hpp | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/vm/callstack.cpp b/vm/callstack.cpp index 4721fc4ece..623db8a3fe 100755 --- a/vm/callstack.cpp +++ b/vm/callstack.cpp @@ -76,7 +76,7 @@ code_block *factor_vm::frame_code(stack_frame *frame) return (code_block *)frame->xt - 1; } -cell factor_vm::frame_type(stack_frame *frame) +code_block_type factor_vm::frame_type(stack_frame *frame) { return frame_code(frame)->type(); } @@ -97,7 +97,7 @@ cell factor_vm::frame_scan(stack_frame *frame) { switch(frame_type(frame)) { - case QUOTATION_TYPE: + case code_block_unoptimized: { cell quot = frame_executing(frame); if(to_boolean(quot)) @@ -111,7 +111,7 @@ cell factor_vm::frame_scan(stack_frame *frame) else return false_object; } - case WORD_TYPE: + case code_block_optimized: return false_object; default: critical_error("Bad frame type",frame_type(frame)); diff --git a/vm/code_block.cpp b/vm/code_block.cpp index 7e6892202a..e29d708c25 100755 --- a/vm/code_block.cpp +++ b/vm/code_block.cpp @@ -479,7 +479,7 @@ code_block *factor_vm::add_code_block(code_block_type type, cell code_, cell lab gc_root relocation(relocation_,this); gc_root literals(literals_,this); - cell code_length = align8(array_capacity(code.untagged())); + cell code_length = array_capacity(code.untagged()); code_block *compiled = allot_code_block(code_length,type); compiled->owner = owner.value(); diff --git a/vm/vm.hpp b/vm/vm.hpp index 921e829bda..a2f979e400 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -571,7 +571,7 @@ struct factor_vm void primitive_callstack(); void primitive_set_callstack(); code_block *frame_code(stack_frame *frame); - cell frame_type(stack_frame *frame); + code_block_type frame_type(stack_frame *frame); cell frame_executing(stack_frame *frame); stack_frame *frame_successor(stack_frame *frame); cell frame_scan(stack_frame *frame); From d85d84697ab57fd0e1b83eb8dde7a0986a6aa1d8 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Tue, 20 Oct 2009 12:45:00 -0500 Subject: [PATCH 07/74] Change data heap alignment to 16 bytes --- basis/bootstrap/image/image.factor | 9 +++++++-- basis/compiler/constants/constants.factor | 2 +- basis/cpu/ppc/ppc.factor | 2 +- basis/cpu/x86/x86.factor | 2 +- core/bootstrap/layouts/layouts.factor | 2 ++ core/layouts/layouts.factor | 2 ++ vm/arrays.cpp | 8 ++++---- vm/bignum.cpp | 2 +- vm/byte_arrays.cpp | 4 ++-- vm/callbacks.cpp | 4 ++-- vm/contexts.cpp | 2 +- vm/data_heap.cpp | 2 +- vm/generic_arrays.hpp | 4 ++-- vm/io.cpp | 2 +- vm/layouts.hpp | 14 ++++++++------ vm/strings.cpp | 2 +- vm/vm.hpp | 2 +- vm/zone.hpp | 2 +- 18 files changed, 39 insertions(+), 28 deletions(-) diff --git a/basis/bootstrap/image/image.factor b/basis/bootstrap/image/image.factor index e086215e91..711f2f36f3 100644 --- a/basis/bootstrap/image/image.factor +++ b/basis/bootstrap/image/image.factor @@ -218,8 +218,12 @@ USERENV: undefined-quot 60 : here-as ( tag -- pointer ) here bitor ; +: (align-here) ( alignment -- ) + [ here neg ] dip rem + [ bootstrap-cell /i [ 0 emit ] times ] unless-zero ; + : align-here ( -- ) - here 8 mod 4 = [ 0 emit ] when ; + data-alignment get (align-here) ; : emit-fixnum ( n -- ) tag-fixnum emit ; @@ -293,7 +297,7 @@ M: fake-bignum ' n>> tag-fixnum ; M: float ' [ float [ - align-here double>bits emit-64 + 8 (align-here) double>bits emit-64 ] emit-object ] cache-eql-object ; @@ -411,6 +415,7 @@ M: byte-array ' [ byte-array [ dup length emit-fixnum + bootstrap-cell 4 = [ 0 emit 0 emit ] when pad-bytes emit-bytes ] emit-object ] cache-eq-object ; diff --git a/basis/compiler/constants/constants.factor b/basis/compiler/constants/constants.factor index a22d522809..ab607d2178 100644 --- a/basis/compiler/constants/constants.factor +++ b/basis/compiler/constants/constants.factor @@ -17,7 +17,7 @@ CONSTANT: deck-bits 18 : string-offset ( -- n ) 4 string tag-number slot-offset ; inline : string-aux-offset ( -- n ) 2 string tag-number slot-offset ; inline : profile-count-offset ( -- n ) 8 \ word tag-number slot-offset ; inline -: byte-array-offset ( -- n ) 2 byte-array tag-number slot-offset ; inline +: byte-array-offset ( -- n ) 16 byte-array tag-number - ; inline : alien-offset ( -- n ) 3 alien tag-number slot-offset ; inline : underlying-alien-offset ( -- n ) 1 alien tag-number slot-offset ; inline : tuple-class-offset ( -- n ) 1 tuple tag-number slot-offset ; inline diff --git a/basis/cpu/ppc/ppc.factor b/basis/cpu/ppc/ppc.factor index 517aa7587d..7226145c27 100644 --- a/basis/cpu/ppc/ppc.factor +++ b/basis/cpu/ppc/ppc.factor @@ -374,7 +374,7 @@ M: ppc %set-alien-double -rot STFD ; [ drop load-zone-ptr ] [ swap 0 LWZ ] 2bi ; :: inc-allot-ptr ( nursery-ptr allot-ptr n -- ) - scratch-reg allot-ptr n 8 align ADDI + scratch-reg allot-ptr n data-alignment get align ADDI scratch-reg nursery-ptr 0 STW ; :: store-header ( dst class -- ) diff --git a/basis/cpu/x86/x86.factor b/basis/cpu/x86/x86.factor index 4576956335..e061ea5d95 100644 --- a/basis/cpu/x86/x86.factor +++ b/basis/cpu/x86/x86.factor @@ -388,7 +388,7 @@ M: x86 %vm-field-ptr ( dst field -- ) [ drop "nursery" %vm-field-ptr ] [ swap [] MOV ] 2bi ; : inc-allot-ptr ( nursery-ptr n -- ) - [ [] ] dip 8 align ADD ; + [ [] ] dip data-alignment get align ADD ; : store-header ( temp class -- ) [ [] ] [ type-number tag-fixnum ] bi* MOV ; diff --git a/core/bootstrap/layouts/layouts.factor b/core/bootstrap/layouts/layouts.factor index 5ed92b7776..fef7ba2a83 100644 --- a/core/bootstrap/layouts/layouts.factor +++ b/core/bootstrap/layouts/layouts.factor @@ -5,6 +5,8 @@ hashtables vectors strings sbufs arrays quotations assocs layouts classes.tuple.private kernel.private ; +16 data-alignment set + BIN: 111 tag-mask set 8 num-tags set 3 tag-bits set diff --git a/core/layouts/layouts.factor b/core/layouts/layouts.factor index be6276a684..2f0fa12d44 100644 --- a/core/layouts/layouts.factor +++ b/core/layouts/layouts.factor @@ -4,6 +4,8 @@ USING: namespaces math words kernel assocs classes math.order kernel.private ; IN: layouts +SYMBOL: data-alignment + SYMBOL: tag-mask SYMBOL: num-tags diff --git a/vm/arrays.cpp b/vm/arrays.cpp index 09c6998e69..3af8b0600b 100644 --- a/vm/arrays.cpp +++ b/vm/arrays.cpp @@ -7,7 +7,7 @@ namespace factor array *factor_vm::allot_array(cell capacity, cell fill_) { gc_root fill(fill_,this); - gc_root new_array(allot_array_internal(capacity),this); + gc_root new_array(allot_uninitialized_array(capacity),this); if(fill.value() == tag_fixnum(0)) memset(new_array->data(),'\0',capacity * sizeof(cell)); @@ -33,7 +33,7 @@ void factor_vm::primitive_array() cell factor_vm::allot_array_1(cell obj_) { gc_root obj(obj_,this); - gc_root a(allot_array_internal(1),this); + gc_root a(allot_uninitialized_array(1),this); set_array_nth(a.untagged(),0,obj.value()); return a.value(); } @@ -42,7 +42,7 @@ cell factor_vm::allot_array_2(cell v1_, cell v2_) { gc_root v1(v1_,this); gc_root v2(v2_,this); - gc_root a(allot_array_internal(2),this); + gc_root a(allot_uninitialized_array(2),this); set_array_nth(a.untagged(),0,v1.value()); set_array_nth(a.untagged(),1,v2.value()); return a.value(); @@ -54,7 +54,7 @@ cell factor_vm::allot_array_4(cell v1_, cell v2_, cell v3_, cell v4_) gc_root v2(v2_,this); gc_root v3(v3_,this); gc_root v4(v4_,this); - gc_root a(allot_array_internal(4),this); + gc_root a(allot_uninitialized_array(4),this); set_array_nth(a.untagged(),0,v1.value()); set_array_nth(a.untagged(),1,v2.value()); set_array_nth(a.untagged(),2,v3.value()); diff --git a/vm/bignum.cpp b/vm/bignum.cpp index d8c5452b08..5a391e7625 100755 --- a/vm/bignum.cpp +++ b/vm/bignum.cpp @@ -1299,7 +1299,7 @@ bignum *factor_vm::bignum_digit_to_bignum(bignum_digit_type digit, int negative_ bignum *factor_vm::allot_bignum(bignum_length_type length, int negative_p) { BIGNUM_ASSERT ((length >= 0) || (length < BIGNUM_RADIX)); - bignum * result = allot_array_internal(length + 1); + bignum * result = allot_uninitialized_array(length + 1); BIGNUM_SET_NEGATIVE_P (result, negative_p); return (result); } diff --git a/vm/byte_arrays.cpp b/vm/byte_arrays.cpp index 56b5db7ad8..fa02ede6c3 100644 --- a/vm/byte_arrays.cpp +++ b/vm/byte_arrays.cpp @@ -5,7 +5,7 @@ namespace factor byte_array *factor_vm::allot_byte_array(cell size) { - byte_array *array = allot_array_internal(size); + byte_array *array = allot_uninitialized_array(size); memset(array + 1,0,size); return array; } @@ -19,7 +19,7 @@ void factor_vm::primitive_byte_array() void factor_vm::primitive_uninitialized_byte_array() { cell size = unbox_array_size(); - dpush(tag(allot_array_internal(size))); + dpush(tag(allot_uninitialized_array(size))); } void factor_vm::primitive_resize_byte_array() diff --git a/vm/callbacks.cpp b/vm/callbacks.cpp index dca0eb6c24..599271555b 100644 --- a/vm/callbacks.cpp +++ b/vm/callbacks.cpp @@ -39,14 +39,14 @@ callback *callback_heap::add(code_block *compiled) tagged insns(array_nth(code_template.untagged(),0)); cell size = array_capacity(insns.untagged()); - cell bump = align8(size) + sizeof(callback); + cell bump = align(size,sizeof(cell)) + sizeof(callback); if(here + bump > seg->end) fatal_error("Out of callback space",0); callback *stub = (callback *)here; stub->compiled = compiled; memcpy(stub + 1,insns->data(),size); - stub->size = align8(size); + stub->size = align(size,sizeof(cell)); here += bump; update(stub); diff --git a/vm/contexts.cpp b/vm/contexts.cpp index cc7029e7f1..ce52555a21 100644 --- a/vm/contexts.cpp +++ b/vm/contexts.cpp @@ -133,7 +133,7 @@ bool factor_vm::stack_to_array(cell bottom, cell top) return false; else { - array *a = allot_array_internal(depth / sizeof(cell)); + array *a = allot_uninitialized_array(depth / sizeof(cell)); memcpy(a + 1,(void*)bottom,depth); dpush(tag(a)); return true; diff --git a/vm/data_heap.cpp b/vm/data_heap.cpp index 335938acab..05ce026973 100755 --- a/vm/data_heap.cpp +++ b/vm/data_heap.cpp @@ -119,7 +119,7 @@ cell factor_vm::object_size(cell tagged) /* Size of the object pointed to by an untagged pointer */ cell factor_vm::untagged_object_size(object *pointer) { - return align8(unaligned_object_size(pointer)); + return align(unaligned_object_size(pointer),data_alignment); } /* Size of the data area of an object pointed to by an untagged pointer */ diff --git a/vm/generic_arrays.hpp b/vm/generic_arrays.hpp index 0ba6d11da2..e1d2c4dc0b 100755 --- a/vm/generic_arrays.hpp +++ b/vm/generic_arrays.hpp @@ -19,7 +19,7 @@ template cell array_size(Array *array) return array_size(array_capacity(array)); } -template Array *factor_vm::allot_array_internal(cell capacity) +template Array *factor_vm::allot_uninitialized_array(cell capacity) { Array *array = allot(array_size(capacity)); array->capacity = tag_fixnum(capacity); @@ -46,7 +46,7 @@ template Array *factor_vm::reallot_array(Array *array_, cell cap if(capacity < to_copy) to_copy = capacity; - Array *new_array = allot_array_internal(capacity); + Array *new_array = allot_uninitialized_array(capacity); memcpy(new_array + 1,array.untagged() + 1,to_copy * Array::element_size); memset((char *)(new_array + 1) + to_copy * Array::element_size, diff --git a/vm/io.cpp b/vm/io.cpp index d5cfc1745c..a75f41c5bf 100755 --- a/vm/io.cpp +++ b/vm/io.cpp @@ -88,7 +88,7 @@ void factor_vm::primitive_fread() return; } - gc_root buf(allot_array_internal(size),this); + gc_root buf(allot_uninitialized_array(size),this); for(;;) { diff --git a/vm/layouts.hpp b/vm/layouts.hpp index 6e8c89ceb3..c25c2c0fc1 100644 --- a/vm/layouts.hpp +++ b/vm/layouts.hpp @@ -23,10 +23,7 @@ inline static cell align(cell a, cell b) return (a + (b-1)) & ~(b-1); } -inline static cell align8(cell a) -{ - return align(a,8); -} +static const cell data_alignment = 16; #define WORD_SIZE (signed)(sizeof(cell)*8) @@ -186,6 +183,11 @@ struct byte_array : public object { /* tagged */ cell capacity; +#ifndef FACTOR_64 + cell padding0; + cell padding1; +#endif + template Scalar *data() { return (Scalar *)(this + 1); } }; @@ -249,9 +251,9 @@ struct code_block : public heap_block return (void *)(this + 1); } - cell type() + code_block_type type() { - return (header >> 1) & 0x3; + return (code_block_type)((header >> 1) & 0x3); } void set_type(code_block_type type) diff --git a/vm/strings.cpp b/vm/strings.cpp index d7434fe660..23fa75acca 100644 --- a/vm/strings.cpp +++ b/vm/strings.cpp @@ -45,7 +45,7 @@ void factor_vm::set_string_nth_slow(string *str_, cell index, cell ch) if the most significant bit of a character is set. Initially all of the bits are clear. */ - aux = allot_array_internal(untag_fixnum(str->length) * sizeof(u16)); + aux = allot_uninitialized_array(untag_fixnum(str->length) * sizeof(u16)); str->aux = tag(aux); write_barrier(&str->aux); diff --git a/vm/vm.hpp b/vm/vm.hpp index a2f979e400..87697d5a0f 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -298,7 +298,7 @@ struct factor_vm } // generic arrays - template Array *allot_array_internal(cell capacity); + template Array *allot_uninitialized_array(cell capacity); template bool reallot_array_in_place_p(Array *array, cell capacity); template Array *reallot_array(Array *array_, cell capacity); diff --git a/vm/zone.hpp b/vm/zone.hpp index 4fe4ae9b6b..2c28922fbc 100644 --- a/vm/zone.hpp +++ b/vm/zone.hpp @@ -18,7 +18,7 @@ struct zone { inline object *allot(cell size) { cell h = here; - here = h + align8(size); + here = h + align(size,data_alignment); return (object *)h; } }; From d22d5466fcbb6164b913d8e263cc8f3d9200a59a Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Tue, 20 Oct 2009 13:13:39 -0500 Subject: [PATCH 08/74] vm: move factor_vm::untagged_object_size() to object::size() --- vm/aging_collector.cpp | 2 +- vm/collector.hpp | 2 +- vm/copying_collector.hpp | 8 ++-- vm/data_heap.cpp | 73 +++++++++++-------------------------- vm/data_heap.hpp | 28 ++++++++++++++ vm/full_collector.cpp | 6 +-- vm/image.cpp | 4 +- vm/layouts.hpp | 16 +++++--- vm/old_space.cpp | 4 +- vm/old_space.hpp | 2 +- vm/to_tenured_collector.cpp | 2 +- vm/vm.hpp | 5 --- 12 files changed, 76 insertions(+), 76 deletions(-) diff --git a/vm/aging_collector.cpp b/vm/aging_collector.cpp index 5e284be587..49b1c520ec 100644 --- a/vm/aging_collector.cpp +++ b/vm/aging_collector.cpp @@ -32,7 +32,7 @@ void factor_vm::collect_aging() current_gc->op = collect_aging_op; std::swap(data->aging,data->aging_semispace); - reset_generation(data->aging); + data->reset_generation(data->aging); aging_collector collector(this); diff --git a/vm/collector.hpp b/vm/collector.hpp index bbaad1d570..9200d95399 100644 --- a/vm/collector.hpp +++ b/vm/collector.hpp @@ -68,7 +68,7 @@ template struct collector { object *promote_object(object *untagged) { - cell size = parent->untagged_object_size(untagged); + cell size = untagged->size(); object *newpointer = target->allot(size); /* XXX not exception-safe */ if(!newpointer) longjmp(current_gc->gc_unwind,1); diff --git a/vm/copying_collector.hpp b/vm/copying_collector.hpp index 640d355bf4..dd36b680a6 100644 --- a/vm/copying_collector.hpp +++ b/vm/copying_collector.hpp @@ -97,7 +97,7 @@ struct copying_collector : collector { { start = gen->find_object_containing_card(card_index - gen_start_card); binary_start = start + this->parent->binary_payload_start((object *)start); - end = start + this->parent->untagged_object_size((object *)start); + end = start + ((object *)start)->size(); } #ifdef FACTOR_DEBUG @@ -113,11 +113,11 @@ scan_next_object: { card_end_address(card_index)); if(end < card_end_address(card_index)) { - start = gen->next_object_after(this->parent,start); + start = gen->next_object_after(start); if(start) { binary_start = start + this->parent->binary_payload_start((object *)start); - end = start + this->parent->untagged_object_size((object *)start); + end = start + ((object *)start)->size(); goto scan_next_object; } } @@ -158,7 +158,7 @@ end: this->parent->gc_stats.card_scan_time += (current_micros() - start_time); while(scan && scan < this->target->here) { this->trace_slots((object *)scan); - scan = this->target->next_object_after(this->parent,scan); + scan = this->target->next_object_after(scan); } } }; diff --git a/vm/data_heap.cpp b/vm/data_heap.cpp index 05ce026973..05f8f3a044 100755 --- a/vm/data_heap.cpp +++ b/vm/data_heap.cpp @@ -65,40 +65,14 @@ data_heap *data_heap::grow(cell requested_bytes) return new data_heap(young_size,aging_size,new_tenured_size); } -void factor_vm::clear_cards(old_space *gen) -{ - cell first_card = addr_to_card(gen->start - data->start); - cell last_card = addr_to_card(gen->end - data->start); - memset(&data->cards[first_card],0,last_card - first_card); -} - -void factor_vm::clear_decks(old_space *gen) -{ - cell first_deck = addr_to_deck(gen->start - data->start); - cell last_deck = addr_to_deck(gen->end - data->start); - memset(&data->decks[first_deck],0,last_deck - first_deck); -} - -/* After garbage collection, any generations which are now empty need to have -their allocation pointers and cards reset. */ -void factor_vm::reset_generation(old_space *gen) -{ - gen->here = gen->start; - if(secure_gc) memset((void*)gen->start,69,gen->size); - - clear_cards(gen); - clear_decks(gen); - gen->clear_object_start_offsets(); -} - void factor_vm::set_data_heap(data_heap *data_) { data = data_; nursery = *data->nursery; nursery.here = nursery.start; init_card_decks(); - reset_generation(data->aging); - reset_generation(data->tenured); + data->reset_generation(data->aging); + data->reset_generation(data->tenured); } void factor_vm::init_data_heap(cell young_size, cell aging_size, cell tenured_size, bool secure_gc_) @@ -113,46 +87,43 @@ cell factor_vm::object_size(cell tagged) if(immediate_p(tagged)) return 0; else - return untagged_object_size(untag(tagged)); + return untag(tagged)->size(); } /* Size of the object pointed to by an untagged pointer */ -cell factor_vm::untagged_object_size(object *pointer) +cell object::size() { - return align(unaligned_object_size(pointer),data_alignment); -} - -/* Size of the data area of an object pointed to by an untagged pointer */ -cell factor_vm::unaligned_object_size(object *pointer) -{ - switch(pointer->h.hi_tag()) + switch(h.hi_tag()) { case ARRAY_TYPE: - return array_size((array*)pointer); + return align(array_size((array*)this),data_alignment); case BIGNUM_TYPE: - return array_size((bignum*)pointer); + return align(array_size((bignum*)this),data_alignment); case BYTE_ARRAY_TYPE: - return array_size((byte_array*)pointer); + return align(array_size((byte_array*)this),data_alignment); case STRING_TYPE: - return string_size(string_capacity((string*)pointer)); + return align(string_size(string_capacity((string*)this)),data_alignment); case TUPLE_TYPE: - return tuple_size(untag(((tuple *)pointer)->layout)); + { + tuple_layout *layout = (tuple_layout *)UNTAG(((tuple *)this)->layout); + return align(tuple_size(layout),data_alignment); + } case QUOTATION_TYPE: - return sizeof(quotation); + return align(sizeof(quotation),data_alignment); case WORD_TYPE: - return sizeof(word); + return align(sizeof(word),data_alignment); case FLOAT_TYPE: - return sizeof(boxed_float); + return align(sizeof(boxed_float),data_alignment); case DLL_TYPE: - return sizeof(dll); + return align(sizeof(dll),data_alignment); case ALIEN_TYPE: - return sizeof(alien); + return align(sizeof(alien),data_alignment); case WRAPPER_TYPE: - return sizeof(wrapper); + return align(sizeof(wrapper),data_alignment); case CALLSTACK_TYPE: - return callstack_size(untag_fixnum(((callstack *)pointer)->length)); + return align(callstack_size(untag_fixnum(((callstack *)this)->length)),data_alignment); default: - critical_error("Invalid header",(cell)pointer); + critical_error("Invalid header",(cell)this); return 0; /* can't happen */ } } @@ -246,7 +217,7 @@ cell factor_vm::next_object() return false_object; object *obj = (object *)heap_scan_ptr; - heap_scan_ptr += untagged_object_size(obj); + heap_scan_ptr += obj->size(); return tag_dynamic(obj); } diff --git a/vm/data_heap.hpp b/vm/data_heap.hpp index 10f3698e74..c882262732 100755 --- a/vm/data_heap.hpp +++ b/vm/data_heap.hpp @@ -25,6 +25,34 @@ struct data_heap { explicit data_heap(cell young_size, cell aging_size, cell tenured_size); ~data_heap(); data_heap *grow(cell requested_size); + template void clear_cards(Generation *gen); + template void clear_decks(Generation *gen); + template void reset_generation(Generation *gen); }; +template void data_heap::clear_cards(Generation *gen) +{ + cell first_card = addr_to_card(gen->start - start); + cell last_card = addr_to_card(gen->end - start); + memset(&cards[first_card],0,last_card - first_card); +} + +template void data_heap::clear_decks(Generation *gen) +{ + cell first_deck = addr_to_deck(gen->start - start); + cell last_deck = addr_to_deck(gen->end - start); + memset(&decks[first_deck],0,last_deck - first_deck); +} + +/* After garbage collection, any generations which are now empty need to have +their allocation pointers and cards reset. */ +template void data_heap::reset_generation(Generation *gen) +{ + gen->here = gen->start; + + clear_cards(gen); + clear_decks(gen); + gen->clear_object_start_offsets(); +} + } diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index 61827fba41..1bb03d08e7 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -100,7 +100,7 @@ void full_collector::cheneys_algorithm() object *obj = (object *)scan; this->trace_slots(obj); this->mark_object_code_block(obj); - scan = target->next_object_after(this->parent,scan); + scan = target->next_object_after(scan); } } @@ -120,7 +120,7 @@ void factor_vm::collect_full_impl(bool trace_contexts_p) collector.cheneys_algorithm(); - reset_generation(data->aging); + data->reset_generation(data->aging); nursery.here = nursery.start; } @@ -146,7 +146,7 @@ void factor_vm::collect_full(bool trace_contexts_p, bool compact_code_heap_p) { /* Copy all live objects to the tenured semispace. */ std::swap(data->tenured,data->tenured_semispace); - reset_generation(data->tenured); + data->reset_generation(data->tenured); collect_full_impl(trace_contexts_p); if(compact_code_heap_p) diff --git a/vm/image.cpp b/vm/image.cpp index c96da6b703..f41ae555a3 100755 --- a/vm/image.cpp +++ b/vm/image.cpp @@ -155,7 +155,7 @@ void factor_vm::relocate_object(object *object, data_fixup(&t->layout,data_relocation_base); cell *scan = t->data(); - cell *end = (cell *)((cell)object + untagged_object_size(object)); + cell *end = (cell *)((cell)object + object->size()); for(; scan < end; scan++) data_fixup(scan,data_relocation_base); @@ -204,7 +204,7 @@ void factor_vm::relocate_data(cell data_relocation_base, cell code_relocation_ba { relocate_object((object *)obj,data_relocation_base,code_relocation_base); data->tenured->record_object_start_offset((object *)obj); - obj = data->tenured->next_object_after(this,obj); + obj = data->tenured->next_object_after(obj); } } diff --git a/vm/layouts.hpp b/vm/layouts.hpp index c25c2c0fc1..f0dbff6c30 100644 --- a/vm/layouts.hpp +++ b/vm/layouts.hpp @@ -114,26 +114,31 @@ struct header { explicit header(cell value_) : value(value_ << TAG_BITS) {} - void check_header() { + void check_header() + { #ifdef FACTOR_DEBUG assert(TAG(value) == FIXNUM_TYPE && untag_fixnum(value) < TYPE_COUNT); #endif } - cell hi_tag() { + cell hi_tag() + { check_header(); return value >> TAG_BITS; } - bool forwarding_pointer_p() { + bool forwarding_pointer_p() + { return TAG(value) == GC_COLLECTED; } - object *forwarding_pointer() { + object *forwarding_pointer() + { return (object *)UNTAG(value); } - void forward_to(object *pointer) { + void forward_to(object *pointer) + { value = RETAG(pointer,GC_COLLECTED); } }; @@ -144,6 +149,7 @@ struct object { NO_TYPE_CHECK; header h; cell *slots() { return (cell *)this; } + cell size(); }; /* Assembly code makes assumptions about the layout of this struct */ diff --git a/vm/old_space.cpp b/vm/old_space.cpp index 5fd78a7cf4..9440749e52 100644 --- a/vm/old_space.cpp +++ b/vm/old_space.cpp @@ -62,9 +62,9 @@ void old_space::clear_object_start_offsets() memset(object_start_offsets,card_starts_inside_object,addr_to_card(size)); } -cell old_space::next_object_after(factor_vm *parent, cell scan) +cell old_space::next_object_after(cell scan) { - cell size = parent->untagged_object_size((object *)scan); + cell size = ((object *)scan)->size(); if(scan + size < here) return scan + size; else diff --git a/vm/old_space.hpp b/vm/old_space.hpp index d037a039ae..e59537cffb 100644 --- a/vm/old_space.hpp +++ b/vm/old_space.hpp @@ -15,7 +15,7 @@ struct old_space : zone { void record_object_start_offset(object *obj); object *allot(cell size); void clear_object_start_offsets(); - cell next_object_after(factor_vm *parent, cell scan); + cell next_object_after(cell scan); }; } diff --git a/vm/to_tenured_collector.cpp b/vm/to_tenured_collector.cpp index b5d4793ceb..3676324ce2 100644 --- a/vm/to_tenured_collector.cpp +++ b/vm/to_tenured_collector.cpp @@ -25,7 +25,7 @@ void factor_vm::collect_to_tenured() update_code_heap_for_minor_gc(&code->points_to_aging); nursery.here = nursery.start; - reset_generation(data->aging); + data->reset_generation(data->aging); code->points_to_nursery.clear(); code->points_to_aging.clear(); } diff --git a/vm/vm.hpp b/vm/vm.hpp index 87697d5a0f..2df66e97b3 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -220,13 +220,8 @@ struct factor_vm //data heap void init_card_decks(); - void clear_cards(old_space *gen); - void clear_decks(old_space *gen); - void reset_generation(old_space *gen); void set_data_heap(data_heap *data_); void init_data_heap(cell young_size, cell aging_size, cell tenured_size, bool secure_gc_); - cell untagged_object_size(object *pointer); - cell unaligned_object_size(object *pointer); void primitive_size(); cell binary_payload_start(object *pointer); void primitive_data_room(); From e482940dca57bc2f701168928ed5e527541efe2f Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Tue, 20 Oct 2009 13:47:04 -0500 Subject: [PATCH 09/74] vm: object start recording in cards is now independent of allocation strategy --- vm/aging_space.hpp | 16 ++++++++++++++-- vm/code_heap.hpp | 5 ----- vm/copying_collector.hpp | 2 +- vm/data_heap.hpp | 2 +- vm/heap.hpp | 13 +++++++++---- vm/image.cpp | 2 +- vm/layouts.hpp | 11 ++++++++++- vm/mark_bits.hpp | 7 ++++++- vm/old_space.cpp | 35 +++++++++-------------------------- vm/old_space.hpp | 9 ++++----- vm/tenured_space.hpp | 29 +++++++++++++++++++++++++++-- vm/zone.hpp | 9 +++++++++ 12 files changed, 91 insertions(+), 49 deletions(-) diff --git a/vm/aging_space.hpp b/vm/aging_space.hpp index c2ec2a645e..1fac4605d2 100644 --- a/vm/aging_space.hpp +++ b/vm/aging_space.hpp @@ -1,8 +1,20 @@ namespace factor { -struct aging_space : old_space { - aging_space(cell size, cell start) : old_space(size,start) {} +struct aging_space : zone { + object_start_map starts; + + aging_space(cell size, cell start) : + zone(size,start), starts(size,start) {} + + object *allot(cell size) + { + if(here + size > end) return NULL; + + object *obj = zone::allot(size); + starts.record_object_start_offset(obj); + return obj; + } }; } diff --git a/vm/code_heap.hpp b/vm/code_heap.hpp index e98d966afe..b29e33ca64 100755 --- a/vm/code_heap.hpp +++ b/vm/code_heap.hpp @@ -6,11 +6,6 @@ struct code_heap_layout { { return block->size(); } - - heap_block *next_block_after(heap_block *block) - { - return (heap_block *)((cell)block + block_size(block)); - } }; struct code_heap : heap { diff --git a/vm/copying_collector.hpp b/vm/copying_collector.hpp index dd36b680a6..4bb56945a1 100644 --- a/vm/copying_collector.hpp +++ b/vm/copying_collector.hpp @@ -95,7 +95,7 @@ struct copying_collector : collector { if(end < card_start_address(card_index)) { - start = gen->find_object_containing_card(card_index - gen_start_card); + start = gen->starts.find_object_containing_card(card_index - gen_start_card); binary_start = start + this->parent->binary_payload_start((object *)start); end = start + ((object *)start)->size(); } diff --git a/vm/data_heap.hpp b/vm/data_heap.hpp index c882262732..3a0af1f36a 100755 --- a/vm/data_heap.hpp +++ b/vm/data_heap.hpp @@ -52,7 +52,7 @@ template void data_heap::reset_generation(Generation *gen) clear_cards(gen); clear_decks(gen); - gen->clear_object_start_offsets(); + gen->starts.clear_object_start_offsets(); } } diff --git a/vm/heap.hpp b/vm/heap.hpp index 653ac2d93d..c82b88ee2f 100644 --- a/vm/heap.hpp +++ b/vm/heap.hpp @@ -28,6 +28,11 @@ template struct heap { return (Block *)seg->end; } + Block *next_block_after(heap_block *block) + { + return (Block *)((cell)block + layout.block_size(block)); + } + void clear_free_list(); void add_to_free_list(free_heap_block *block); void build_free_list(cell size); @@ -50,7 +55,7 @@ template struct heap { while(scan != end) { - Block *next = layout.next_block_after(scan); + Block *next = next_block_after(scan); if(!scan->free_p()) iter(scan,layout.block_size(scan)); scan = next; } @@ -229,7 +234,7 @@ void heap::heap_usage(cell *used, cell *total_free, cell *max_ else *used += size; - scan = layout.next_block_after(scan); + scan = next_block_after(scan); } } @@ -243,7 +248,7 @@ cell heap::heap_size() while(scan != end) { if(scan->free_p()) break; - else scan = layout.next_block_after(scan); + else scan = next_block_after(scan); } if(scan != end) @@ -308,7 +313,7 @@ void heap::sweep_heap(Iterator &iter) } } - scan = layout.next_block_after(scan); + scan = next_block_after(scan); } if(prev && prev->free_p()) diff --git a/vm/image.cpp b/vm/image.cpp index f41ae555a3..7d8c4a2d32 100755 --- a/vm/image.cpp +++ b/vm/image.cpp @@ -203,7 +203,7 @@ void factor_vm::relocate_data(cell data_relocation_base, cell code_relocation_ba while(obj) { relocate_object((object *)obj,data_relocation_base,code_relocation_base); - data->tenured->record_object_start_offset((object *)obj); + data->tenured->starts.record_object_start_offset((object *)obj); obj = data->tenured->next_object_after(obj); } } diff --git a/vm/layouts.hpp b/vm/layouts.hpp index f0dbff6c30..ca51fd6dca 100644 --- a/vm/layouts.hpp +++ b/vm/layouts.hpp @@ -148,8 +148,17 @@ struct header { struct object { NO_TYPE_CHECK; header h; - cell *slots() { return (cell *)this; } + cell size(); + + cell *slots() { return (cell *)this; } + + /* Only valid for objects in tenured space; must fast to free_heap_block + to do anything with it if its free */ + bool free_p() + { + return h.value & 1 == 1; + } }; /* Assembly code makes assumptions about the layout of this struct */ diff --git a/vm/mark_bits.hpp b/vm/mark_bits.hpp index d9c9534edb..bc318524b4 100644 --- a/vm/mark_bits.hpp +++ b/vm/mark_bits.hpp @@ -70,10 +70,15 @@ template struct mark_bits { return (bits[pair.first] & ((u64)1 << pair.second)) != 0; } + Block *next_block_after(Block *block) + { + return (Block *)((cell)block + layout.block_size(block)); + } + void set_bitmap_range(u64 *bits, Block *address) { std::pair start = bitmap_deref(address); - std::pair end = bitmap_deref(layout.next_block_after(address)); + std::pair end = bitmap_deref(next_block_after(address)); u64 start_mask = ((u64)1 << start.second) - 1; u64 end_mask = ((u64)1 << end.second) - 1; diff --git a/vm/old_space.cpp b/vm/old_space.cpp index 9440749e52..5f992e783e 100644 --- a/vm/old_space.cpp +++ b/vm/old_space.cpp @@ -3,23 +3,24 @@ namespace factor { -old_space::old_space(cell size_, cell start_) : zone(size_,start_) +object_start_map::object_start_map(cell size_, cell start_) : + size(size_), start(start_) { object_start_offsets = new card[addr_to_card(size_)]; object_start_offsets_end = object_start_offsets + addr_to_card(size_); } -old_space::~old_space() +object_start_map::~object_start_map() { delete[] object_start_offsets; } -cell old_space::first_object_in_card(cell card_index) +cell object_start_map::first_object_in_card(cell card_index) { return object_start_offsets[card_index]; } -cell old_space::find_object_containing_card(cell card_index) +cell object_start_map::find_object_containing_card(cell card_index) { if(card_index == 0) return start; @@ -41,34 +42,16 @@ cell old_space::find_object_containing_card(cell card_index) } /* we need to remember the first object allocated in the card */ -void old_space::record_object_start_offset(object *obj) +void object_start_map::record_object_start_offset(object *obj) { cell idx = addr_to_card((cell)obj - start); - if(object_start_offsets[idx] == card_starts_inside_object) - object_start_offsets[idx] = ((cell)obj & addr_card_mask); + card obj_start = ((cell)obj & addr_card_mask); + object_start_offsets[idx] = std::min(object_start_offsets[idx],obj_start); } -object *old_space::allot(cell size) -{ - if(here + size > end) return NULL; - - object *obj = zone::allot(size); - record_object_start_offset(obj); - return obj; -} - -void old_space::clear_object_start_offsets() +void object_start_map::clear_object_start_offsets() { memset(object_start_offsets,card_starts_inside_object,addr_to_card(size)); } -cell old_space::next_object_after(cell scan) -{ - cell size = ((object *)scan)->size(); - if(scan + size < here) - return scan + size; - else - return 0; -} - } diff --git a/vm/old_space.hpp b/vm/old_space.hpp index e59537cffb..640e205852 100644 --- a/vm/old_space.hpp +++ b/vm/old_space.hpp @@ -3,19 +3,18 @@ namespace factor static const cell card_starts_inside_object = 0xff; -struct old_space : zone { +struct object_start_map { + cell size, start; card *object_start_offsets; card *object_start_offsets_end; - old_space(cell size_, cell start_); - ~old_space(); + object_start_map(cell size_, cell start_); + ~object_start_map(); cell first_object_in_card(cell card_index); cell find_object_containing_card(cell card_index); void record_object_start_offset(object *obj); - object *allot(cell size); void clear_object_start_offsets(); - cell next_object_after(cell scan); }; } diff --git a/vm/tenured_space.hpp b/vm/tenured_space.hpp index f9f584b200..a700b58bfd 100644 --- a/vm/tenured_space.hpp +++ b/vm/tenured_space.hpp @@ -1,8 +1,33 @@ namespace factor { -struct tenured_space : old_space { - tenured_space(cell size, cell start) : old_space(size,start) {} +struct tenured_space_layout { + cell block_size(object *block) + { + if(block->free_p()) + { + free_heap_block *free_block = (free_heap_block *)block; + return free_block->size(); + } + else + return block->size(); + } +}; + +struct tenured_space : zone { + object_start_map starts; + + tenured_space(cell size, cell start) : + zone(size,start), starts(size,start) {} + + object *allot(cell size) + { + if(here + size > end) return NULL; + + object *obj = zone::allot(size); + starts.record_object_start_offset(obj); + return obj; + } }; } diff --git a/vm/zone.hpp b/vm/zone.hpp index 2c28922fbc..42196f315b 100644 --- a/vm/zone.hpp +++ b/vm/zone.hpp @@ -21,6 +21,15 @@ struct zone { here = h + align(size,data_alignment); return (object *)h; } + + cell next_object_after(cell scan) + { + cell size = ((object *)scan)->size(); + if(scan + size < here) + return scan + size; + else + return 0; + } }; } From a9dbbd1efb5c8b3cbda11f6511457b4f4a5bcca7 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Tue, 20 Oct 2009 14:01:46 -0500 Subject: [PATCH 10/74] vm: simplify code heap by eliminating HeapLayout template parameter --- vm/code_heap.cpp | 3 +- vm/code_heap.hpp | 9 +---- vm/copying_collector.hpp | 4 +- vm/data_heap.cpp | 2 + vm/full_collector.cpp | 2 +- vm/heap.hpp | 82 ++++++++++++++++------------------------ vm/image.cpp | 2 +- vm/mark_bits.hpp | 11 +++--- vm/tenured_space.hpp | 13 ------- vm/zone.hpp | 2 +- 10 files changed, 47 insertions(+), 83 deletions(-) diff --git a/vm/code_heap.cpp b/vm/code_heap.cpp index df557074af..c65dec9a69 100755 --- a/vm/code_heap.cpp +++ b/vm/code_heap.cpp @@ -3,8 +3,7 @@ namespace factor { -code_heap::code_heap(bool secure_gc, cell size) : - heap(secure_gc,size,true) {} +code_heap::code_heap(bool secure_gc, cell size) : heap(secure_gc,size,true) {} void code_heap::write_barrier(code_block *compiled) { diff --git a/vm/code_heap.hpp b/vm/code_heap.hpp index b29e33ca64..d4c5d4e40b 100755 --- a/vm/code_heap.hpp +++ b/vm/code_heap.hpp @@ -1,14 +1,7 @@ namespace factor { -struct code_heap_layout { - cell block_size(heap_block *block) - { - return block->size(); - } -}; - -struct code_heap : heap { +struct code_heap : heap { /* Set of blocks which need full relocation. */ std::set needs_fixup; diff --git a/vm/copying_collector.hpp b/vm/copying_collector.hpp index 4bb56945a1..ea7faf2423 100644 --- a/vm/copying_collector.hpp +++ b/vm/copying_collector.hpp @@ -113,7 +113,7 @@ scan_next_object: { card_end_address(card_index)); if(end < card_end_address(card_index)) { - start = gen->next_object_after(start); + start = gen->next_allocated_block_after(start); if(start) { binary_start = start + this->parent->binary_payload_start((object *)start); @@ -158,7 +158,7 @@ end: this->parent->gc_stats.card_scan_time += (current_micros() - start_time); while(scan && scan < this->target->here) { this->trace_slots((object *)scan); - scan = this->target->next_object_after(scan); + scan = this->target->next_allocated_block_after(scan); } } }; diff --git a/vm/data_heap.cpp b/vm/data_heap.cpp index 05f8f3a044..d3951b2b27 100755 --- a/vm/data_heap.cpp +++ b/vm/data_heap.cpp @@ -93,6 +93,8 @@ cell factor_vm::object_size(cell tagged) /* Size of the object pointed to by an untagged pointer */ cell object::size() { + if(free_p()) return ((free_heap_block *)this)->size(); + switch(h.hi_tag()) { case ARRAY_TYPE: diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index 1bb03d08e7..9c8a825c1f 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -100,7 +100,7 @@ void full_collector::cheneys_algorithm() object *obj = (object *)scan; this->trace_slots(obj); this->mark_object_code_block(obj); - scan = target->next_object_after(scan); + scan = target->next_allocated_block_after(scan); } } diff --git a/vm/heap.hpp b/vm/heap.hpp index c82b88ee2f..c06eab8dc7 100644 --- a/vm/heap.hpp +++ b/vm/heap.hpp @@ -8,12 +8,11 @@ struct heap_free_list { free_heap_block *large_blocks; }; -template struct heap { +template struct heap { bool secure_gc; segment *seg; heap_free_list free; - mark_bits *state; - HeapLayout layout; + mark_bits *state; explicit heap(bool secure_gc_, cell size, bool executable_p); ~heap(); @@ -30,7 +29,7 @@ template struct heap { Block *next_block_after(heap_block *block) { - return (Block *)((cell)block + layout.block_size(block)); + return (Block *)((cell)block + block->size()); } void clear_free_list(); @@ -55,31 +54,29 @@ template struct heap { while(scan != end) { - Block *next = next_block_after(scan); - if(!scan->free_p()) iter(scan,layout.block_size(scan)); + cell size = scan->size(); + Block *next = (Block *)((cell)scan + size); + if(!scan->free_p()) iter(scan,size); scan = next; } } }; -template -void heap::clear_free_list() +template void heap::clear_free_list() { memset(&free,0,sizeof(heap_free_list)); } -template -heap::heap(bool secure_gc_, cell size, bool executable_p) : secure_gc(secure_gc_) +template heap::heap(bool secure_gc_, cell size, bool executable_p) : secure_gc(secure_gc_) { if(size > (1L << (sizeof(cell) * 8 - 6))) fatal_error("Heap too large",size); seg = new segment(align_page(size),executable_p); if(!seg) fatal_error("Out of memory in heap allocator",size); - state = new mark_bits(seg->start,size); + state = new mark_bits(seg->start,size); clear_free_list(); } -template -heap::~heap() +template heap::~heap() { delete seg; seg = NULL; @@ -87,8 +84,7 @@ heap::~heap() state = NULL; } -template -void heap::add_to_free_list(free_heap_block *block) +template void heap::add_to_free_list(free_heap_block *block) { if(block->size() < free_list_count * block_granularity) { @@ -105,8 +101,7 @@ void heap::add_to_free_list(free_heap_block *block) /* Called after reading the code heap from the image file, and after code heap compaction. Makes a free list consisting of one free block, at the very end. */ -template -void heap::build_free_list(cell size) +template void heap::build_free_list(cell size) { clear_free_list(); free_heap_block *end = (free_heap_block *)(seg->start + size); @@ -115,16 +110,14 @@ void heap::build_free_list(cell size) add_to_free_list(end); } -template -void heap::assert_free_block(free_heap_block *block) +template void heap::assert_free_block(free_heap_block *block) { #ifdef FACTOR_DEBUG assert(block->free_p()); #endif } -template -free_heap_block *heap::find_free_block(cell size) +template free_heap_block *heap::find_free_block(cell size) { cell attempt = size; @@ -164,8 +157,7 @@ free_heap_block *heap::find_free_block(cell size) return NULL; } -template -free_heap_block *heap::split_free_block(free_heap_block *block, cell size) +template free_heap_block *heap::split_free_block(free_heap_block *block, cell size) { if(block->size() != size) { @@ -181,8 +173,7 @@ free_heap_block *heap::split_free_block(free_heap_block *block return block; } -template -Block *heap::heap_allot(cell size) +template Block *heap::heap_allot(cell size) { size = align(size,block_granularity); @@ -196,23 +187,20 @@ Block *heap::heap_allot(cell size) return NULL; } -template -void heap::heap_free(Block *block) +template void heap::heap_free(Block *block) { free_heap_block *free_block = (free_heap_block *)block; free_block->set_free(); add_to_free_list(free_block); } -template -void heap::mark_block(Block *block) +template void heap::mark_block(Block *block) { state->set_marked_p(block); } /* Compute total sum of sizes of free blocks, and size of largest free block */ -template -void heap::heap_usage(cell *used, cell *total_free, cell *max_free) +template void heap::heap_usage(cell *used, cell *total_free, cell *max_free) { *used = 0; *total_free = 0; @@ -223,7 +211,7 @@ void heap::heap_usage(cell *used, cell *total_free, cell *max_ while(scan != end) { - cell size = layout.block_size(scan); + cell size = scan->size(); if(scan->free_p()) { @@ -239,8 +227,7 @@ void heap::heap_usage(cell *used, cell *total_free, cell *max_ } /* The size of the heap after compaction */ -template -cell heap::heap_size() +template cell heap::heap_size() { Block *scan = first_block(); Block *end = last_block(); @@ -255,7 +242,7 @@ cell heap::heap_size() { free_heap_block *free_block = (free_heap_block *)scan; assert(free_block->free_p()); - assert((cell)scan + scan->size() == seg->end); + assert((cell)scan + free_block->size() == seg->end); return (cell)scan - (cell)first_block(); } @@ -265,9 +252,9 @@ cell heap::heap_size() /* After code GC, all live code blocks are marked, so any which are not marked can be reclaimed. */ -template +template template -void heap::sweep_heap(Iterator &iter) +void heap::sweep_heap(Iterator &iter) { this->clear_free_list(); @@ -277,14 +264,14 @@ void heap::sweep_heap(Iterator &iter) while(scan != end) { + cell size = scan->size(); + if(scan->free_p()) { - free_heap_block *free_scan = (free_heap_block *)scan; - if(prev && prev->free_p()) { free_heap_block *free_prev = (free_heap_block *)prev; - free_prev->set_size(free_prev->size() + free_scan->size()); + free_prev->set_size(free_prev->size() + size); } else prev = scan; @@ -294,17 +281,14 @@ void heap::sweep_heap(Iterator &iter) if(prev && prev->free_p()) this->add_to_free_list((free_heap_block *)prev); prev = scan; - iter(scan,layout.block_size(scan)); + iter(scan,size); } else { - if(secure_gc) - memset(scan + 1,0,layout.block_size(scan) - sizeof(heap_block)); - if(prev && prev->free_p()) { free_heap_block *free_prev = (free_heap_block *)prev; - free_prev->set_size(free_prev->size() + layout.block_size(scan)); + free_prev->set_size(free_prev->size() + size); } else { @@ -313,7 +297,7 @@ void heap::sweep_heap(Iterator &iter) } } - scan = next_block_after(scan); + scan = (Block *)((cell)scan + size); } if(prev && prev->free_p()) @@ -322,11 +306,11 @@ void heap::sweep_heap(Iterator &iter) /* The forwarding map must be computed first by calling state->compute_forwarding(). */ -template +template template -void heap::compact_heap(Iterator &iter) +void heap::compact_heap(Iterator &iter) { - heap_compactor compactor(state,first_block(),iter); + heap_compactor compactor(state,first_block(),iter); this->iterate_heap(compactor); /* Now update the free list; there will be a single free block at diff --git a/vm/image.cpp b/vm/image.cpp index 7d8c4a2d32..d45e09a600 100755 --- a/vm/image.cpp +++ b/vm/image.cpp @@ -204,7 +204,7 @@ void factor_vm::relocate_data(cell data_relocation_base, cell code_relocation_ba { relocate_object((object *)obj,data_relocation_base,code_relocation_base); data->tenured->starts.record_object_start_offset((object *)obj); - obj = data->tenured->next_object_after(obj); + obj = data->tenured->next_allocated_block_after(obj); } } diff --git a/vm/mark_bits.hpp b/vm/mark_bits.hpp index bc318524b4..0f14622fcf 100644 --- a/vm/mark_bits.hpp +++ b/vm/mark_bits.hpp @@ -4,8 +4,7 @@ namespace factor const int block_granularity = 16; const int forwarding_granularity = 64; -template struct mark_bits { - HeapLayout layout; +template struct mark_bits { cell start; cell size; cell bits_size; @@ -72,7 +71,7 @@ template struct mark_bits { Block *next_block_after(Block *block) { - return (Block *)((cell)block + layout.block_size(block)); + return (Block *)((cell)block + block->size()); } void set_bitmap_range(u64 *bits, Block *address) @@ -146,12 +145,12 @@ template struct mark_bits { } }; -template struct heap_compactor { - mark_bits *state; +template struct heap_compactor { + mark_bits *state; char *address; Iterator &iter; - explicit heap_compactor(mark_bits *state_, Block *address_, Iterator &iter_) : + explicit heap_compactor(mark_bits *state_, Block *address_, Iterator &iter_) : state(state_), address((char *)address_), iter(iter_) {} void operator()(Block *block, cell size) diff --git a/vm/tenured_space.hpp b/vm/tenured_space.hpp index a700b58bfd..b36d96c4aa 100644 --- a/vm/tenured_space.hpp +++ b/vm/tenured_space.hpp @@ -1,19 +1,6 @@ namespace factor { -struct tenured_space_layout { - cell block_size(object *block) - { - if(block->free_p()) - { - free_heap_block *free_block = (free_heap_block *)block; - return free_block->size(); - } - else - return block->size(); - } -}; - struct tenured_space : zone { object_start_map starts; diff --git a/vm/zone.hpp b/vm/zone.hpp index 42196f315b..55f34fe67e 100644 --- a/vm/zone.hpp +++ b/vm/zone.hpp @@ -22,7 +22,7 @@ struct zone { return (object *)h; } - cell next_object_after(cell scan) + cell next_allocated_block_after(cell scan) { cell size = ((object *)scan)->size(); if(scan + size < here) From f0816d72f1748d6a6dff470d4bf7f45a18d9f6ac Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Tue, 20 Oct 2009 15:15:05 -0500 Subject: [PATCH 11/74] vm: split off free_list_allocator from heap class, rename zone to bump_allocator --- Makefile | 2 +- vm/aging_collector.hpp | 3 +- vm/aging_space.hpp | 6 +- vm/{zone.hpp => bump_allocator.hpp} | 5 +- vm/code_block.cpp | 6 +- vm/code_heap.cpp | 47 ++++++-- vm/code_heap.hpp | 14 ++- vm/data_heap.cpp | 5 +- vm/data_heap.hpp | 2 +- vm/debug.cpp | 6 +- vm/factor.cpp | 2 - vm/{heap.hpp => free_list_allocator.hpp} | 130 +++++++++------------ vm/full_collector.cpp | 4 +- vm/full_collector.hpp | 2 +- vm/image.cpp | 11 +- vm/image.hpp | 1 - vm/mark_bits.hpp | 4 +- vm/master.hpp | 8 +- vm/{old_space.cpp => object_start_map.cpp} | 0 vm/{old_space.hpp => object_start_map.hpp} | 0 vm/tenured_space.hpp | 6 +- vm/to_tenured_collector.hpp | 2 +- vm/vm.cpp | 1 - vm/vm.hpp | 11 +- 24 files changed, 147 insertions(+), 131 deletions(-) rename vm/{zone.hpp => bump_allocator.hpp} (79%) rename vm/{heap.hpp => free_list_allocator.hpp} (56%) rename vm/{old_space.cpp => object_start_map.cpp} (100%) rename vm/{old_space.hpp => object_start_map.hpp} (100%) diff --git a/Makefile b/Makefile index 5a44333d42..78f59a38bb 100755 --- a/Makefile +++ b/Makefile @@ -55,7 +55,7 @@ DLL_OBJS = $(PLAF_DLL_OBJS) \ vm/jit.o \ vm/math.o \ vm/nursery_collector.o \ - vm/old_space.o \ + vm/object_start_map.o \ vm/primitives.o \ vm/profiler.o \ vm/quotations.o \ diff --git a/vm/aging_collector.hpp b/vm/aging_collector.hpp index 1fa82972ff..8f2677c8b6 100644 --- a/vm/aging_collector.hpp +++ b/vm/aging_collector.hpp @@ -3,7 +3,8 @@ namespace factor struct aging_policy { factor_vm *parent; - zone *aging, *tenured; + aging_space *aging; + tenured_space *tenured; aging_policy(factor_vm *parent_) : parent(parent_), diff --git a/vm/aging_space.hpp b/vm/aging_space.hpp index 1fac4605d2..20e2506539 100644 --- a/vm/aging_space.hpp +++ b/vm/aging_space.hpp @@ -1,17 +1,17 @@ namespace factor { -struct aging_space : zone { +struct aging_space : bump_allocator { object_start_map starts; aging_space(cell size, cell start) : - zone(size,start), starts(size,start) {} + bump_allocator(size,start), starts(size,start) {} object *allot(cell size) { if(here + size > end) return NULL; - object *obj = zone::allot(size); + object *obj = bump_allocator::allot(size); starts.record_object_start_offset(obj); return obj; } diff --git a/vm/zone.hpp b/vm/bump_allocator.hpp similarity index 79% rename from vm/zone.hpp rename to vm/bump_allocator.hpp index 55f34fe67e..64011e0bb6 100644 --- a/vm/zone.hpp +++ b/vm/bump_allocator.hpp @@ -1,14 +1,15 @@ namespace factor { -struct zone { +struct bump_allocator { /* offset of 'here' and 'end' is hardcoded in compiler backends */ cell here; cell start; cell end; cell size; - zone(cell size_, cell start_) : here(0), start(start_), end(start_ + size_), size(size_) {} + bump_allocator(cell size_, cell start_) : + here(0), start(start_), end(start_ + size_), size(size_) {} inline bool contains_p(object *pointer) { diff --git a/vm/code_block.cpp b/vm/code_block.cpp index e29d708c25..1c15f23382 100755 --- a/vm/code_block.cpp +++ b/vm/code_block.cpp @@ -439,7 +439,7 @@ void factor_vm::fixup_labels(array *labels, code_block *compiled) /* Might GC */ code_block *factor_vm::allot_code_block(cell size, code_block_type type) { - heap_block *block = code->heap_allot(size + sizeof(code_block)); + heap_block *block = code->allocator->allot(size + sizeof(code_block)); /* If allocation failed, do a full GC and compact the code heap. A full GC that occurs as a result of the data heap filling up does not @@ -449,13 +449,13 @@ code_block *factor_vm::allot_code_block(cell size, code_block_type type) if(block == NULL) { primitive_compact_gc(); - block = code->heap_allot(size + sizeof(code_block)); + block = code->allocator->allot(size + sizeof(code_block)); /* Insufficient room even after code GC, give up */ if(block == NULL) { cell used, total_free, max_free; - code->heap_usage(&used,&total_free,&max_free); + code->allocator->usage(&used,&total_free,&max_free); print_string("Code heap stats:\n"); print_string("Used: "); print_cell(used); nl(); diff --git a/vm/code_heap.cpp b/vm/code_heap.cpp index c65dec9a69..2a466857f9 100755 --- a/vm/code_heap.cpp +++ b/vm/code_heap.cpp @@ -3,7 +3,21 @@ namespace factor { -code_heap::code_heap(bool secure_gc, cell size) : heap(secure_gc,size,true) {} +code_heap::code_heap(cell size) +{ + if(size > (1L << (sizeof(cell) * 8 - 6))) fatal_error("Heap too large",size); + seg = new segment(align_page(size),true); + if(!seg) fatal_error("Out of memory in heap allocator",size); + allocator = new free_list_allocator(seg->start,size); +} + +code_heap::~code_heap() +{ + delete allocator; + allocator = NULL; + delete seg; + seg = NULL; +} void code_heap::write_barrier(code_block *compiled) { @@ -22,18 +36,33 @@ bool code_heap::needs_fixup_p(code_block *compiled) return needs_fixup.count(compiled) > 0; } +bool code_heap::marked_p(heap_block *compiled) +{ + return allocator->state.marked_p(compiled); +} + +void code_heap::set_marked_p(code_block *compiled) +{ + allocator->state.set_marked_p(compiled); +} + +void code_heap::clear_mark_bits() +{ + allocator->state.clear_mark_bits(); +} + void code_heap::code_heap_free(code_block *compiled) { points_to_nursery.erase(compiled); points_to_aging.erase(compiled); needs_fixup.erase(compiled); - heap_free(compiled); + allocator->free(compiled); } /* Allocate a code heap during startup */ void factor_vm::init_code_heap(cell size) { - code = new code_heap(secure_gc,size); + code = new code_heap(size); } bool factor_vm::in_code_heap_p(cell ptr) @@ -89,7 +118,7 @@ struct word_and_literal_code_heap_updater { void factor_vm::update_code_heap_words_and_literals() { word_and_literal_code_heap_updater updater(this); - code->sweep_heap(updater); + code->allocator->sweep(updater); } /* After growing the heap, we have to perform a full relocation to update @@ -109,7 +138,7 @@ void factor_vm::relocate_code_heap() { code_heap_relocator relocator(this); code_heap_iterator iter(relocator); - code->sweep_heap(iter); + code->allocator->sweep(iter); } void factor_vm::primitive_modify_code_heap() @@ -169,7 +198,7 @@ void factor_vm::primitive_modify_code_heap() void factor_vm::primitive_code_room() { cell used, total_free, max_free; - code->heap_usage(&used,&total_free,&max_free); + code->allocator->usage(&used,&total_free,&max_free); dpush(tag_fixnum(code->seg->size / 1024)); dpush(tag_fixnum(used / 1024)); dpush(tag_fixnum(total_free / 1024)); @@ -178,7 +207,7 @@ void factor_vm::primitive_code_room() code_block *code_heap::forward_code_block(code_block *compiled) { - return (code_block *)state->forward_block(compiled); + return (code_block *)allocator->state.forward_block(compiled); } struct callframe_forwarder { @@ -277,7 +306,7 @@ function returns. */ void factor_vm::compact_code_heap(bool trace_contexts_p) { /* Figure out where blocks are going to go */ - code->state->compute_forwarding(); + code->allocator->state.compute_forwarding(); /* Update references to the code heap from the data heap */ forward_object_xts(); @@ -291,7 +320,7 @@ void factor_vm::compact_code_heap(bool trace_contexts_p) that the data heap is up to date since relocation looks up object XTs) */ code_heap_relocator relocator(this); code_heap_iterator iter(relocator); - code->compact_heap(iter); + code->allocator->compact(iter); } struct stack_trace_stripper { diff --git a/vm/code_heap.hpp b/vm/code_heap.hpp index d4c5d4e40b..2d9961c03a 100755 --- a/vm/code_heap.hpp +++ b/vm/code_heap.hpp @@ -1,7 +1,13 @@ namespace factor { -struct code_heap : heap { +struct code_heap { + /* The actual memory area */ + segment *seg; + + /* Memory allocator */ + free_list_allocator *allocator; + /* Set of blocks which need full relocation. */ std::set needs_fixup; @@ -11,10 +17,14 @@ struct code_heap : heap { /* Code blocks which may reference objects in aging space or the nursery */ std::set points_to_aging; - explicit code_heap(bool secure_gc, cell size); + explicit code_heap(cell size); + ~code_heap(); void write_barrier(code_block *compiled); void clear_remembered_set(); bool needs_fixup_p(code_block *compiled); + bool marked_p(heap_block *compiled); + void set_marked_p(code_block *compiled); + void clear_mark_bits(); void code_heap_free(code_block *compiled); code_block *forward_code_block(code_block *compiled); }; diff --git a/vm/data_heap.cpp b/vm/data_heap.cpp index d3951b2b27..5a8f4d6b36 100755 --- a/vm/data_heap.cpp +++ b/vm/data_heap.cpp @@ -42,7 +42,7 @@ data_heap::data_heap(cell young_size_, cell aging_size_, cell tenured_size_) aging = new aging_space(aging_size,tenured_semispace->end); aging_semispace = new aging_space(aging_size,aging->end); - nursery = new zone(young_size,aging_semispace->end); + nursery = new bump_allocator(young_size,aging_semispace->end); assert(seg->end - nursery->end <= deck_size); } @@ -75,10 +75,9 @@ void factor_vm::set_data_heap(data_heap *data_) data->reset_generation(data->tenured); } -void factor_vm::init_data_heap(cell young_size, cell aging_size, cell tenured_size, bool secure_gc_) +void factor_vm::init_data_heap(cell young_size, cell aging_size, cell tenured_size) { set_data_heap(new data_heap(young_size,aging_size,tenured_size)); - secure_gc = secure_gc_; } /* Size of the object pointed to by a tagged pointer */ diff --git a/vm/data_heap.hpp b/vm/data_heap.hpp index 3a0af1f36a..526a5eb9ae 100755 --- a/vm/data_heap.hpp +++ b/vm/data_heap.hpp @@ -10,7 +10,7 @@ struct data_heap { segment *seg; - zone *nursery; + bump_allocator *nursery; aging_space *aging; aging_space *aging_semispace; tenured_space *tenured; diff --git a/vm/debug.cpp b/vm/debug.cpp index ddf4877eab..31164e972b 100755 --- a/vm/debug.cpp +++ b/vm/debug.cpp @@ -209,7 +209,7 @@ void factor_vm::dump_memory(cell from, cell to) dump_cell(from); } -void factor_vm::dump_zone(const char *name, zone *z) +void factor_vm::dump_zone(const char *name, bump_allocator *z) { print_string(name); print_string(": "); print_string("Start="); print_cell(z->start); @@ -296,7 +296,7 @@ struct code_block_printer { const char *status; if(scan->free_p()) status = "free"; - else if(parent->code->state->is_marked_p(scan)) + else if(parent->code->marked_p(scan)) { reloc_size += parent->object_size(((code_block *)scan)->relocation); literal_size += parent->object_size(((code_block *)scan)->literals); @@ -319,7 +319,7 @@ struct code_block_printer { void factor_vm::dump_code_heap() { code_block_printer printer(this); - code->iterate_heap(printer); + code->allocator->iterate(printer); print_cell(printer.reloc_size); print_string(" bytes of relocation data\n"); print_cell(printer.literal_size); print_string(" bytes of literal data\n"); } diff --git a/vm/factor.cpp b/vm/factor.cpp index f2b0d4c92a..9c87c0a9a7 100755 --- a/vm/factor.cpp +++ b/vm/factor.cpp @@ -37,7 +37,6 @@ void factor_vm::default_parameters(vm_parameters *p) p->max_pic_size = 3; - p->secure_gc = false; p->fep = false; p->signals = true; @@ -85,7 +84,6 @@ void factor_vm::init_parameters_from_args(vm_parameters *p, int argc, vm_char ** else if(factor_arg(arg,STRING_LITERAL("-codeheap=%d"),&p->code_size)); else if(factor_arg(arg,STRING_LITERAL("-pic=%d"),&p->max_pic_size)); else if(factor_arg(arg,STRING_LITERAL("-callbacks=%d"),&p->callback_size)); - else if(STRCMP(arg,STRING_LITERAL("-securegc")) == 0) p->secure_gc = true; else if(STRCMP(arg,STRING_LITERAL("-fep")) == 0) p->fep = true; else if(STRCMP(arg,STRING_LITERAL("-nosignals")) == 0) p->signals = false; else if(STRNCMP(arg,STRING_LITERAL("-i="),3) == 0) p->image_path = arg + 3; diff --git a/vm/heap.hpp b/vm/free_list_allocator.hpp similarity index 56% rename from vm/heap.hpp rename to vm/free_list_allocator.hpp index c06eab8dc7..da2622bb0b 100644 --- a/vm/heap.hpp +++ b/vm/free_list_allocator.hpp @@ -3,28 +3,28 @@ namespace factor static const cell free_list_count = 32; -struct heap_free_list { +struct free_list { free_heap_block *small_blocks[free_list_count]; free_heap_block *large_blocks; }; -template struct heap { - bool secure_gc; - segment *seg; - heap_free_list free; - mark_bits *state; +template struct free_list_allocator { + cell start; + cell size; + cell end; + free_list free_blocks; + mark_bits state; - explicit heap(bool secure_gc_, cell size, bool executable_p); - ~heap(); + explicit free_list_allocator(cell start, cell size); inline Block *first_block() { - return (Block *)seg->start; + return (Block *)start; } inline Block *last_block() { - return (Block *)seg->end; + return (Block *)end; } Block *next_block_after(heap_block *block) @@ -38,16 +38,15 @@ template struct heap { void assert_free_block(free_heap_block *block); free_heap_block *find_free_block(cell size); free_heap_block *split_free_block(free_heap_block *block, cell size); - Block *heap_allot(cell size); - void heap_free(Block *block); - void mark_block(Block *block); - void heap_usage(cell *used, cell *total_free, cell *max_free); - cell heap_size(); + Block *allot(cell size); + void free(Block *block); + void usage(cell *used, cell *total_free, cell *max_free); + cell occupied(); - template void sweep_heap(Iterator &iter); - template void compact_heap(Iterator &iter); + template void sweep(Iterator &iter); + template void compact(Iterator &iter); - template void iterate_heap(Iterator &iter) + template void iterate(Iterator &iter) { Block *scan = first_block(); Block *end = last_block(); @@ -62,73 +61,63 @@ template struct heap { } }; -template void heap::clear_free_list() +template void free_list_allocator::clear_free_list() { - memset(&free,0,sizeof(heap_free_list)); + memset(&free_blocks,0,sizeof(free_list)); } -template heap::heap(bool secure_gc_, cell size, bool executable_p) : secure_gc(secure_gc_) +template +free_list_allocator::free_list_allocator(cell start_, cell size_) : + start(start_), size(size_), end(start_ + size_), state(mark_bits(start_,size_)) { - if(size > (1L << (sizeof(cell) * 8 - 6))) fatal_error("Heap too large",size); - seg = new segment(align_page(size),executable_p); - if(!seg) fatal_error("Out of memory in heap allocator",size); - state = new mark_bits(seg->start,size); clear_free_list(); } -template heap::~heap() -{ - delete seg; - seg = NULL; - delete state; - state = NULL; -} - -template void heap::add_to_free_list(free_heap_block *block) +template void free_list_allocator::add_to_free_list(free_heap_block *block) { if(block->size() < free_list_count * block_granularity) { int index = block->size() / block_granularity; - block->next_free = free.small_blocks[index]; - free.small_blocks[index] = block; + block->next_free = free_blocks.small_blocks[index]; + free_blocks.small_blocks[index] = block; } else { - block->next_free = free.large_blocks; - free.large_blocks = block; + block->next_free = free_blocks.large_blocks; + free_blocks.large_blocks = block; } } -/* Called after reading the code heap from the image file, and after code heap -compaction. Makes a free list consisting of one free block, at the very end. */ -template void heap::build_free_list(cell size) +/* Called after reading the heap from the image file, and after heap compaction. +Makes a free list consisting of one free block, at the very end. */ +template void free_list_allocator::build_free_list(cell size) { clear_free_list(); - free_heap_block *end = (free_heap_block *)(seg->start + size); - end->set_free(); - end->set_size(seg->end - (cell)end); - add_to_free_list(end); + free_heap_block *last_block = (free_heap_block *)(start + size); + last_block->set_free(); + last_block->set_size(end - (cell)last_block); + add_to_free_list(last_block); } -template void heap::assert_free_block(free_heap_block *block) +template void free_list_allocator::assert_free_block(free_heap_block *block) { #ifdef FACTOR_DEBUG assert(block->free_p()); #endif } -template free_heap_block *heap::find_free_block(cell size) +template free_heap_block *free_list_allocator::find_free_block(cell size) { cell attempt = size; while(attempt < free_list_count * block_granularity) { int index = attempt / block_granularity; - free_heap_block *block = free.small_blocks[index]; + free_heap_block *block = free_blocks.small_blocks[index]; if(block) { assert_free_block(block); - free.small_blocks[index] = block->next_free; + free_blocks.small_blocks[index] = block->next_free; return block; } @@ -136,7 +125,7 @@ template free_heap_block *heap::find_free_block(cell size } free_heap_block *prev = NULL; - free_heap_block *block = free.large_blocks; + free_heap_block *block = free_blocks.large_blocks; while(block) { @@ -146,7 +135,7 @@ template free_heap_block *heap::find_free_block(cell size if(prev) prev->next_free = block->next_free; else - free.large_blocks = block->next_free; + free_blocks.large_blocks = block->next_free; return block; } @@ -157,7 +146,7 @@ template free_heap_block *heap::find_free_block(cell size return NULL; } -template free_heap_block *heap::split_free_block(free_heap_block *block, cell size) +template free_heap_block *free_list_allocator::split_free_block(free_heap_block *block, cell size) { if(block->size() != size) { @@ -173,7 +162,7 @@ template free_heap_block *heap::split_free_block(free_hea return block; } -template Block *heap::heap_allot(cell size) +template Block *free_list_allocator::allot(cell size) { size = align(size,block_granularity); @@ -187,20 +176,15 @@ template Block *heap::heap_allot(cell size) return NULL; } -template void heap::heap_free(Block *block) +template void free_list_allocator::free(Block *block) { free_heap_block *free_block = (free_heap_block *)block; free_block->set_free(); add_to_free_list(free_block); } -template void heap::mark_block(Block *block) -{ - state->set_marked_p(block); -} - /* Compute total sum of sizes of free blocks, and size of largest free block */ -template void heap::heap_usage(cell *used, cell *total_free, cell *max_free) +template void free_list_allocator::usage(cell *used, cell *total_free, cell *max_free) { *used = 0; *total_free = 0; @@ -227,34 +211,34 @@ template void heap::heap_usage(cell *used, cell *total_fr } /* The size of the heap after compaction */ -template cell heap::heap_size() +template cell free_list_allocator::occupied() { Block *scan = first_block(); - Block *end = last_block(); + Block *last = last_block(); - while(scan != end) + while(scan != last) { if(scan->free_p()) break; else scan = next_block_after(scan); } - if(scan != end) + if(scan != last) { free_heap_block *free_block = (free_heap_block *)scan; assert(free_block->free_p()); - assert((cell)scan + free_block->size() == seg->end); + assert((cell)scan + free_block->size() == end); return (cell)scan - (cell)first_block(); } else - return seg->size; + return size; } /* After code GC, all live code blocks are marked, so any which are not marked can be reclaimed. */ template template -void heap::sweep_heap(Iterator &iter) +void free_list_allocator::sweep(Iterator &iter) { this->clear_free_list(); @@ -276,7 +260,7 @@ void heap::sweep_heap(Iterator &iter) else prev = scan; } - else if(this->state->is_marked_p(scan)) + else if(this->state.marked_p(scan)) { if(prev && prev->free_p()) this->add_to_free_list((free_heap_block *)prev); @@ -305,17 +289,17 @@ void heap::sweep_heap(Iterator &iter) } /* The forwarding map must be computed first by calling -state->compute_forwarding(). */ +state.compute_forwarding(). */ template template -void heap::compact_heap(Iterator &iter) +void free_list_allocator::compact(Iterator &iter) { - heap_compactor compactor(state,first_block(),iter); - this->iterate_heap(compactor); + heap_compactor compactor(&state,first_block(),iter); + this->iterate(compactor); /* Now update the free list; there will be a single free block at the end */ - this->build_free_list((cell)compactor.address - this->seg->start); + this->build_free_list((cell)compactor.address - this->start); } } diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index 9c8a825c1f..924cad6777 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -89,7 +89,7 @@ void full_collector::trace_literal_references(code_block *compiled) collections */ void full_collector::mark_code_block(code_block *compiled) { - this->code->mark_block(compiled); + this->code->set_marked_p(compiled); trace_literal_references(compiled); } @@ -108,7 +108,7 @@ void factor_vm::collect_full_impl(bool trace_contexts_p) { full_collector collector(this); - code->state->clear_mark_bits(); + code->clear_mark_bits(); collector.trace_roots(); if(trace_contexts_p) diff --git a/vm/full_collector.hpp b/vm/full_collector.hpp index 8cc37f782d..5298f56637 100644 --- a/vm/full_collector.hpp +++ b/vm/full_collector.hpp @@ -3,7 +3,7 @@ namespace factor struct full_policy { factor_vm *parent; - zone *tenured; + tenured_space *tenured; full_policy(factor_vm *parent_) : parent(parent_), tenured(parent->data->tenured) {} diff --git a/vm/image.cpp b/vm/image.cpp index d45e09a600..b47fe4e86a 100755 --- a/vm/image.cpp +++ b/vm/image.cpp @@ -23,8 +23,7 @@ void factor_vm::load_data_heap(FILE *file, image_header *h, vm_parameters *p) init_data_heap(p->young_size, p->aging_size, - p->tenured_size, - p->secure_gc); + p->tenured_size); clear_gc_stats(); @@ -52,7 +51,7 @@ void factor_vm::load_code_heap(FILE *file, image_header *h, vm_parameters *p) if(h->code_size != 0) { - size_t bytes_read = fread(code->first_block(),1,h->code_size,file); + size_t bytes_read = fread(code->allocator->first_block(),1,h->code_size,file); if(bytes_read != h->code_size) { print_string("truncated image: "); @@ -64,7 +63,7 @@ void factor_vm::load_code_heap(FILE *file, image_header *h, vm_parameters *p) } } - code->build_free_list(h->code_size); + code->allocator->build_free_list(h->code_size); } void factor_vm::data_fixup(cell *handle, cell data_relocation_base) @@ -292,7 +291,7 @@ bool factor_vm::save_image(const vm_char *filename) h.data_relocation_base = data->tenured->start; h.data_size = data->tenured->here - data->tenured->start; h.code_relocation_base = code->seg->start; - h.code_size = code->heap_size(); + h.code_size = code->allocator->occupied(); h.true_object = true_object; h.bignum_zero = bignum_zero; @@ -306,7 +305,7 @@ bool factor_vm::save_image(const vm_char *filename) if(fwrite(&h,sizeof(image_header),1,file) != 1) ok = false; if(fwrite((void*)data->tenured->start,h.data_size,1,file) != 1) ok = false; - if(fwrite(code->first_block(),h.code_size,1,file) != 1) ok = false; + if(fwrite(code->allocator->first_block(),h.code_size,1,file) != 1) ok = false; if(fclose(file)) ok = false; if(!ok) diff --git a/vm/image.hpp b/vm/image.hpp index 8a7080110c..62ab7e8392 100755 --- a/vm/image.hpp +++ b/vm/image.hpp @@ -34,7 +34,6 @@ struct vm_parameters { cell ds_size, rs_size; cell young_size, aging_size, tenured_size; cell code_size; - bool secure_gc; bool fep; bool console; bool signals; diff --git a/vm/mark_bits.hpp b/vm/mark_bits.hpp index 0f14622fcf..44f8b17e35 100644 --- a/vm/mark_bits.hpp +++ b/vm/mark_bits.hpp @@ -95,7 +95,7 @@ template struct mark_bits { } } - bool is_marked_p(Block *address) + bool marked_p(Block *address) { return bitmap_elt(marked,address); } @@ -155,7 +155,7 @@ template struct heap_compactor { void operator()(Block *block, cell size) { - if(this->state->is_marked_p(block)) + if(this->state->marked_p(block)) { memmove(address,block,size); iter((Block *)address,size); diff --git a/vm/master.hpp b/vm/master.hpp index b0e73a4b29..293923ca7b 100755 --- a/vm/master.hpp +++ b/vm/master.hpp @@ -48,9 +48,11 @@ namespace factor #include "bignumint.hpp" #include "bignum.hpp" #include "code_block.hpp" -#include "zone.hpp" +#include "bump_allocator.hpp" +#include "mark_bits.hpp" +#include "free_list_allocator.hpp" #include "write_barrier.hpp" -#include "old_space.hpp" +#include "object_start_map.hpp" #include "aging_space.hpp" #include "tenured_space.hpp" #include "data_heap.hpp" @@ -61,8 +63,6 @@ namespace factor #include "words.hpp" #include "float_bits.hpp" #include "io.hpp" -#include "mark_bits.hpp" -#include "heap.hpp" #include "image.hpp" #include "alien.hpp" #include "code_heap.hpp" diff --git a/vm/old_space.cpp b/vm/object_start_map.cpp similarity index 100% rename from vm/old_space.cpp rename to vm/object_start_map.cpp diff --git a/vm/old_space.hpp b/vm/object_start_map.hpp similarity index 100% rename from vm/old_space.hpp rename to vm/object_start_map.hpp diff --git a/vm/tenured_space.hpp b/vm/tenured_space.hpp index b36d96c4aa..1162bb5fd3 100644 --- a/vm/tenured_space.hpp +++ b/vm/tenured_space.hpp @@ -1,17 +1,17 @@ namespace factor { -struct tenured_space : zone { +struct tenured_space : bump_allocator { object_start_map starts; tenured_space(cell size, cell start) : - zone(size,start), starts(size,start) {} + bump_allocator(size,start), starts(size,start) {} object *allot(cell size) { if(here + size > end) return NULL; - object *obj = zone::allot(size); + object *obj = bump_allocator::allot(size); starts.record_object_start_offset(obj); return obj; } diff --git a/vm/to_tenured_collector.hpp b/vm/to_tenured_collector.hpp index 64bd9aa04d..9a4cf3764b 100644 --- a/vm/to_tenured_collector.hpp +++ b/vm/to_tenured_collector.hpp @@ -3,7 +3,7 @@ namespace factor struct to_tenured_policy { factor_vm *myvm; - zone *tenured; + tenured_space *tenured; to_tenured_policy(factor_vm *myvm_) : myvm(myvm_), tenured(myvm->data->tenured) {} diff --git a/vm/vm.cpp b/vm/vm.cpp index 50dc441086..bcdead7da5 100755 --- a/vm/vm.cpp +++ b/vm/vm.cpp @@ -6,7 +6,6 @@ namespace factor factor_vm::factor_vm() : nursery(0,0), profiling_p(false), - secure_gc(false), gc_off(false), current_gc(NULL), fep_disabled(false), diff --git a/vm/vm.hpp b/vm/vm.hpp index 2df66e97b3..0509127918 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -11,7 +11,7 @@ struct factor_vm context *ctx; /* New objects are allocated here */ - zone nursery; + bump_allocator nursery; /* Add this to a shifted address to compute write barrier offsets */ cell cards_offset; @@ -39,9 +39,6 @@ struct factor_vm unsigned int signal_fpu_status; stack_frame *signal_callstack_top; - /* Zeroes out deallocated memory; set by the -securegc command line argument */ - bool secure_gc; - /* A heap walk allows useful things to be done, like finding all references to an object for debugging purposes. */ cell heap_scan_ptr; @@ -221,7 +218,7 @@ struct factor_vm //data heap void init_card_decks(); void set_data_heap(data_heap *data_); - void init_data_heap(cell young_size, cell aging_size, cell tenured_size, bool secure_gc_); + void init_data_heap(cell young_size, cell aging_size, cell tenured_size); void primitive_size(); cell binary_payload_start(object *pointer); void primitive_data_room(); @@ -311,7 +308,7 @@ struct factor_vm void print_callstack(); void dump_cell(cell x); void dump_memory(cell from, cell to); - void dump_zone(const char *name, zone *z); + void dump_zone(const char *name, bump_allocator *z); void dump_generations(); void dump_objects(cell type); void find_data_references_step(cell *scan); @@ -531,7 +528,7 @@ struct factor_vm template void iterate_code_heap(Iterator &iter_) { code_heap_iterator iter(iter_); - code->iterate_heap(iter); + code->allocator->iterate(iter); } //callbacks From 814f6371d6278a01a36c1c8737db5eea3fd81765 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Tue, 20 Oct 2009 22:20:49 -0500 Subject: [PATCH 12/74] vm: mark sweep gc for tenured space work in progress --- vm/aging_collector.cpp | 2 +- vm/aging_collector.hpp | 4 + vm/aging_space.hpp | 15 +++- vm/bump_allocator.hpp | 21 ++---- vm/code_heap.cpp | 2 +- vm/collector.hpp | 143 ++++++++++++++++++++++++++++++++++++ vm/copying_collector.hpp | 137 +--------------------------------- vm/data_heap.cpp | 33 +++++---- vm/data_heap.hpp | 4 +- vm/debug.cpp | 16 ++-- vm/free_list_allocator.hpp | 137 ++++++++++++++++++++++++---------- vm/full_collector.cpp | 25 ++++--- vm/full_collector.hpp | 15 +++- vm/gc.cpp | 6 +- vm/image.cpp | 6 +- vm/master.hpp | 1 + vm/nursery_collector.hpp | 8 +- vm/nursery_space.hpp | 9 +++ vm/tenured_space.hpp | 58 +++++++++++++-- vm/to_tenured_collector.cpp | 15 +++- vm/to_tenured_collector.hpp | 10 ++- vm/vm.hpp | 4 +- 22 files changed, 420 insertions(+), 251 deletions(-) create mode 100644 vm/nursery_space.hpp diff --git a/vm/aging_collector.cpp b/vm/aging_collector.cpp index 49b1c520ec..2972528cb3 100644 --- a/vm/aging_collector.cpp +++ b/vm/aging_collector.cpp @@ -25,7 +25,7 @@ void factor_vm::collect_aging() collector.trace_cards(data->tenured, card_points_to_aging, simple_unmarker(card_mark_mask)); - collector.cheneys_algorithm(); + collector.tenure_reachable_objects(); } { /* If collection fails here, do a to_tenured collection. */ diff --git a/vm/aging_collector.hpp b/vm/aging_collector.hpp index 8f2677c8b6..a04261d826 100644 --- a/vm/aging_collector.hpp +++ b/vm/aging_collector.hpp @@ -15,6 +15,10 @@ struct aging_policy { { return !(aging->contains_p(untagged) || tenured->contains_p(untagged)); } + + void promoted_object(object *obj) {} + + void visited_object(object *obj) {} }; struct aging_collector : copying_collector { diff --git a/vm/aging_space.hpp b/vm/aging_space.hpp index 20e2506539..99efd44de5 100644 --- a/vm/aging_space.hpp +++ b/vm/aging_space.hpp @@ -1,20 +1,29 @@ namespace factor { -struct aging_space : bump_allocator { +struct aging_space : bump_allocator { object_start_map starts; aging_space(cell size, cell start) : - bump_allocator(size,start), starts(size,start) {} + bump_allocator(size,start), starts(size,start) {} object *allot(cell size) { if(here + size > end) return NULL; - object *obj = bump_allocator::allot(size); + object *obj = bump_allocator::allot(size); starts.record_object_start_offset(obj); return obj; } + + cell next_object_after(cell scan) + { + cell size = ((object *)scan)->size(); + if(scan + size < here) + return scan + size; + else + return 0; + } }; } diff --git a/vm/bump_allocator.hpp b/vm/bump_allocator.hpp index 64011e0bb6..b41613b540 100644 --- a/vm/bump_allocator.hpp +++ b/vm/bump_allocator.hpp @@ -1,7 +1,7 @@ namespace factor { -struct bump_allocator { +template struct bump_allocator { /* offset of 'here' and 'end' is hardcoded in compiler backends */ cell here; cell start; @@ -9,27 +9,18 @@ struct bump_allocator { cell size; bump_allocator(cell size_, cell start_) : - here(0), start(start_), end(start_ + size_), size(size_) {} + here(start_), start(start_), end(start_ + size_), size(size_) {} - inline bool contains_p(object *pointer) + inline bool contains_p(Block *block) { - return ((cell)pointer - start) < size; + return ((cell)block - start) < size; } - inline object *allot(cell size) + inline Block *allot(cell size) { cell h = here; here = h + align(size,data_alignment); - return (object *)h; - } - - cell next_allocated_block_after(cell scan) - { - cell size = ((object *)scan)->size(); - if(scan + size < here) - return scan + size; - else - return 0; + return (Block *)h; } }; diff --git a/vm/code_heap.cpp b/vm/code_heap.cpp index 2a466857f9..5ae55cb760 100755 --- a/vm/code_heap.cpp +++ b/vm/code_heap.cpp @@ -8,7 +8,7 @@ code_heap::code_heap(cell size) if(size > (1L << (sizeof(cell) * 8 - 6))) fatal_error("Heap too large",size); seg = new segment(align_page(size),true); if(!seg) fatal_error("Out of memory in heap allocator",size); - allocator = new free_list_allocator(seg->start,size); + allocator = new free_list_allocator(size,seg->start); } code_heap::~code_heap() diff --git a/vm/collector.hpp b/vm/collector.hpp index 9200d95399..bc04ee4de7 100644 --- a/vm/collector.hpp +++ b/vm/collector.hpp @@ -40,7 +40,10 @@ template struct collector { object *untagged = parent->untag(pointer); if(!policy.should_copy_p(untagged)) + { + policy.visited_object(untagged); return; + } object *forwarding = resolve_forwarding(untagged); @@ -49,7 +52,10 @@ template struct collector { else if(policy.should_copy_p(forwarding)) untagged = promote_object(forwarding); else + { untagged = forwarding; + policy.visited_object(untagged); + } *handle = RETAG(untagged,TAG(pointer)); } @@ -79,6 +85,8 @@ template struct collector { stats->object_count++; stats->bytes_copied += size; + policy.promoted_object(newpointer); + return newpointer; } @@ -145,6 +153,141 @@ template struct collector { ctx = ctx->next; } } + + inline cell first_card_in_deck(cell deck) + { + return deck << (deck_bits - card_bits); + } + + inline cell last_card_in_deck(cell deck) + { + return first_card_in_deck(deck + 1); + } + + inline cell card_deck_for_address(cell a) + { + return addr_to_deck(a - this->data->start); + } + + inline cell card_start_address(cell card) + { + return (card << card_bits) + this->data->start; + } + + inline cell card_end_address(cell card) + { + return ((card + 1) << card_bits) + this->data->start; + } + + void trace_partial_objects(cell start, cell end, cell card_start, cell card_end) + { + if(card_start < end) + { + start += sizeof(cell); + + if(start < card_start) start = card_start; + if(end > card_end) end = card_end; + + cell *slot_ptr = (cell *)start; + cell *end_ptr = (cell *)end; + + if(slot_ptr != end_ptr) + { + for(; slot_ptr < end_ptr; slot_ptr++) + this->trace_handle(slot_ptr); + } + } + } + + template + void trace_cards(SourceGeneration *gen, card mask, Unmarker unmarker) + { + u64 start_time = current_micros(); + + card_deck *decks = this->data->decks; + card_deck *cards = this->data->cards; + + cell gen_start_card = addr_to_card(gen->start - this->data->start); + + cell first_deck = card_deck_for_address(gen->start); + cell last_deck = card_deck_for_address(gen->end); + + cell start = 0, binary_start = 0, end = 0; + + for(cell deck_index = first_deck; deck_index < last_deck; deck_index++) + { + if(decks[deck_index] & mask) + { + this->parent->gc_stats.decks_scanned++; + + cell first_card = first_card_in_deck(deck_index); + cell last_card = last_card_in_deck(deck_index); + + for(cell card_index = first_card; card_index < last_card; card_index++) + { + if(cards[card_index] & mask) + { + this->parent->gc_stats.cards_scanned++; + + if(end < card_start_address(card_index)) + { + start = gen->starts.find_object_containing_card(card_index - gen_start_card); + binary_start = start + this->parent->binary_payload_start((object *)start); + end = start + ((object *)start)->size(); + } + +#ifdef FACTOR_DEBUG + assert(addr_to_card(start - this->data->start) <= card_index); + assert(start < card_end_address(card_index)); +#endif + +scan_next_object: { + trace_partial_objects( + start, + binary_start, + card_start_address(card_index), + card_end_address(card_index)); + if(end < card_end_address(card_index)) + { + start = gen->next_object_after(start); + if(start) + { + binary_start = start + this->parent->binary_payload_start((object *)start); + end = start + ((object *)start)->size(); + goto scan_next_object; + } + } + } + + unmarker(&cards[card_index]); + + if(!start) goto end; + } + } + + unmarker(&decks[deck_index]); + } + } + +end: this->parent->gc_stats.card_scan_time += (current_micros() - start_time); + } + + /* Trace all literals referenced from a code block. Only for aging and nursery collections */ + void trace_literal_references(code_block *compiled) + { + this->trace_handle(&compiled->owner); + this->trace_handle(&compiled->literals); + this->trace_handle(&compiled->relocation); + this->parent->gc_stats.code_blocks_scanned++; + } + + void trace_code_heap_roots(std::set *remembered_set) + { + std::set::const_iterator iter = remembered_set->begin(); + std::set::const_iterator end = remembered_set->end(); + + for(; iter != end; iter++) trace_literal_references(*iter); + } }; } diff --git a/vm/copying_collector.hpp b/vm/copying_collector.hpp index ea7faf2423..012aa4ec10 100644 --- a/vm/copying_collector.hpp +++ b/vm/copying_collector.hpp @@ -18,147 +18,12 @@ struct copying_collector : collector { explicit copying_collector(factor_vm *parent_, generation_statistics *stats_, TargetGeneration *target_, Policy policy_) : collector(parent_,stats_,target_,policy_), scan(target_->here) {} - inline cell first_card_in_deck(cell deck) - { - return deck << (deck_bits - card_bits); - } - - inline cell last_card_in_deck(cell deck) - { - return first_card_in_deck(deck + 1); - } - - inline cell card_deck_for_address(cell a) - { - return addr_to_deck(a - this->data->start); - } - - inline cell card_start_address(cell card) - { - return (card << card_bits) + this->data->start; - } - - inline cell card_end_address(cell card) - { - return ((card + 1) << card_bits) + this->data->start; - } - - void trace_partial_objects(cell start, cell end, cell card_start, cell card_end) - { - if(card_start < end) - { - start += sizeof(cell); - - if(start < card_start) start = card_start; - if(end > card_end) end = card_end; - - cell *slot_ptr = (cell *)start; - cell *end_ptr = (cell *)end; - - if(slot_ptr != end_ptr) - { - for(; slot_ptr < end_ptr; slot_ptr++) - this->trace_handle(slot_ptr); - } - } - } - - template - void trace_cards(SourceGeneration *gen, card mask, Unmarker unmarker) - { - u64 start_time = current_micros(); - - card_deck *decks = this->data->decks; - card_deck *cards = this->data->cards; - - cell gen_start_card = addr_to_card(gen->start - this->data->start); - - cell first_deck = card_deck_for_address(gen->start); - cell last_deck = card_deck_for_address(gen->end); - - cell start = 0, binary_start = 0, end = 0; - - for(cell deck_index = first_deck; deck_index < last_deck; deck_index++) - { - if(decks[deck_index] & mask) - { - this->parent->gc_stats.decks_scanned++; - - cell first_card = first_card_in_deck(deck_index); - cell last_card = last_card_in_deck(deck_index); - - for(cell card_index = first_card; card_index < last_card; card_index++) - { - if(cards[card_index] & mask) - { - this->parent->gc_stats.cards_scanned++; - - if(end < card_start_address(card_index)) - { - start = gen->starts.find_object_containing_card(card_index - gen_start_card); - binary_start = start + this->parent->binary_payload_start((object *)start); - end = start + ((object *)start)->size(); - } - -#ifdef FACTOR_DEBUG - assert(addr_to_card(start - this->data->start) <= card_index); - assert(start < card_end_address(card_index)); -#endif - -scan_next_object: { - trace_partial_objects( - start, - binary_start, - card_start_address(card_index), - card_end_address(card_index)); - if(end < card_end_address(card_index)) - { - start = gen->next_allocated_block_after(start); - if(start) - { - binary_start = start + this->parent->binary_payload_start((object *)start); - end = start + ((object *)start)->size(); - goto scan_next_object; - } - } - } - - unmarker(&cards[card_index]); - - if(!start) goto end; - } - } - - unmarker(&decks[deck_index]); - } - } - -end: this->parent->gc_stats.card_scan_time += (current_micros() - start_time); - } - - /* Trace all literals referenced from a code block. Only for aging and nursery collections */ - void trace_literal_references(code_block *compiled) - { - this->trace_handle(&compiled->owner); - this->trace_handle(&compiled->literals); - this->trace_handle(&compiled->relocation); - this->parent->gc_stats.code_blocks_scanned++; - } - - void trace_code_heap_roots(std::set *remembered_set) - { - std::set::const_iterator iter = remembered_set->begin(); - std::set::const_iterator end = remembered_set->end(); - - for(; iter != end; iter++) trace_literal_references(*iter); - } - void cheneys_algorithm() { while(scan && scan < this->target->here) { this->trace_slots((object *)scan); - scan = this->target->next_allocated_block_after(scan); + scan = this->target->next_object_after(scan); } } }; diff --git a/vm/data_heap.cpp b/vm/data_heap.cpp index 5a8f4d6b36..7c887c7419 100755 --- a/vm/data_heap.cpp +++ b/vm/data_heap.cpp @@ -19,7 +19,7 @@ data_heap::data_heap(cell young_size_, cell aging_size_, cell tenured_size_) aging_size = aging_size_; tenured_size = tenured_size_; - cell total_size = young_size + 2 * aging_size + 2 * tenured_size; + cell total_size = young_size + 2 * aging_size + tenured_size; total_size += deck_size; @@ -29,20 +29,21 @@ data_heap::data_heap(cell young_size_, cell aging_size_, cell tenured_size_) cards = new card[cards_size]; cards_end = cards + cards_size; + memset(cards,0,cards_size); cell decks_size = addr_to_deck(total_size); decks = new card_deck[decks_size]; decks_end = decks + decks_size; + memset(decks,0,decks_size); start = align(seg->start,deck_size); tenured = new tenured_space(tenured_size,start); - tenured_semispace = new tenured_space(tenured_size,tenured->end); - aging = new aging_space(aging_size,tenured_semispace->end); + aging = new aging_space(aging_size,tenured->end); aging_semispace = new aging_space(aging_size,aging->end); - nursery = new bump_allocator(young_size,aging_semispace->end); + nursery = new nursery_space(young_size,aging_semispace->end); assert(seg->end - nursery->end <= deck_size); } @@ -54,7 +55,6 @@ data_heap::~data_heap() delete aging; delete aging_semispace; delete tenured; - delete tenured_semispace; delete[] cards; delete[] decks; } @@ -71,8 +71,6 @@ void factor_vm::set_data_heap(data_heap *data_) nursery = *data->nursery; nursery.here = nursery.start; init_card_decks(); - data->reset_generation(data->aging); - data->reset_generation(data->tenured); } void factor_vm::init_data_heap(cell young_size, cell aging_size, cell tenured_size) @@ -185,8 +183,11 @@ void factor_vm::primitive_data_room() a.add(tag_fixnum((data->aging->end - data->aging->here) >> 10)); a.add(tag_fixnum((data->aging->size) >> 10)); - a.add(tag_fixnum((data->tenured->end - data->tenured->here) >> 10)); - a.add(tag_fixnum((data->tenured->size) >> 10)); + //XXX + cell used, total_free, max_free; + data->tenured->usage(&used,&total_free,&max_free); + a.add(tag_fixnum(total_free >> 10)); + a.add(tag_fixnum(used >> 10)); a.trim(); dpush(a.elements.value()); @@ -195,7 +196,7 @@ void factor_vm::primitive_data_room() /* Disables GC and activates next-object ( -- obj ) primitive */ void factor_vm::begin_scan() { - heap_scan_ptr = data->tenured->start; + heap_scan_ptr = data->tenured->first_object(); gc_off = true; } @@ -214,12 +215,14 @@ cell factor_vm::next_object() if(!gc_off) general_error(ERROR_HEAP_SCAN,false_object,false_object,NULL); - if(heap_scan_ptr >= data->tenured->here) + if(heap_scan_ptr) + { + cell current = heap_scan_ptr; + heap_scan_ptr = data->tenured->next_object_after(heap_scan_ptr); + return tag_dynamic((object *)current); + } + else return false_object; - - object *obj = (object *)heap_scan_ptr; - heap_scan_ptr += obj->size(); - return tag_dynamic(obj); } /* Push object at heap scan cursor and advance; pushes f when done */ diff --git a/vm/data_heap.hpp b/vm/data_heap.hpp index 526a5eb9ae..fe714b91b0 100755 --- a/vm/data_heap.hpp +++ b/vm/data_heap.hpp @@ -10,11 +10,10 @@ struct data_heap { segment *seg; - bump_allocator *nursery; + nursery_space *nursery; aging_space *aging; aging_space *aging_semispace; tenured_space *tenured; - tenured_space *tenured_semispace; card *cards; card *cards_end; @@ -49,7 +48,6 @@ their allocation pointers and cards reset. */ template void data_heap::reset_generation(Generation *gen) { gen->here = gen->start; - clear_cards(gen); clear_decks(gen); gen->starts.clear_object_start_offsets(); diff --git a/vm/debug.cpp b/vm/debug.cpp index 31164e972b..afc0b43f7a 100755 --- a/vm/debug.cpp +++ b/vm/debug.cpp @@ -209,19 +209,21 @@ void factor_vm::dump_memory(cell from, cell to) dump_cell(from); } -void factor_vm::dump_zone(const char *name, bump_allocator *z) +template +void factor_vm::dump_generation(const char *name, Generation *gen) { print_string(name); print_string(": "); - print_string("Start="); print_cell(z->start); - print_string(", size="); print_cell(z->size); - print_string(", here="); print_cell(z->here - z->start); nl(); + print_string("Start="); print_cell(gen->start); + print_string(", size="); print_cell(gen->size); + print_string(", end="); print_cell(gen->end); + nl(); } void factor_vm::dump_generations() { - dump_zone("Nursery",&nursery); - dump_zone("Aging",data->aging); - dump_zone("Tenured",data->tenured); + dump_generation("Nursery",&nursery); + dump_generation("Aging",data->aging); + dump_generation("Tenured",data->tenured); print_string("Cards: base="); print_cell((cell)data->cards); diff --git a/vm/free_list_allocator.hpp b/vm/free_list_allocator.hpp index da2622bb0b..c8f3bd6f47 100644 --- a/vm/free_list_allocator.hpp +++ b/vm/free_list_allocator.hpp @@ -9,29 +9,17 @@ struct free_list { }; template struct free_list_allocator { - cell start; cell size; + cell start; cell end; free_list free_blocks; mark_bits state; - explicit free_list_allocator(cell start, cell size); - - inline Block *first_block() - { - return (Block *)start; - } - - inline Block *last_block() - { - return (Block *)end; - } - - Block *next_block_after(heap_block *block) - { - return (Block *)((cell)block + block->size()); - } - + explicit free_list_allocator(cell size, cell start); + bool contains_p(Block *block); + Block *first_block(); + Block *last_block(); + Block *next_block_after(Block *block); void clear_free_list(); void add_to_free_list(free_heap_block *block); void build_free_list(cell size); @@ -42,35 +30,42 @@ template struct free_list_allocator { void free(Block *block); void usage(cell *used, cell *total_free, cell *max_free); cell occupied(); - + void sweep(); template void sweep(Iterator &iter); template void compact(Iterator &iter); - - template void iterate(Iterator &iter) - { - Block *scan = first_block(); - Block *end = last_block(); - - while(scan != end) - { - cell size = scan->size(); - Block *next = (Block *)((cell)scan + size); - if(!scan->free_p()) iter(scan,size); - scan = next; - } - } + template void iterate(Iterator &iter); }; +template +free_list_allocator::free_list_allocator(cell size_, cell start_) : + size(size_), start(start_), end(start_ + size_), state(mark_bits(size_,start_)) +{ + clear_free_list(); +} + template void free_list_allocator::clear_free_list() { memset(&free_blocks,0,sizeof(free_list)); } -template -free_list_allocator::free_list_allocator(cell start_, cell size_) : - start(start_), size(size_), end(start_ + size_), state(mark_bits(start_,size_)) +template bool free_list_allocator::contains_p(Block *block) { - clear_free_list(); + return ((cell)block - start) < size; +} + +template Block *free_list_allocator::first_block() +{ + return (Block *)start; +} + +template Block *free_list_allocator::last_block() +{ + return (Block *)end; +} + +template Block *free_list_allocator::next_block_after(Block *block) +{ + return (Block *)((cell)block + block->size()); } template void free_list_allocator::add_to_free_list(free_heap_block *block) @@ -234,8 +229,56 @@ template cell free_list_allocator::occupied() return size; } -/* After code GC, all live code blocks are marked, so any -which are not marked can be reclaimed. */ +template +void free_list_allocator::sweep() +{ + this->clear_free_list(); + + Block *prev = NULL; + Block *scan = this->first_block(); + Block *end = this->last_block(); + + while(scan != end) + { + cell size = scan->size(); + + if(scan->free_p()) + { + if(prev && prev->free_p()) + { + free_heap_block *free_prev = (free_heap_block *)prev; + free_prev->set_size(free_prev->size() + size); + } + else + prev = scan; + } + else if(this->state.marked_p(scan)) + { + if(prev && prev->free_p()) + this->add_to_free_list((free_heap_block *)prev); + prev = scan; + } + else + { + if(prev && prev->free_p()) + { + free_heap_block *free_prev = (free_heap_block *)prev; + free_prev->set_size(free_prev->size() + size); + } + else + { + ((free_heap_block *)scan)->set_free(); + prev = scan; + } + } + + scan = (Block *)((cell)scan + size); + } + + if(prev && prev->free_p()) + this->add_to_free_list((free_heap_block *)prev); +} + template template void free_list_allocator::sweep(Iterator &iter) @@ -302,4 +345,20 @@ void free_list_allocator::compact(Iterator &iter) this->build_free_list((cell)compactor.address - this->start); } +template +template +void free_list_allocator::iterate(Iterator &iter) +{ + Block *scan = first_block(); + Block *end = last_block(); + + while(scan != end) + { + cell size = scan->size(); + Block *next = (Block *)((cell)scan + size); + if(!scan->free_p()) iter(scan,size); + scan = next; + } +} + } diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index 924cad6777..9191823d75 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -4,7 +4,7 @@ namespace factor { full_collector::full_collector(factor_vm *parent_) : - copying_collector( + collector( parent_, &parent_->gc_stats.full_stats, parent_->data->tenured, @@ -89,18 +89,22 @@ void full_collector::trace_literal_references(code_block *compiled) collections */ void full_collector::mark_code_block(code_block *compiled) { - this->code->set_marked_p(compiled); - trace_literal_references(compiled); + if(!this->code->marked_p(compiled)) + { + this->code->set_marked_p(compiled); + trace_literal_references(compiled); + } } -void full_collector::cheneys_algorithm() +void full_collector::mark_reachable_objects() { - while(scan && scan < target->here) + std::vector *mark_stack = &this->target->mark_stack; + while(!mark_stack->empty()) { - object *obj = (object *)scan; + object *obj = mark_stack->back(); + mark_stack->pop_back(); this->trace_slots(obj); this->mark_object_code_block(obj); - scan = target->next_allocated_block_after(scan); } } @@ -109,6 +113,7 @@ void factor_vm::collect_full_impl(bool trace_contexts_p) full_collector collector(this); code->clear_mark_bits(); + data->tenured->clear_mark_bits(); collector.trace_roots(); if(trace_contexts_p) @@ -118,8 +123,9 @@ void factor_vm::collect_full_impl(bool trace_contexts_p) collector.trace_callbacks(); } - collector.cheneys_algorithm(); + collector.mark_reachable_objects(); + data->tenured->sweep(); data->reset_generation(data->aging); nursery.here = nursery.start; } @@ -144,9 +150,6 @@ void factor_vm::collect_growing_heap(cell requested_bytes, void factor_vm::collect_full(bool trace_contexts_p, bool compact_code_heap_p) { - /* Copy all live objects to the tenured semispace. */ - std::swap(data->tenured,data->tenured_semispace); - data->reset_generation(data->tenured); collect_full_impl(trace_contexts_p); if(compact_code_heap_p) diff --git a/vm/full_collector.hpp b/vm/full_collector.hpp index 5298f56637..9aef352b4b 100644 --- a/vm/full_collector.hpp +++ b/vm/full_collector.hpp @@ -11,9 +11,20 @@ struct full_policy { { return !tenured->contains_p(untagged); } + + void promoted_object(object *obj) + { + tenured->mark_and_push(obj); + } + + void visited_object(object *obj) + { + if(!tenured->marked_p(obj)) + tenured->mark_and_push(obj); + } }; -struct full_collector : copying_collector { +struct full_collector : collector { bool trace_contexts_p; full_collector(factor_vm *parent_); @@ -22,7 +33,7 @@ struct full_collector : copying_collector { void trace_callbacks(); void trace_literal_references(code_block *compiled); void mark_code_block(code_block *compiled); - void cheneys_algorithm(); + void mark_reachable_objects(); }; } diff --git a/vm/gc.cpp b/vm/gc.cpp index c8ba57b7f2..6cb99b6da0 100755 --- a/vm/gc.cpp +++ b/vm/gc.cpp @@ -235,11 +235,13 @@ object *factor_vm::allot_object(header header, cell size) else { /* If tenured space does not have enough room, collect */ - if(data->tenured->here + size > data->tenured->end) + //XXX + //if(data->tenured->here + size > data->tenured->end) primitive_full_gc(); /* If it still won't fit, grow the heap */ - if(data->tenured->here + size > data->tenured->end) + //XXX + //if(data->tenured->here + size > data->tenured->end) { gc(collect_growing_heap_op, size, /* requested size */ diff --git a/vm/image.cpp b/vm/image.cpp index b47fe4e86a..ee0a1064ed 100755 --- a/vm/image.cpp +++ b/vm/image.cpp @@ -39,7 +39,7 @@ void factor_vm::load_data_heap(FILE *file, image_header *h, vm_parameters *p) fatal_error("load_data_heap failed",0); } - data->tenured->here = data->tenured->start + h->data_size; + data->tenured->build_free_list(h->data_size); } void factor_vm::load_code_heap(FILE *file, image_header *h, vm_parameters *p) @@ -203,7 +203,7 @@ void factor_vm::relocate_data(cell data_relocation_base, cell code_relocation_ba { relocate_object((object *)obj,data_relocation_base,code_relocation_base); data->tenured->starts.record_object_start_offset((object *)obj); - obj = data->tenured->next_allocated_block_after(obj); + obj = data->tenured->next_object_after(obj); } } @@ -289,7 +289,7 @@ bool factor_vm::save_image(const vm_char *filename) h.magic = image_magic; h.version = image_version; h.data_relocation_base = data->tenured->start; - h.data_size = data->tenured->here - data->tenured->start; + h.data_size = data->tenured->occupied(); h.code_relocation_base = code->seg->start; h.code_size = code->allocator->occupied(); diff --git a/vm/master.hpp b/vm/master.hpp index 293923ca7b..0282a0597d 100755 --- a/vm/master.hpp +++ b/vm/master.hpp @@ -53,6 +53,7 @@ namespace factor #include "free_list_allocator.hpp" #include "write_barrier.hpp" #include "object_start_map.hpp" +#include "nursery_space.hpp" #include "aging_space.hpp" #include "tenured_space.hpp" #include "data_heap.hpp" diff --git a/vm/nursery_collector.hpp b/vm/nursery_collector.hpp index f9d2172929..778efab138 100644 --- a/vm/nursery_collector.hpp +++ b/vm/nursery_collector.hpp @@ -6,10 +6,14 @@ struct nursery_policy { nursery_policy(factor_vm *parent_) : parent(parent_) {} - bool should_copy_p(object *untagged) + bool should_copy_p(object *obj) { - return parent->nursery.contains_p(untagged); + return parent->nursery.contains_p(obj); } + + void promoted_object(object *obj) {} + + void visited_object(object *obj) {} }; struct nursery_collector : copying_collector { diff --git a/vm/nursery_space.hpp b/vm/nursery_space.hpp new file mode 100644 index 0000000000..4425c1612b --- /dev/null +++ b/vm/nursery_space.hpp @@ -0,0 +1,9 @@ +namespace factor +{ + +struct nursery_space : bump_allocator +{ + nursery_space(cell size, cell start) : bump_allocator(size,start) {} +}; + +} diff --git a/vm/tenured_space.hpp b/vm/tenured_space.hpp index 1162bb5fd3..c0c12d3f58 100644 --- a/vm/tenured_space.hpp +++ b/vm/tenured_space.hpp @@ -1,19 +1,65 @@ namespace factor { -struct tenured_space : bump_allocator { +struct tenured_space : free_list_allocator { object_start_map starts; + std::vector mark_stack; tenured_space(cell size, cell start) : - bump_allocator(size,start), starts(size,start) {} + free_list_allocator(size,start), starts(size,start) {} object *allot(cell size) { - if(here + size > end) return NULL; + object *obj = free_list_allocator::allot(size); + if(obj) + { + starts.record_object_start_offset(obj); + return obj; + } + else + return NULL; + } - object *obj = bump_allocator::allot(size); - starts.record_object_start_offset(obj); - return obj; + object *first_allocated_block_after(object *block) + { + while(block != this->last_block() && block->free_p()) + { + free_heap_block *free_block = (free_heap_block *)block; + block = (object *)((cell)free_block + free_block->size()); + } + + if(block == this->last_block()) + return NULL; + else + return block; + } + + cell first_object() + { + return (cell)first_allocated_block_after(this->first_block()); + } + + cell next_object_after(cell scan) + { + cell size = ((object *)scan)->size(); + object *next = (object *)(scan + size); + return (cell)first_allocated_block_after(next); + } + + void clear_mark_bits() + { + state.clear_mark_bits(); + } + + bool marked_p(object *obj) + { + return this->state.marked_p(obj); + } + + void mark_and_push(object *obj) + { + this->state.set_marked_p(obj); + this->mark_stack.push_back(obj); } }; diff --git a/vm/to_tenured_collector.cpp b/vm/to_tenured_collector.cpp index 3676324ce2..3150647cd2 100644 --- a/vm/to_tenured_collector.cpp +++ b/vm/to_tenured_collector.cpp @@ -4,12 +4,23 @@ namespace factor { to_tenured_collector::to_tenured_collector(factor_vm *myvm_) : - copying_collector( + collector( myvm_, &myvm_->gc_stats.aging_stats, myvm_->data->tenured, to_tenured_policy(myvm_)) {} +void to_tenured_collector::tenure_reachable_objects() +{ + std::vector *mark_stack = &this->target->mark_stack; + while(!mark_stack->empty()) + { + object *obj = mark_stack->back(); + mark_stack->pop_back(); + this->trace_slots(obj); + } +} + void factor_vm::collect_to_tenured() { /* Copy live objects from aging space to tenured space. */ @@ -21,7 +32,7 @@ void factor_vm::collect_to_tenured() card_points_to_aging, dummy_unmarker()); collector.trace_code_heap_roots(&code->points_to_aging); - collector.cheneys_algorithm(); + collector.tenure_reachable_objects(); update_code_heap_for_minor_gc(&code->points_to_aging); nursery.here = nursery.start; diff --git a/vm/to_tenured_collector.hpp b/vm/to_tenured_collector.hpp index 9a4cf3764b..e87ba5ee29 100644 --- a/vm/to_tenured_collector.hpp +++ b/vm/to_tenured_collector.hpp @@ -11,10 +11,18 @@ struct to_tenured_policy { { return !tenured->contains_p(untagged); } + + void promoted_object(object *obj) + { + tenured->mark_stack.push_back(obj); + } + + void visited_object(object *obj) {} }; -struct to_tenured_collector : copying_collector { +struct to_tenured_collector : collector { to_tenured_collector(factor_vm *myvm_); + void tenure_reachable_objects(); }; } diff --git a/vm/vm.hpp b/vm/vm.hpp index 0509127918..615efe35ed 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -11,7 +11,7 @@ struct factor_vm context *ctx; /* New objects are allocated here */ - bump_allocator nursery; + nursery_space nursery; /* Add this to a shifted address to compute write barrier offsets */ cell cards_offset; @@ -308,7 +308,7 @@ struct factor_vm void print_callstack(); void dump_cell(cell x); void dump_memory(cell from, cell to); - void dump_zone(const char *name, bump_allocator *z); + template void dump_generation(const char *name, Generation *gen); void dump_generations(); void dump_objects(cell type); void find_data_references_step(cell *scan); From af855b7fa9ec681ae056395047cc4929de80b06c Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Wed, 21 Oct 2009 19:41:54 -0500 Subject: [PATCH 13/74] vm: debugging mark-sweep --- vm/aging_collector.cpp | 3 ++- vm/data_heap.cpp | 34 +++++++++++++++++++++++++++++++++- vm/data_heap.hpp | 28 +++------------------------- vm/free_list_allocator.hpp | 23 +++++++++++++---------- vm/full_collector.cpp | 25 +++++++++++++++++++------ vm/layouts.hpp | 25 +++++++++++-------------- vm/mark_bits.hpp | 5 ----- vm/nursery_collector.cpp | 2 +- vm/tenured_space.hpp | 5 +++++ vm/to_tenured_collector.cpp | 9 +++++---- 10 files changed, 92 insertions(+), 67 deletions(-) diff --git a/vm/aging_collector.cpp b/vm/aging_collector.cpp index 2972528cb3..d33823b624 100644 --- a/vm/aging_collector.cpp +++ b/vm/aging_collector.cpp @@ -40,9 +40,10 @@ void factor_vm::collect_aging() collector.trace_contexts(); collector.trace_code_heap_roots(&code->points_to_aging); collector.cheneys_algorithm(); + update_code_heap_for_minor_gc(&code->points_to_aging); - nursery.here = nursery.start; + data->reset_generation(&nursery); code->points_to_nursery.clear(); } } diff --git a/vm/data_heap.cpp b/vm/data_heap.cpp index 7c887c7419..915ed8e32a 100755 --- a/vm/data_heap.cpp +++ b/vm/data_heap.cpp @@ -65,11 +65,43 @@ data_heap *data_heap::grow(cell requested_bytes) return new data_heap(young_size,aging_size,new_tenured_size); } +template void data_heap::clear_cards(Generation *gen) +{ + cell first_card = addr_to_card(gen->start - start); + cell last_card = addr_to_card(gen->end - start); + memset(&cards[first_card],0,last_card - first_card); +} + +template void data_heap::clear_decks(Generation *gen) +{ + cell first_deck = addr_to_deck(gen->start - start); + cell last_deck = addr_to_deck(gen->end - start); + memset(&decks[first_deck],0,last_deck - first_deck); +} + +void data_heap::reset_generation(nursery_space *gen) +{ + gen->here = gen->start; +} + +void data_heap::reset_generation(aging_space *gen) +{ + gen->here = gen->start; + clear_cards(gen); + clear_decks(gen); + gen->starts.clear_object_start_offsets(); +} + +void data_heap::reset_generation(tenured_space *gen) +{ + clear_cards(gen); + clear_decks(gen); +} + void factor_vm::set_data_heap(data_heap *data_) { data = data_; nursery = *data->nursery; - nursery.here = nursery.start; init_card_decks(); } diff --git a/vm/data_heap.hpp b/vm/data_heap.hpp index fe714b91b0..c8d6ce0b70 100755 --- a/vm/data_heap.hpp +++ b/vm/data_heap.hpp @@ -26,31 +26,9 @@ struct data_heap { data_heap *grow(cell requested_size); template void clear_cards(Generation *gen); template void clear_decks(Generation *gen); - template void reset_generation(Generation *gen); + void reset_generation(nursery_space *gen); + void reset_generation(aging_space *gen); + void reset_generation(tenured_space *gen); }; -template void data_heap::clear_cards(Generation *gen) -{ - cell first_card = addr_to_card(gen->start - start); - cell last_card = addr_to_card(gen->end - start); - memset(&cards[first_card],0,last_card - first_card); -} - -template void data_heap::clear_decks(Generation *gen) -{ - cell first_deck = addr_to_deck(gen->start - start); - cell last_deck = addr_to_deck(gen->end - start); - memset(&decks[first_deck],0,last_deck - first_deck); -} - -/* After garbage collection, any generations which are now empty need to have -their allocation pointers and cards reset. */ -template void data_heap::reset_generation(Generation *gen) -{ - gen->here = gen->start; - clear_cards(gen); - clear_decks(gen); - gen->starts.clear_object_start_offsets(); -} - } diff --git a/vm/free_list_allocator.hpp b/vm/free_list_allocator.hpp index c8f3bd6f47..efdad508cb 100644 --- a/vm/free_list_allocator.hpp +++ b/vm/free_list_allocator.hpp @@ -88,10 +88,12 @@ Makes a free list consisting of one free block, at the very end. */ template void free_list_allocator::build_free_list(cell size) { clear_free_list(); - free_heap_block *last_block = (free_heap_block *)(start + size); - last_block->set_free(); - last_block->set_size(end - (cell)last_block); - add_to_free_list(last_block); + if(size != this->size) + { + free_heap_block *last_block = (free_heap_block *)(start + size); + last_block->make_free(end - (cell)last_block); + add_to_free_list(last_block); + } } template void free_list_allocator::assert_free_block(free_heap_block *block) @@ -147,10 +149,9 @@ template free_heap_block *free_list_allocator::split_free { /* split the block in two */ free_heap_block *split = (free_heap_block *)((cell)block + size); - split->set_free(); - split->set_size(block->size() - size); + split->make_free(block->size() - size); split->next_free = block->next_free; - block->set_size(size); + block->make_free(size); add_to_free_list(split); } @@ -174,7 +175,7 @@ template Block *free_list_allocator::allot(cell size) template void free_list_allocator::free(Block *block) { free_heap_block *free_block = (free_heap_block *)block; - free_block->set_free(); + free_block->make_free(block->size()); add_to_free_list(free_block); } @@ -267,7 +268,8 @@ void free_list_allocator::sweep() } else { - ((free_heap_block *)scan)->set_free(); + free_heap_block *free_block = (free_heap_block *)scan; + free_block->make_free(size); prev = scan; } } @@ -319,7 +321,8 @@ void free_list_allocator::sweep(Iterator &iter) } else { - scan->set_free(); + free_heap_block *free_block = (free_heap_block *)scan; + free_block->make_free(size); prev = scan; } } diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index 9191823d75..817908ece5 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -108,12 +108,24 @@ void full_collector::mark_reachable_objects() } } +struct object_start_map_updater { + object_start_map *starts; + + object_start_map_updater(object_start_map *starts_) : starts(starts_) {} + + void operator()(object *obj, cell size) + { + starts->record_object_start_offset(obj); + } +}; + void factor_vm::collect_full_impl(bool trace_contexts_p) { full_collector collector(this); code->clear_mark_bits(); data->tenured->clear_mark_bits(); + data->tenured->clear_mark_stack(); collector.trace_roots(); if(trace_contexts_p) @@ -125,9 +137,14 @@ void factor_vm::collect_full_impl(bool trace_contexts_p) collector.mark_reachable_objects(); - data->tenured->sweep(); + data->tenured->starts.clear_object_start_offsets(); + object_start_map_updater updater(&data->tenured->starts); + data->tenured->sweep(updater); + + data->reset_generation(data->tenured); data->reset_generation(data->aging); - nursery.here = nursery.start; + data->reset_generation(&nursery); + code->clear_remembered_set(); } void factor_vm::collect_growing_heap(cell requested_bytes, @@ -144,8 +161,6 @@ void factor_vm::collect_growing_heap(cell requested_bytes, compact_code_heap(trace_contexts_p); else relocate_code_heap(); - - code->clear_remembered_set(); } void factor_vm::collect_full(bool trace_contexts_p, bool compact_code_heap_p) @@ -156,8 +171,6 @@ void factor_vm::collect_full(bool trace_contexts_p, bool compact_code_heap_p) compact_code_heap(trace_contexts_p); else update_code_heap_words_and_literals(); - - code->clear_remembered_set(); } } diff --git a/vm/layouts.hpp b/vm/layouts.hpp index ca51fd6dca..76581621ca 100644 --- a/vm/layouts.hpp +++ b/vm/layouts.hpp @@ -58,8 +58,6 @@ static const cell data_alignment = 16; #define TYPE_COUNT 15 -/* Not real types, but code_block's type can be set to this */ - enum code_block_type { code_block_unoptimized, @@ -229,30 +227,29 @@ struct heap_block return header & 1 == 1; } - void set_free() - { - header |= 1; - } - - void clear_free() - { - header &= ~1; - } - cell size() { - return header >> 3; + cell bytes = header >> 3; +#ifdef FACTOR_DEBUG + assert(bytes > 0); +#endif + return bytes; } void set_size(cell size) { - header = (header & 0x7) | (size << 3); + header = ((header & 0x7) | (size << 3)); } }; struct free_heap_block : public heap_block { free_heap_block *next_free; + + void make_free(cell size) + { + header = (size << 3) | 1; + } }; struct code_block : public heap_block diff --git a/vm/mark_bits.hpp b/vm/mark_bits.hpp index 44f8b17e35..161a7fd755 100644 --- a/vm/mark_bits.hpp +++ b/vm/mark_bits.hpp @@ -55,11 +55,6 @@ template struct mark_bits { cell line_number = block_line(address); cell word_index = (line_number >> 6); cell word_shift = (line_number & 63); - -#ifdef FACTOR_DEBUG - assert(word_index < bits_size); -#endif - return std::make_pair(word_index,word_shift); } diff --git a/vm/nursery_collector.cpp b/vm/nursery_collector.cpp index 909cde02f8..07f9666f37 100644 --- a/vm/nursery_collector.cpp +++ b/vm/nursery_collector.cpp @@ -28,7 +28,7 @@ void factor_vm::collect_nursery() collector.cheneys_algorithm(); update_code_heap_for_minor_gc(&code->points_to_nursery); - nursery.here = nursery.start; + data->reset_generation(&nursery); code->points_to_nursery.clear(); } diff --git a/vm/tenured_space.hpp b/vm/tenured_space.hpp index c0c12d3f58..7cc4131fa0 100644 --- a/vm/tenured_space.hpp +++ b/vm/tenured_space.hpp @@ -51,6 +51,11 @@ struct tenured_space : free_list_allocator { state.clear_mark_bits(); } + void clear_mark_stack() + { + mark_stack.clear(); + } + bool marked_p(object *obj) { return this->state.marked_p(obj); diff --git a/vm/to_tenured_collector.cpp b/vm/to_tenured_collector.cpp index 3150647cd2..ea7cb8ed72 100644 --- a/vm/to_tenured_collector.cpp +++ b/vm/to_tenured_collector.cpp @@ -26,19 +26,20 @@ void factor_vm::collect_to_tenured() /* Copy live objects from aging space to tenured space. */ to_tenured_collector collector(this); + data->tenured->clear_mark_stack(); + collector.trace_roots(); collector.trace_contexts(); collector.trace_cards(data->tenured, card_points_to_aging, - dummy_unmarker()); + simple_unmarker(card_mark_mask)); collector.trace_code_heap_roots(&code->points_to_aging); collector.tenure_reachable_objects(); update_code_heap_for_minor_gc(&code->points_to_aging); - nursery.here = nursery.start; + data->reset_generation(&nursery); data->reset_generation(data->aging); - code->points_to_nursery.clear(); - code->points_to_aging.clear(); + code->clear_remembered_set(); } } From 40351d40be3d880a62003e3a7b3075ca2b1edc4c Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Wed, 21 Oct 2009 20:12:57 -0500 Subject: [PATCH 14/74] vm: use iostreams instead of printf for debug messages, clean up a few things --- vm/code_block.cpp | 10 +-- vm/collector.hpp | 2 +- vm/data_heap.cpp | 2 +- vm/debug.cpp | 208 ++++++++++++++++++++++---------------------- vm/errors.cpp | 20 +++-- vm/factor.cpp | 9 +- vm/gc.cpp | 11 ++- vm/image.cpp | 26 ++---- vm/image.hpp | 1 + vm/inline_cache.cpp | 8 +- vm/jit.hpp | 2 +- vm/layouts.hpp | 44 +++++----- vm/master.hpp | 1 + vm/os-windows.hpp | 2 - vm/quotations.cpp | 4 +- vm/strings.cpp | 8 +- vm/strings.hpp | 2 +- vm/tagged.hpp | 2 +- vm/utilities.cpp | 32 ------- vm/utilities.hpp | 5 -- vm/vm.hpp | 8 +- 21 files changed, 191 insertions(+), 216 deletions(-) diff --git a/vm/code_block.cpp b/vm/code_block.cpp index 1c15f23382..687bbcf500 100755 --- a/vm/code_block.cpp +++ b/vm/code_block.cpp @@ -286,7 +286,7 @@ struct literal_references_updater { if(parent->relocation_type_of(rel) == RT_IMMEDIATE) { cell offset = parent->relocation_offset_of(rel) + (cell)(compiled + 1); - array *literals = parent->untag(compiled->literals); + array *literals = untag(compiled->literals); fixnum absolute_value = array_nth(literals,index); parent->store_address_in_code_block(parent->relocation_class_of(rel),offset,absolute_value); } @@ -457,10 +457,10 @@ code_block *factor_vm::allot_code_block(cell size, code_block_type type) cell used, total_free, max_free; code->allocator->usage(&used,&total_free,&max_free); - print_string("Code heap stats:\n"); - print_string("Used: "); print_cell(used); nl(); - print_string("Total free space: "); print_cell(total_free); nl(); - print_string("Largest free block: "); print_cell(max_free); nl(); + std::cout << "Code heap stats:\n"; + std::cout << "Used: " << used << "\n"; + std::cout << "Total free space: " << total_free << "\n"; + std::cout << "Largest free block: " << max_free << "\n"; fatal_error("Out of memory in add-compiled-block",0); } } diff --git a/vm/collector.hpp b/vm/collector.hpp index bc04ee4de7..4479fc7b45 100644 --- a/vm/collector.hpp +++ b/vm/collector.hpp @@ -38,7 +38,7 @@ template struct collector { if(immediate_p(pointer)) return; - object *untagged = parent->untag(pointer); + object *untagged = untag(pointer); if(!policy.should_copy_p(untagged)) { policy.visited_object(untagged); diff --git a/vm/data_heap.cpp b/vm/data_heap.cpp index 915ed8e32a..6178dc8861 100755 --- a/vm/data_heap.cpp +++ b/vm/data_heap.cpp @@ -120,7 +120,7 @@ cell factor_vm::object_size(cell tagged) } /* Size of the object pointed to by an untagged pointer */ -cell object::size() +cell object::size() const { if(free_p()) return ((free_heap_block *)this)->size(); diff --git a/vm/debug.cpp b/vm/debug.cpp index afc0b43f7a..b37a7e6f82 100755 --- a/vm/debug.cpp +++ b/vm/debug.cpp @@ -3,36 +3,31 @@ namespace factor { -void factor_vm::print_chars(string* str) +std::ostream &operator<<(std::ostream &out, const string *str) { - cell i; - for(i = 0; i < string_capacity(str); i++) - putchar(string_nth(str,i)); + for(cell i = 0; i < string_capacity(str); i++) + out << (char)str->nth(i); + return out; } void factor_vm::print_word(word* word, cell nesting) { if(tagged(word->vocabulary).type_p(STRING_TYPE)) - { - print_chars(untag(word->vocabulary)); - print_string(":"); - } + std::cout << untag(word->vocabulary) << ":"; if(tagged(word->name).type_p(STRING_TYPE)) - print_chars(untag(word->name)); + std::cout << untag(word->name); else { - print_string("#name,nesting); - print_string(">"); + std::cout << ">"; } } -void factor_vm::print_factor_string(string* str) +void factor_vm::print_factor_string(string *str) { - putchar('"'); - print_chars(str); - putchar('"'); + std::cout << '"' << str << '"'; } void factor_vm::print_array(array* array, cell nesting) @@ -51,12 +46,12 @@ void factor_vm::print_array(array* array, cell nesting) for(i = 0; i < length; i++) { - print_string(" "); + std::cout << " "; print_nested_obj(array_nth(array,i),nesting); } if(trimmed) - print_string("..."); + std::cout << "..."; } void factor_vm::print_tuple(tuple *tuple, cell nesting) @@ -64,12 +59,10 @@ void factor_vm::print_tuple(tuple *tuple, cell nesting) tuple_layout *layout = untag(tuple->layout); cell length = to_fixnum(layout->size); - print_string(" "); + std::cout << " "; print_nested_obj(layout->klass,nesting); - cell i; bool trimmed; - if(length > 10 && !full_output) { trimmed = true; @@ -78,21 +71,21 @@ void factor_vm::print_tuple(tuple *tuple, cell nesting) else trimmed = false; - for(i = 0; i < length; i++) + for(cell i = 0; i < length; i++) { - print_string(" "); + std::cout << " "; print_nested_obj(tuple->data()[i],nesting); } if(trimmed) - print_string("..."); + std::cout << "..."; } void factor_vm::print_nested_obj(cell obj, fixnum nesting) { if(nesting <= 0 && !full_output) { - print_string(" ... "); + std::cout << " ... "; return; } @@ -101,7 +94,7 @@ void factor_vm::print_nested_obj(cell obj, fixnum nesting) switch(tagged(obj).type()) { case FIXNUM_TYPE: - print_fixnum(untag_fixnum(obj)); + std::cout << untag_fixnum(obj); break; case WORD_TYPE: print_word(untag(obj),nesting - 1); @@ -110,30 +103,27 @@ void factor_vm::print_nested_obj(cell obj, fixnum nesting) print_factor_string(untag(obj)); break; case F_TYPE: - print_string("f"); + std::cout << "f"; break; case TUPLE_TYPE: - print_string("T{"); + std::cout << "T{"; print_tuple(untag(obj),nesting - 1); - print_string(" }"); + std::cout << " }"; break; case ARRAY_TYPE: - print_string("{"); + std::cout << "{"; print_array(untag(obj),nesting - 1); - print_string(" }"); + std::cout << " }"; break; case QUOTATION_TYPE: - print_string("["); + std::cout << "["; quot = untag(obj); print_array(untag(quot->array),nesting - 1); - print_string(" ]"); + std::cout << " ]"; break; default: - print_string("#(obj).type()); - print_string(" @ "); - print_cell_hex(obj); - print_string(">"); + std::cout << "#(obj).type() << " @ "; + std::cout << std::hex << obj << std::dec << ">"; break; } } @@ -148,19 +138,19 @@ void factor_vm::print_objects(cell *start, cell *end) for(; start <= end; start++) { print_obj(*start); - nl(); + std::cout << std::endl; } } void factor_vm::print_datastack() { - print_string("==== DATA STACK:\n"); + std::cout << "==== DATA STACK:\n"; print_objects((cell *)ds_bot,(cell *)ds); } void factor_vm::print_retainstack() { - print_string("==== RETAIN STACK:\n"); + std::cout << "==== RETAIN STACK:\n"; print_objects((cell *)rs_bot,(cell *)rs); } @@ -171,34 +161,48 @@ struct stack_frame_printer { void operator()(stack_frame *frame) { parent->print_obj(parent->frame_executing(frame)); - print_string("\n"); + std::cout << std::endl; parent->print_obj(parent->frame_scan(frame)); - print_string("\n"); - print_string("word/quot addr: "); - print_cell_hex((cell)parent->frame_executing(frame)); - print_string("\n"); - print_string("word/quot xt: "); - print_cell_hex((cell)frame->xt); - print_string("\n"); - print_string("return address: "); - print_cell_hex((cell)FRAME_RETURN_ADDRESS(frame,parent)); - print_string("\n"); + std::cout << std::endl; + std::cout << "word/quot addr: "; + std::cout << std::hex << (cell)parent->frame_executing(frame) << std::dec; + std::cout << std::endl; + std::cout << "word/quot xt: "; + std::cout << std::hex << (cell)frame->xt << std::dec; + std::cout << std::endl; + std::cout << "return address: "; + std::cout << std::hex << (cell)FRAME_RETURN_ADDRESS(frame,parent) << std::dec; + std::cout << std::endl; } }; void factor_vm::print_callstack() { - print_string("==== CALL STACK:\n"); + std::cout << "==== CALL STACK:\n"; stack_frame_printer printer(this); iterate_callstack(ctx,printer); } +struct padded_address { + cell value; + + explicit padded_address(cell value_) : value(value_) {} +}; + +std::ostream &operator<<(std::ostream &out, const padded_address &value) +{ + char prev = out.fill('0'); + out.width(sizeof(cell) * 2); + out << std::hex << value.value << std::dec; + out.fill(prev); + return out; +} + void factor_vm::dump_cell(cell x) { - print_cell_hex_pad(x); print_string(": "); + std::cout << padded_address(x) << ": "; x = *(cell *)x; - print_cell_hex_pad(x); print_string(" tag "); print_cell(TAG(x)); - nl(); + std::cout << padded_address(x) << " tag " << TAG(x) << std::endl; } void factor_vm::dump_memory(cell from, cell to) @@ -212,11 +216,11 @@ void factor_vm::dump_memory(cell from, cell to) template void factor_vm::dump_generation(const char *name, Generation *gen) { - print_string(name); print_string(": "); - print_string("Start="); print_cell(gen->start); - print_string(", size="); print_cell(gen->size); - print_string(", end="); print_cell(gen->end); - nl(); + std::cout << name << ": "; + std::cout << "Start=" << gen->start; + std::cout << ", size=" << gen->size; + std::cout << ", end=" << gen->end; + std::cout << std::endl; } void factor_vm::dump_generations() @@ -225,11 +229,9 @@ void factor_vm::dump_generations() dump_generation("Aging",data->aging); dump_generation("Tenured",data->tenured); - print_string("Cards: base="); - print_cell((cell)data->cards); - print_string(", size="); - print_cell((cell)(data->cards_end - data->cards)); - nl(); + std::cout << "Cards:"; + std::cout << "base=" << (cell)data->cards << ", "; + std::cout << "size=" << (cell)(data->cards_end - data->cards) << std::endl; } void factor_vm::dump_objects(cell type) @@ -242,10 +244,9 @@ void factor_vm::dump_objects(cell type) { if(type == TYPE_COUNT || tagged(obj).type_p(type)) { - print_cell_hex_pad(obj); - print_string(" "); + std::cout << padded_address(obj) << " "; print_nested_obj(obj,2); - nl(); + std::cout << std::endl; } } @@ -263,10 +264,9 @@ struct data_references_finder { { if(look_for == *scan) { - print_cell_hex_pad(obj); - print_string(" "); + std::cout << padded_address(obj) << " "; parent->print_nested_obj(obj,2); - nl(); + std::cout << std::endl; } } }; @@ -311,9 +311,9 @@ struct code_block_printer { status = "allocated"; } - print_cell_hex((cell)scan); print_string(" "); - print_cell_hex(size); print_string(" "); - print_string(status); print_string("\n"); + std::cout << std::hex << (cell)scan << std::dec << " "; + std::cout << std::hex << size << std::dec << " "; + std::cout << status << std::endl; } }; @@ -322,40 +322,40 @@ void factor_vm::dump_code_heap() { code_block_printer printer(this); code->allocator->iterate(printer); - print_cell(printer.reloc_size); print_string(" bytes of relocation data\n"); - print_cell(printer.literal_size); print_string(" bytes of literal data\n"); + std::cout << printer.reloc_size << " bytes of relocation data\n"; + std::cout << printer.literal_size << " bytes of literal data\n"; } void factor_vm::factorbug() { if(fep_disabled) { - print_string("Low level debugger disabled\n"); + std::cout << "Low level debugger disabled\n"; exit(1); } /* open_console(); */ - print_string("Starting low level debugger...\n"); - print_string(" Basic commands:\n"); - print_string("q -- continue executing Factor - NOT SAFE\n"); - print_string("im -- save image to fep.image\n"); - print_string("x -- exit Factor\n"); - print_string(" Advanced commands:\n"); - print_string("d -- dump memory\n"); - print_string("u -- dump object at tagged \n"); - print_string(". -- print object at tagged \n"); - print_string("t -- toggle output trimming\n"); - print_string("s r -- dump data, retain stacks\n"); - print_string(".s .r .c -- print data, retain, call stacks\n"); - print_string("e -- dump environment\n"); - print_string("g -- dump generations\n"); - print_string("data -- data heap dump\n"); - print_string("words -- words dump\n"); - print_string("tuples -- tuples dump\n"); - print_string("refs -- find data heap references to object\n"); - print_string("push -- push object on data stack - NOT SAFE\n"); - print_string("code -- code heap dump\n"); + std::cout << "Starting low level debugger...\n"; + std::cout << " Basic commands:\n"; + std::cout << "q -- continue executing Factor - NOT SAFE\n"; + std::cout << "im -- save image to fep.image\n"; + std::cout << "x -- exit Factor\n"; + std::cout << " Advanced commands:\n"; + std::cout << "d -- dump memory\n"; + std::cout << "u -- dump object at tagged \n"; + std::cout << ". -- print object at tagged \n"; + std::cout << "t -- toggle output trimming\n"; + std::cout << "s r -- dump data, retain stacks\n"; + std::cout << ".s .r .c -- print data, retain, call stacks\n"; + std::cout << "e -- dump environment\n"; + std::cout << "g -- dump generations\n"; + std::cout << "data -- data heap dump\n"; + std::cout << "words -- words dump\n"; + std::cout << "tuples -- tuples dump\n"; + std::cout << "refs -- find data heap references to object\n"; + std::cout << "push -- push object on data stack - NOT SAFE\n"; + std::cout << "code -- code heap dump\n"; bool seen_command = false; @@ -363,7 +363,7 @@ void factor_vm::factorbug() { char cmd[1024]; - print_string("READY\n"); + std::cout << "READY\n"; fflush(stdout); if(scanf("%1000s",cmd) <= 0) @@ -403,7 +403,7 @@ void factor_vm::factorbug() { cell addr = read_cell_hex(); print_obj(addr); - print_string("\n"); + std::cout << std::endl; } else if(strcmp(cmd,"t") == 0) full_output = !full_output; @@ -436,9 +436,9 @@ void factor_vm::factorbug() else if(strcmp(cmd,"refs") == 0) { cell addr = read_cell_hex(); - print_string("Data heap references:\n"); + std::cout << "Data heap references:\n"; find_data_references(addr); - nl(); + std::cout << std::endl; } else if(strcmp(cmd,"words") == 0) dump_objects(WORD_TYPE); @@ -452,14 +452,14 @@ void factor_vm::factorbug() else if(strcmp(cmd,"code") == 0) dump_code_heap(); else - print_string("unknown command\n"); + std::cout << "unknown command\n"; } } void factor_vm::primitive_die() { - print_string("The die word was called by the library. Unless you called it yourself,\n"); - print_string("you have triggered a bug in Factor. Please report.\n"); + std::cout << "The die word was called by the library. Unless you called it yourself,\n"; + std::cout << "you have triggered a bug in Factor. Please report.\n"; factorbug(); } diff --git a/vm/errors.cpp b/vm/errors.cpp index a1fc71ffbc..6268ca695d 100755 --- a/vm/errors.cpp +++ b/vm/errors.cpp @@ -5,22 +5,24 @@ namespace factor void fatal_error(const char *msg, cell tagged) { - print_string("fatal_error: "); print_string(msg); - print_string(": "); print_cell_hex(tagged); nl(); + std::cout << "fatal_error: " << msg; + std::cout << ": " << std::hex << tagged << std::dec; + std::cout << std::endl; exit(1); } void critical_error(const char *msg, cell tagged) { - print_string("You have triggered a bug in Factor. Please report.\n"); - print_string("critical_error: "); print_string(msg); - print_string(": "); print_cell_hex(tagged); nl(); + std::cout << "You have triggered a bug in Factor. Please report.\n"; + std::cout << "critical_error: " << msg; + std::cout << ": " << std::hex << tagged << std::dec; + std::cout << std::endl; tls_vm()->factorbug(); } void out_of_memory() { - print_string("Out of memory\n\n"); + std::cout << "Out of memory\n\n"; tls_vm()->dump_generations(); exit(1); } @@ -59,10 +61,10 @@ void factor_vm::throw_error(cell error, stack_frame *callstack_top) crash. */ else { - print_string("You have triggered a bug in Factor. Please report.\n"); - print_string("early_error: "); + std::cout << "You have triggered a bug in Factor. Please report.\n"; + std::cout << "early_error: "; print_obj(error); - nl(); + std::cout << std::endl; factorbug(); } } diff --git a/vm/factor.cpp b/vm/factor.cpp index 9c87c0a9a7..f3eb351d94 100755 --- a/vm/factor.cpp +++ b/vm/factor.cpp @@ -38,6 +38,7 @@ void factor_vm::default_parameters(vm_parameters *p) p->max_pic_size = 3; p->fep = false; + p->verbosegc = false; p->signals = true; #ifdef WINDOWS @@ -86,6 +87,7 @@ void factor_vm::init_parameters_from_args(vm_parameters *p, int argc, vm_char ** else if(factor_arg(arg,STRING_LITERAL("-callbacks=%d"),&p->callback_size)); else if(STRCMP(arg,STRING_LITERAL("-fep")) == 0) p->fep = true; else if(STRCMP(arg,STRING_LITERAL("-nosignals")) == 0) p->signals = false; + else if(STRCMP(arg,STRING_LITERAL("-verbosegc")) == 0) p->verbosegc = true; else if(STRNCMP(arg,STRING_LITERAL("-i="),3) == 0) p->image_path = arg + 3; else if(STRCMP(arg,STRING_LITERAL("-console")) == 0) p->console = true; } @@ -94,14 +96,13 @@ void factor_vm::init_parameters_from_args(vm_parameters *p, int argc, vm_char ** /* Do some initialization that we do once only */ void factor_vm::do_stage1_init() { - print_string("*** Stage 2 early init... "); + std::cout << "*** Stage 2 early init... "; fflush(stdout); compile_all_words(); userenv[STAGE2_ENV] = true_object; - print_string("done\n"); - fflush(stdout); + std::cout << "done\n"; } void factor_vm::init_factor(vm_parameters *p) @@ -141,6 +142,8 @@ void factor_vm::init_factor(vm_parameters *p) if(p->signals) init_signals(); + verbosegc = p->verbosegc; + if(p->console) open_console(); diff --git a/vm/gc.cpp b/vm/gc.cpp index 6cb99b6da0..706831136b 100755 --- a/vm/gc.cpp +++ b/vm/gc.cpp @@ -37,6 +37,9 @@ void factor_vm::gc(gc_op op, current_gc = new gc_state(op); + if(verbosegc) + std::cout << "GC requested, op=" << op << std::endl; + /* Keep trying to GC higher and higher generations until we don't run out of space */ if(setjmp(current_gc->gc_unwind)) @@ -60,6 +63,9 @@ void factor_vm::gc(gc_op op, critical_error("Bad GC op\n",op); break; } + + if(verbosegc) + std::cout << "GC rewind, op=" << op << std::endl; } switch(current_gc->op) @@ -91,11 +97,14 @@ void factor_vm::gc(gc_op op, delete current_gc; current_gc = NULL; + + if(verbosegc) + std::cout << "GC done, op=" << op << std::endl; } void factor_vm::primitive_minor_gc() { - gc(collect_nursery_op, + gc(collect_full_op, 0, /* requested size */ true, /* trace contexts? */ false /* compact code heap? */); diff --git a/vm/image.cpp b/vm/image.cpp index ee0a1064ed..845ca6c1bc 100755 --- a/vm/image.cpp +++ b/vm/image.cpp @@ -31,11 +31,8 @@ void factor_vm::load_data_heap(FILE *file, image_header *h, vm_parameters *p) if((cell)bytes_read != h->data_size) { - print_string("truncated image: "); - print_fixnum(bytes_read); - print_string(" bytes read, "); - print_cell(h->data_size); - print_string(" bytes expected\n"); + std::cout << "truncated image: " << bytes_read << " bytes read, "; + std::cout << h->data_size << " bytes expected\n"; fatal_error("load_data_heap failed",0); } @@ -54,11 +51,8 @@ void factor_vm::load_code_heap(FILE *file, image_header *h, vm_parameters *p) size_t bytes_read = fread(code->allocator->first_block(),1,h->code_size,file); if(bytes_read != h->code_size) { - print_string("truncated image: "); - print_fixnum(bytes_read); - print_string(" bytes read, "); - print_cell(h->code_size); - print_string(" bytes expected\n"); + std::cout << "truncated image: " << bytes_read << " bytes read, "; + std::cout << h->code_size << " bytes expected\n"; fatal_error("load_code_heap failed",0); } } @@ -243,8 +237,8 @@ void factor_vm::load_image(vm_parameters *p) FILE *file = OPEN_READ(p->image_path); if(file == NULL) { - print_string("Cannot open image file: "); print_native_string(p->image_path); nl(); - print_string(strerror(errno)); nl(); + std::cout << "Cannot open image file: " << p->image_path << std::endl; + std::cout << strerror(errno) << std::endl; exit(1); } @@ -281,8 +275,8 @@ bool factor_vm::save_image(const vm_char *filename) file = OPEN_WRITE(filename); if(file == NULL) { - print_string("Cannot open image file: "); print_native_string(filename); nl(); - print_string(strerror(errno)); nl(); + std::cout << "Cannot open image file: " << filename << std::endl; + std::cout << strerror(errno) << std::endl; return false; } @@ -309,9 +303,7 @@ bool factor_vm::save_image(const vm_char *filename) if(fclose(file)) ok = false; if(!ok) - { - print_string("save-image failed: "); print_string(strerror(errno)); nl(); - } + std::cout << "save-image failed: " << strerror(errno) << std::endl; return ok; } diff --git a/vm/image.hpp b/vm/image.hpp index 62ab7e8392..127709ffb7 100755 --- a/vm/image.hpp +++ b/vm/image.hpp @@ -35,6 +35,7 @@ struct vm_parameters { cell young_size, aging_size, tenured_size; cell code_size; bool fep; + bool verbosegc; bool console; bool signals; cell max_pic_size; diff --git a/vm/inline_cache.cpp b/vm/inline_cache.cpp index 772631d1ce..8d8709fdea 100755 --- a/vm/inline_cache.cpp +++ b/vm/inline_cache.cpp @@ -233,10 +233,10 @@ void *factor_vm::inline_cache_miss(cell return_address) set_call_target(return_address,xt); #ifdef PIC_DEBUG - printf("Updated %s call site 0x%lx with 0x%lx\n", - tail_call_site_p(return_address) ? "tail" : "non-tail", - return_address, - (cell)xt); + std::cout << "Updated " + << (tail_call_site_p(return_address) ? "tail" : "non-tail") + << " call site 0x" << std::hex << return_address << std::dec + << " with " << std::hex << (cell)xt << std::dec; #endif return xt; diff --git a/vm/jit.hpp b/vm/jit.hpp index 4928962fc6..1940da9c7c 100644 --- a/vm/jit.hpp +++ b/vm/jit.hpp @@ -43,7 +43,7 @@ struct jit { void emit_subprimitive(cell word_) { gc_root word(word_,parent); gc_root code_pair(word->subprimitive,parent); - literals.append(parent->untag(array_nth(code_pair.untagged(),0))); + literals.append(untag(array_nth(code_pair.untagged(),0))); emit(array_nth(code_pair.untagged(),1)); } diff --git a/vm/layouts.hpp b/vm/layouts.hpp index 76581621ca..c90be1b2dd 100644 --- a/vm/layouts.hpp +++ b/vm/layouts.hpp @@ -112,25 +112,25 @@ struct header { explicit header(cell value_) : value(value_ << TAG_BITS) {} - void check_header() + void check_header() const { #ifdef FACTOR_DEBUG assert(TAG(value) == FIXNUM_TYPE && untag_fixnum(value) < TYPE_COUNT); #endif } - cell hi_tag() + cell hi_tag() const { check_header(); return value >> TAG_BITS; } - bool forwarding_pointer_p() + bool forwarding_pointer_p() const { return TAG(value) == GC_COLLECTED; } - object *forwarding_pointer() + object *forwarding_pointer() const { return (object *)UNTAG(value); } @@ -147,13 +147,13 @@ struct object { NO_TYPE_CHECK; header h; - cell size(); + cell size() const; - cell *slots() { return (cell *)this; } + cell *slots() const { return (cell *)this; } /* Only valid for objects in tenured space; must fast to free_heap_block to do anything with it if its free */ - bool free_p() + bool free_p() const { return h.value & 1 == 1; } @@ -166,7 +166,7 @@ struct array : public object { /* tagged */ cell capacity; - cell *data() { return (cell *)(this + 1); } + cell *data() const { return (cell *)(this + 1); } }; /* These are really just arrays, but certain elements have special @@ -187,7 +187,7 @@ struct bignum : public object { /* tagged */ cell capacity; - cell *data() { return (cell *)(this + 1); } + cell *data() const { return (cell *)(this + 1); } }; struct byte_array : public object { @@ -201,7 +201,7 @@ struct byte_array : public object { cell padding1; #endif - template Scalar *data() { return (Scalar *)(this + 1); } + template Scalar *data() const { return (Scalar *)(this + 1); } }; /* Assembly code makes assumptions about the layout of this struct */ @@ -214,7 +214,9 @@ struct string : public object { /* tagged */ cell hashcode; - u8 *data() { return (u8 *)(this + 1); } + u8 *data() const { return (u8 *)(this + 1); } + + cell nth(cell i) const; }; /* The compiled code heap is structured into blocks. */ @@ -222,12 +224,12 @@ struct heap_block { cell header; - bool free_p() + bool free_p() const { return header & 1 == 1; } - cell size() + cell size() const { cell bytes = header >> 3; #ifdef FACTOR_DEBUG @@ -258,12 +260,12 @@ struct code_block : public heap_block cell literals; /* tagged pointer to array or f */ cell relocation; /* tagged pointer to byte-array or f */ - void *xt() + void *xt() const { return (void *)(this + 1); } - code_block_type type() + code_block_type type() const { return (code_block_type)((header >> 1) & 0x3); } @@ -273,12 +275,12 @@ struct code_block : public heap_block header = ((header & ~0x7) | (type << 1)); } - bool pic_p() + bool pic_p() const { return type() == code_block_pic; } - bool optimized_p() + bool optimized_p() const { return type() == code_block_optimized; } @@ -376,13 +378,13 @@ struct callstack : public object { /* tagged */ cell length; - stack_frame *frame_at(cell offset) + stack_frame *frame_at(cell offset) const { return (stack_frame *)((char *)(this + 1) + offset); } - stack_frame *top() { return (stack_frame *)(this + 1); } - stack_frame *bottom() { return (stack_frame *)((cell)(this + 1) + untag_fixnum(length)); } + stack_frame *top() const { return (stack_frame *)(this + 1); } + stack_frame *bottom() const { return (stack_frame *)((cell)(this + 1) + untag_fixnum(length)); } }; struct tuple : public object { @@ -390,7 +392,7 @@ struct tuple : public object { /* tagged layout */ cell layout; - cell *data() { return (cell *)(this + 1); } + cell *data() const { return (cell *)(this + 1); } }; } diff --git a/vm/master.hpp b/vm/master.hpp index 0282a0597d..9168cecce4 100755 --- a/vm/master.hpp +++ b/vm/master.hpp @@ -28,6 +28,7 @@ #include #include #include +#include /* Forward-declare this since it comes up in function prototypes */ namespace factor diff --git a/vm/os-windows.hpp b/vm/os-windows.hpp index b12ebd0610..403842b2cb 100644 --- a/vm/os-windows.hpp +++ b/vm/os-windows.hpp @@ -37,8 +37,6 @@ typedef wchar_t vm_char; #define OPEN_READ(path) _wfopen(path,L"rb") #define OPEN_WRITE(path) _wfopen(path,L"wb") -#define print_native_string(string) wprintf(L"%s",string) - /* Difference between Jan 1 00:00:00 1601 and Jan 1 00:00:00 1970 */ #define EPOCH_OFFSET 0x019db1ded53e8000LL diff --git a/vm/quotations.cpp b/vm/quotations.cpp index e06b5c23d5..46087abeab 100755 --- a/vm/quotations.cpp +++ b/vm/quotations.cpp @@ -88,7 +88,7 @@ bool quotation_jit::stack_frame_p() switch(tagged(obj).type()) { case WORD_TYPE: - if(!parent->to_boolean(parent->untag(obj)->subprimitive)) + if(!parent->to_boolean(untag(obj)->subprimitive)) return true; break; case QUOTATION_TYPE: @@ -112,7 +112,7 @@ void quotation_jit::emit_quot(cell quot_) { gc_root quot(quot_,parent); - array *elements = parent->untag(quot->array); + array *elements = untag(quot->array); /* If the quotation consists of a single word, compile a direct call to the word. */ diff --git a/vm/strings.cpp b/vm/strings.cpp index 23fa75acca..3022611319 100644 --- a/vm/strings.cpp +++ b/vm/strings.cpp @@ -3,20 +3,20 @@ namespace factor { -cell factor_vm::string_nth(string* str, cell index) +cell string::nth(cell index) const { /* If high bit is set, the most significant 16 bits of the char come from the aux vector. The least significant bit of the corresponding aux vector entry is negated, so that we can XOR the two components together and get the original code point back. */ - cell lo_bits = str->data()[index]; + cell lo_bits = data()[index]; if((lo_bits & 0x80) == 0) return lo_bits; else { - byte_array *aux = untag(str->aux); + byte_array *aux = untag(this->aux); cell hi_bits = aux->data()[index]; return (hi_bits << 7) ^ lo_bits; } @@ -166,7 +166,7 @@ void factor_vm::primitive_string_nth() { string *str = untag(dpop()); cell index = untag_fixnum(dpop()); - dpush(tag_fixnum(string_nth(str,index))); + dpush(tag_fixnum(str->nth(index))); } void factor_vm::primitive_set_string_nth_fast() diff --git a/vm/strings.hpp b/vm/strings.hpp index 727ca8516e..54ff981d99 100644 --- a/vm/strings.hpp +++ b/vm/strings.hpp @@ -1,7 +1,7 @@ namespace factor { -inline static cell string_capacity(string *str) +inline static cell string_capacity(const string *str) { return untag_fixnum(str->length); } diff --git a/vm/tagged.hpp b/vm/tagged.hpp index 02fcdee26c..c5325542cb 100755 --- a/vm/tagged.hpp +++ b/vm/tagged.hpp @@ -75,7 +75,7 @@ template Type *factor_vm::untag_check(cell value) return tagged(value).untag_check(this); } -template Type *factor_vm::untag(cell value) +template Type *untag(cell value) { return tagged(value).untagged(); } diff --git a/vm/utilities.cpp b/vm/utilities.cpp index 0595430283..8f063a9ad4 100755 --- a/vm/utilities.cpp +++ b/vm/utilities.cpp @@ -11,38 +11,6 @@ vm_char *safe_strdup(const vm_char *str) return ptr; } -/* We don't use printf directly, because format directives are not portable. -Instead we define the common cases here. */ -void nl() -{ - fputs("\n",stdout); -} - -void print_string(const char *str) -{ - fputs(str,stdout); -} - -void print_cell(cell x) -{ - printf(CELL_FORMAT,x); -} - -void print_cell_hex(cell x) -{ - printf(CELL_HEX_FORMAT,x); -} - -void print_cell_hex_pad(cell x) -{ - printf(CELL_HEX_PAD_FORMAT,x); -} - -void print_fixnum(fixnum x) -{ - printf(FIXNUM_FORMAT,x); -} - cell read_cell_hex() { cell cell; diff --git a/vm/utilities.hpp b/vm/utilities.hpp index f93fe13f78..497e1a3bfb 100755 --- a/vm/utilities.hpp +++ b/vm/utilities.hpp @@ -1,11 +1,6 @@ namespace factor { vm_char *safe_strdup(const vm_char *str); - void print_string(const char *str); - void nl(); - void print_cell(cell x); - void print_cell_hex(cell x); void print_cell_hex_pad(cell x); - void print_fixnum(fixnum x); cell read_cell_hex(); } diff --git a/vm/vm.hpp b/vm/vm.hpp index 615efe35ed..86ed092ff7 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -46,6 +46,9 @@ struct factor_vm /* GC is off during heap walking */ bool gc_off; + /* GC logging */ + bool verbosegc; + /* Data heap */ data_heap *data; @@ -327,7 +330,7 @@ struct factor_vm inline void set_array_nth(array *array, cell slot, cell value); //strings - cell string_nth(string* str, cell index); + cell string_nth(const string *str, cell index); void set_string_nth_fast(string *str, cell index, cell ch); void set_string_nth_slow(string *str_, cell index, cell ch); void set_string_nth(string *str, cell index, cell ch); @@ -450,8 +453,9 @@ struct factor_vm inline double untag_float_check(cell tagged); inline fixnum float_to_fixnum(cell tagged); inline double fixnum_to_float(cell tagged); + + // tagged template Type *untag_check(cell value); - template Type *untag(cell value); //io void init_c_io(); From 606a805d7d94b1a44a01c15c17bafdfc728e57d4 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Wed, 21 Oct 2009 23:24:35 -0500 Subject: [PATCH 15/74] vm: debugging mark and sweep --- vm/arrays.hpp | 1 - vm/free_list_allocator.hpp | 8 ++++---- vm/gc.cpp | 14 +++++++------- vm/image.cpp | 4 ++-- vm/local_roots.hpp | 2 +- vm/mark_bits.hpp | 11 ++++++++++- vm/vm.hpp | 12 ------------ 7 files changed, 24 insertions(+), 28 deletions(-) diff --git a/vm/arrays.hpp b/vm/arrays.hpp index 48be881230..6063269e7f 100755 --- a/vm/arrays.hpp +++ b/vm/arrays.hpp @@ -15,7 +15,6 @@ inline void factor_vm::set_array_nth(array *array, cell slot, cell value) #ifdef FACTOR_DEBUG assert(slot < array_capacity(array)); assert(array->h.hi_tag() == ARRAY_TYPE); - check_tagged_pointer(value); #endif cell *slot_ptr = &array->data()[slot]; *slot_ptr = value; diff --git a/vm/free_list_allocator.hpp b/vm/free_list_allocator.hpp index efdad508cb..8332399279 100644 --- a/vm/free_list_allocator.hpp +++ b/vm/free_list_allocator.hpp @@ -22,7 +22,7 @@ template struct free_list_allocator { Block *next_block_after(Block *block); void clear_free_list(); void add_to_free_list(free_heap_block *block); - void build_free_list(cell size); + void initial_free_list(cell size); void assert_free_block(free_heap_block *block); free_heap_block *find_free_block(cell size); free_heap_block *split_free_block(free_heap_block *block, cell size); @@ -40,7 +40,7 @@ template free_list_allocator::free_list_allocator(cell size_, cell start_) : size(size_), start(start_), end(start_ + size_), state(mark_bits(size_,start_)) { - clear_free_list(); + initial_free_list(0); } template void free_list_allocator::clear_free_list() @@ -85,7 +85,7 @@ template void free_list_allocator::add_to_free_list(free_ /* Called after reading the heap from the image file, and after heap compaction. Makes a free list consisting of one free block, at the very end. */ -template void free_list_allocator::build_free_list(cell size) +template void free_list_allocator::initial_free_list(cell size) { clear_free_list(); if(size != this->size) @@ -345,7 +345,7 @@ void free_list_allocator::compact(Iterator &iter) /* Now update the free list; there will be a single free block at the end */ - this->build_free_list((cell)compactor.address - this->start); + this->initial_free_list((cell)compactor.address - this->start); } template diff --git a/vm/gc.cpp b/vm/gc.cpp index 706831136b..6b3ec80481 100755 --- a/vm/gc.cpp +++ b/vm/gc.cpp @@ -60,12 +60,12 @@ void factor_vm::gc(gc_op op, current_gc->op = collect_growing_heap_op; break; default: - critical_error("Bad GC op\n",op); + critical_error("Bad GC op",current_gc->op); break; } if(verbosegc) - std::cout << "GC rewind, op=" << op << std::endl; + std::cout << "GC rewind, op=" << current_gc->op << std::endl; } switch(current_gc->op) @@ -91,20 +91,20 @@ void factor_vm::gc(gc_op op, record_gc_stats(&gc_stats.full_stats); break; default: - critical_error("Bad GC op\n",op); + critical_error("Bad GC op\n",current_gc->op); break; } + if(verbosegc) + std::cout << "GC done, op=" << current_gc->op << std::endl; + delete current_gc; current_gc = NULL; - - if(verbosegc) - std::cout << "GC done, op=" << op << std::endl; } void factor_vm::primitive_minor_gc() { - gc(collect_full_op, + gc(collect_nursery_op, 0, /* requested size */ true, /* trace contexts? */ false /* compact code heap? */); diff --git a/vm/image.cpp b/vm/image.cpp index 845ca6c1bc..f5879e7a32 100755 --- a/vm/image.cpp +++ b/vm/image.cpp @@ -36,7 +36,7 @@ void factor_vm::load_data_heap(FILE *file, image_header *h, vm_parameters *p) fatal_error("load_data_heap failed",0); } - data->tenured->build_free_list(h->data_size); + data->tenured->initial_free_list(h->data_size); } void factor_vm::load_code_heap(FILE *file, image_header *h, vm_parameters *p) @@ -57,7 +57,7 @@ void factor_vm::load_code_heap(FILE *file, image_header *h, vm_parameters *p) } } - code->allocator->build_free_list(h->code_size); + code->allocator->initial_free_list(h->code_size); } void factor_vm::data_fixup(cell *handle, cell data_relocation_base) diff --git a/vm/local_roots.hpp b/vm/local_roots.hpp index 6ae059f4c4..58142be8f2 100644 --- a/vm/local_roots.hpp +++ b/vm/local_roots.hpp @@ -6,7 +6,7 @@ struct gc_root : public tagged { factor_vm *parent; - void push() { parent->check_tagged_pointer(tagged::value()); parent->gc_locals.push_back((cell)this); } + void push() { parent->gc_locals.push_back((cell)this); } explicit gc_root(cell value_,factor_vm *vm) : tagged(value_),parent(vm) { push(); } explicit gc_root(Type *value_, factor_vm *vm) : tagged(value_),parent(vm) { push(); } diff --git a/vm/mark_bits.hpp b/vm/mark_bits.hpp index 161a7fd755..279c04a23a 100644 --- a/vm/mark_bits.hpp +++ b/vm/mark_bits.hpp @@ -81,12 +81,21 @@ template struct mark_bits { bits[start.first] |= start_mask ^ end_mask; else { +#ifdef FACTOR_DEBUG + assert(start.first < bits_size); +#endif bits[start.first] |= ~start_mask; for(cell index = start.first + 1; index < end.first; index++) bits[index] = (u64)-1; - bits[end.first] |= end_mask; + if(end_mask != 0) + { +#ifdef FACTOR_DEBUG + assert(end.first < bits_size); +#endif + bits[end.first] |= end_mask; + } } } diff --git a/vm/vm.hpp b/vm/vm.hpp index 86ed092ff7..40dcb4f3bc 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -280,18 +280,6 @@ struct factor_vm #endif } - inline void check_tagged_pointer(cell tagged) - { - #ifdef FACTOR_DEBUG - if(!immediate_p(tagged)) - { - object *obj = untag(tagged); - check_data_pointer(obj); - obj->h.hi_tag(); - } - #endif - } - // generic arrays template Array *allot_uninitialized_array(cell capacity); template bool reallot_array_in_place_p(Array *array, cell capacity); From 6597e0ea3ab445253264595efb108cda462d97f7 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Thu, 22 Oct 2009 23:41:02 -0500 Subject: [PATCH 16/74] vm/mark_bits.hpp: fix incorrect constructor argument order --- vm/mark_bits.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vm/mark_bits.hpp b/vm/mark_bits.hpp index 279c04a23a..cd739346f0 100644 --- a/vm/mark_bits.hpp +++ b/vm/mark_bits.hpp @@ -5,8 +5,8 @@ const int block_granularity = 16; const int forwarding_granularity = 64; template struct mark_bits { - cell start; cell size; + cell start; cell bits_size; u64 *marked; cell *forwarding; @@ -21,9 +21,9 @@ template struct mark_bits { memset(forwarding,0,bits_size * sizeof(cell)); } - explicit mark_bits(cell start_, cell size_) : - start(start_), + explicit mark_bits(cell size_, cell start_) : size(size_), + start(start_), bits_size(size / block_granularity / forwarding_granularity), marked(new u64[bits_size]), forwarding(new cell[bits_size]) From 810e309e0cea10a0836cb4c44822b08a1c71c981 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Fri, 23 Oct 2009 00:33:53 -0500 Subject: [PATCH 17/74] vm: rename userenv to special_objects --- vm/callbacks.cpp | 4 +-- vm/collector.hpp | 3 ++- vm/contexts.cpp | 12 ++++----- vm/contexts.hpp | 2 +- vm/debug.cpp | 5 ++-- vm/dispatch.cpp | 10 +++---- vm/errors.cpp | 6 ++--- vm/factor.cpp | 26 +++++++++--------- vm/image.cpp | 18 ++++++------- vm/image.hpp | 2 +- vm/inline_cache.cpp | 8 +++--- vm/io.cpp | 6 ++--- vm/jit.cpp | 4 +-- vm/jit.hpp | 29 ++++++++++++-------- vm/os-macosx.mm | 2 +- vm/profiler.cpp | 2 +- vm/quotations.cpp | 46 +++++++++++++++---------------- vm/run.cpp | 4 +-- vm/run.hpp | 66 ++++++++++++++++++++++----------------------- vm/vm.hpp | 2 +- vm/words.cpp | 2 +- 21 files changed, 133 insertions(+), 126 deletions(-) diff --git a/vm/callbacks.cpp b/vm/callbacks.cpp index 599271555b..4fe19c0bc0 100644 --- a/vm/callbacks.cpp +++ b/vm/callbacks.cpp @@ -21,7 +21,7 @@ void factor_vm::init_callbacks(cell size) void callback_heap::update(callback *stub) { - tagged code_template(parent->userenv[CALLBACK_STUB]); + tagged code_template(parent->special_objects[CALLBACK_STUB]); cell rel_class = untag_fixnum(array_nth(code_template.untagged(),1)); cell offset = untag_fixnum(array_nth(code_template.untagged(),3)); @@ -35,7 +35,7 @@ void callback_heap::update(callback *stub) callback *callback_heap::add(code_block *compiled) { - tagged code_template(parent->userenv[CALLBACK_STUB]); + tagged code_template(parent->special_objects[CALLBACK_STUB]); tagged insns(array_nth(code_template.untagged(),0)); cell size = array_capacity(insns.untagged()); diff --git a/vm/collector.hpp b/vm/collector.hpp index 4479fc7b45..a1a7dc5695 100644 --- a/vm/collector.hpp +++ b/vm/collector.hpp @@ -135,7 +135,8 @@ template struct collector { trace_registered_locals(); trace_registered_bignums(); - for(int i = 0; i < USER_ENV; i++) trace_handle(&parent->userenv[i]); + for(cell i = 0; i < special_object_count; i++) + trace_handle(&parent->special_objects[i]); } void trace_contexts() diff --git a/vm/contexts.cpp b/vm/contexts.cpp index ce52555a21..7af7fdaa57 100644 --- a/vm/contexts.cpp +++ b/vm/contexts.cpp @@ -80,9 +80,9 @@ void factor_vm::nest_stacks(stack_frame *magic_frame) new_ctx->magic_frame = magic_frame; - /* save per-callback userenv */ - new_ctx->current_callback_save = userenv[CURRENT_CALLBACK_ENV]; - new_ctx->catchstack_save = userenv[CATCHSTACK_ENV]; + /* save per-callback special_objects */ + new_ctx->current_callback_save = special_objects[OBJ_CURRENT_CALLBACK]; + new_ctx->catchstack_save = special_objects[OBJ_CATCHSTACK]; new_ctx->next = ctx; ctx = new_ctx; @@ -102,9 +102,9 @@ void factor_vm::unnest_stacks() ds = ctx->datastack_save; rs = ctx->retainstack_save; - /* restore per-callback userenv */ - userenv[CURRENT_CALLBACK_ENV] = ctx->current_callback_save; - userenv[CATCHSTACK_ENV] = ctx->catchstack_save; + /* restore per-callback special_objects */ + special_objects[OBJ_CURRENT_CALLBACK] = ctx->current_callback_save; + special_objects[OBJ_CATCHSTACK] = ctx->catchstack_save; context *old_ctx = ctx; ctx = old_ctx->next; diff --git a/vm/contexts.hpp b/vm/contexts.hpp index f66b5d0fe2..aa6f9ec8ce 100644 --- a/vm/contexts.hpp +++ b/vm/contexts.hpp @@ -41,7 +41,7 @@ struct context { /* memory region holding current retain stack */ segment *retainstack_region; - /* saved userenv slots on entry to callback */ + /* saved special_objects slots on entry to callback */ cell catchstack_save; cell current_callback_save; diff --git a/vm/debug.cpp b/vm/debug.cpp index b37a7e6f82..91fb1ea1d3 100755 --- a/vm/debug.cpp +++ b/vm/debug.cpp @@ -419,9 +419,8 @@ void factor_vm::factorbug() print_callstack(); else if(strcmp(cmd,"e") == 0) { - int i; - for(i = 0; i < USER_ENV; i++) - dump_cell((cell)&userenv[i]); + for(cell i = 0; i < special_object_count; i++) + dump_cell((cell)&special_objects[i]); } else if(strcmp(cmd,"g") == 0) dump_generations(); diff --git a/vm/dispatch.cpp b/vm/dispatch.cpp index 0abde2e711..bbe86c0fd6 100755 --- a/vm/dispatch.cpp +++ b/vm/dispatch.cpp @@ -187,21 +187,21 @@ void quotation_jit::emit_mega_cache_lookup(cell methods_, fixnum index, cell cac emit_class_lookup(index,PIC_HI_TAG_TUPLE); /* Do a cache lookup. */ - emit_with(parent->userenv[MEGA_LOOKUP],cache.value()); + emit_with(parent->special_objects[MEGA_LOOKUP],cache.value()); /* If we end up here, the cache missed. */ - emit(parent->userenv[JIT_PROLOG]); + emit(parent->special_objects[JIT_PROLOG]); /* Push index, method table and cache on the stack. */ push(methods.value()); push(tag_fixnum(index)); push(cache.value()); - word_call(parent->userenv[MEGA_MISS_WORD]); + word_call(parent->special_objects[MEGA_MISS_WORD]); /* Now the new method has been stored into the cache, and its on the stack. */ - emit(parent->userenv[JIT_EPILOG]); - emit(parent->userenv[JIT_EXECUTE_JUMP]); + emit(parent->special_objects[JIT_EPILOG]); + emit(parent->special_objects[JIT_EXECUTE_JUMP]); } } diff --git a/vm/errors.cpp b/vm/errors.cpp index 148799446a..3161f625cd 100755 --- a/vm/errors.cpp +++ b/vm/errors.cpp @@ -31,7 +31,7 @@ void factor_vm::throw_error(cell error, stack_frame *callstack_top) { /* If the error handler is set, we rewind any C stack frames and pass the error to user-space. */ - if(!current_gc && to_boolean(userenv[BREAK_ENV])) + if(!current_gc && to_boolean(special_objects[OBJ_BREAK])) { /* If error was thrown during heap scan, we re-enable the GC */ gc_off = false; @@ -55,7 +55,7 @@ void factor_vm::throw_error(cell error, stack_frame *callstack_top) else callstack_top = ctx->callstack_top; - throw_impl(userenv[BREAK_ENV],callstack_top,this); + throw_impl(special_objects[OBJ_BREAK],callstack_top,this); } /* Error was thrown in early startup before error handler is set, just crash. */ @@ -71,7 +71,7 @@ void factor_vm::throw_error(cell error, stack_frame *callstack_top) void factor_vm::general_error(vm_error_type error, cell arg1, cell arg2, stack_frame *callstack_top) { - throw_error(allot_array_4(userenv[ERROR_ENV], + throw_error(allot_array_4(special_objects[OBJ_ERROR], tag_fixnum(error),arg1,arg2),callstack_top); } diff --git a/vm/factor.cpp b/vm/factor.cpp index f3eb351d94..df27de84fd 100755 --- a/vm/factor.cpp +++ b/vm/factor.cpp @@ -100,7 +100,7 @@ void factor_vm::do_stage1_init() fflush(stdout); compile_all_words(); - userenv[STAGE2_ENV] = true_object; + special_objects[OBJ_STAGE2] = true_object; std::cout << "done\n"; } @@ -149,17 +149,17 @@ void factor_vm::init_factor(vm_parameters *p) init_profiler(); - userenv[CPU_ENV] = allot_alien(false_object,(cell)FACTOR_CPU_STRING); - userenv[OS_ENV] = allot_alien(false_object,(cell)FACTOR_OS_STRING); - userenv[CELL_SIZE_ENV] = tag_fixnum(sizeof(cell)); - userenv[EXECUTABLE_ENV] = allot_alien(false_object,(cell)p->executable_path); - userenv[ARGS_ENV] = false_object; - userenv[EMBEDDED_ENV] = false_object; + special_objects[OBJ_CPU] = allot_alien(false_object,(cell)FACTOR_CPU_STRING); + special_objects[OBJ_OS] = allot_alien(false_object,(cell)FACTOR_OS_STRING); + special_objects[OBJ_CELL_SIZE] = tag_fixnum(sizeof(cell)); + special_objects[OBJ_EXECUTABLE] = allot_alien(false_object,(cell)p->executable_path); + special_objects[OBJ_ARGS] = false_object; + special_objects[OBJ_EMBEDDED] = false_object; /* We can GC now */ gc_off = false; - if(!to_boolean(userenv[STAGE2_ENV])) + if(!to_boolean(special_objects[OBJ_STAGE2])) do_stage1_init(); } @@ -174,7 +174,7 @@ void factor_vm::pass_args_to_factor(int argc, vm_char **argv) } args.trim(); - userenv[ARGS_ENV] = args.elements.value(); + special_objects[OBJ_ARGS] = args.elements.value(); } void factor_vm::start_factor(vm_parameters *p) @@ -182,13 +182,13 @@ void factor_vm::start_factor(vm_parameters *p) if(p->fep) factorbug(); nest_stacks(NULL); - c_to_factor_toplevel(userenv[BOOT_ENV]); + c_to_factor_toplevel(special_objects[OBJ_BOOT]); unnest_stacks(); } char *factor_vm::factor_eval_string(char *string) { - char *(*callback)(char *) = (char *(*)(char *))alien_offset(userenv[EVAL_CALLBACK_ENV]); + char *(*callback)(char *) = (char *(*)(char *))alien_offset(special_objects[OBJ_EVAL_CALLBACK]); return callback(string); } @@ -199,13 +199,13 @@ void factor_vm::factor_eval_free(char *result) void factor_vm::factor_yield() { - void (*callback)() = (void (*)())alien_offset(userenv[YIELD_CALLBACK_ENV]); + void (*callback)() = (void (*)())alien_offset(special_objects[OBJ_YIELD_CALLBACK]); callback(); } void factor_vm::factor_sleep(long us) { - void (*callback)(long) = (void (*)(long))alien_offset(userenv[SLEEP_CALLBACK_ENV]); + void (*callback)(long) = (void (*)(long))alien_offset(special_objects[OBJ_SLEEP_CALLBACK]); callback(us); } diff --git a/vm/image.cpp b/vm/image.cpp index f5879e7a32..1b7debc2b2 100755 --- a/vm/image.cpp +++ b/vm/image.cpp @@ -6,7 +6,7 @@ namespace factor /* Certain special objects in the image are known to the runtime */ void factor_vm::init_objects(image_header *h) { - memcpy(userenv,h->userenv,sizeof(userenv)); + memcpy(special_objects,h->special_objects,sizeof(special_objects)); true_object = h->true_object; bignum_zero = h->bignum_zero; @@ -183,8 +183,8 @@ void factor_vm::relocate_object(object *object, where it is loaded, we need to fix up pointers in the image. */ void factor_vm::relocate_data(cell data_relocation_base, cell code_relocation_base) { - for(cell i = 0; i < USER_ENV; i++) - data_fixup(&userenv[i],data_relocation_base); + for(cell i = 0; i < special_object_count; i++) + data_fixup(&special_objects[i],data_relocation_base); data_fixup(&true_object,data_relocation_base); data_fixup(&bignum_zero,data_relocation_base); @@ -263,7 +263,7 @@ void factor_vm::load_image(vm_parameters *p) relocate_code(h.data_relocation_base); /* Store image path name */ - userenv[IMAGE_ENV] = allot_alien(false_object,(cell)p->image_path); + special_objects[OBJ_IMAGE] = allot_alien(false_object,(cell)p->image_path); } /* Save the current image to disk */ @@ -292,8 +292,8 @@ bool factor_vm::save_image(const vm_char *filename) h.bignum_pos_one = bignum_pos_one; h.bignum_neg_one = bignum_neg_one; - for(cell i = 0; i < USER_ENV; i++) - h.userenv[i] = (save_env_p(i) ? userenv[i] : false_object); + for(cell i = 0; i < special_object_count; i++) + h.special_objects[i] = (save_env_p(i) ? special_objects[i] : false_object); bool ok = true; @@ -326,9 +326,9 @@ void factor_vm::primitive_save_image_and_exit() gc_root path(dpop(),this); path.untag_check(this); - /* strip out userenv data which is set on startup anyway */ - for(cell i = 0; i < USER_ENV; i++) - if(!save_env_p(i)) userenv[i] = false_object; + /* strip out special_objects data which is set on startup anyway */ + for(cell i = 0; i < special_object_count; i++) + if(!save_env_p(i)) special_objects[i] = false_object; gc(collect_full_op, 0, /* requested size */ diff --git a/vm/image.hpp b/vm/image.hpp index 127709ffb7..3a5447c63b 100755 --- a/vm/image.hpp +++ b/vm/image.hpp @@ -25,7 +25,7 @@ struct image_header { /* tagged pointer to bignum -1 */ cell bignum_neg_one; /* Initial user environment */ - cell userenv[USER_ENV]; + cell special_objects[special_object_count]; }; struct vm_parameters { diff --git a/vm/inline_cache.cpp b/vm/inline_cache.cpp index 8d8709fdea..ee221c3797 100755 --- a/vm/inline_cache.cpp +++ b/vm/inline_cache.cpp @@ -86,9 +86,9 @@ void inline_cache_jit::emit_check(cell klass) { cell code_template; if(TAG(klass) == FIXNUM_TYPE && untag_fixnum(klass) < HEADER_TYPE) - code_template = parent->userenv[PIC_CHECK_TAG]; + code_template = parent->special_objects[PIC_CHECK_TAG]; else - code_template = parent->userenv[PIC_CHECK]; + code_template = parent->special_objects[PIC_CHECK]; emit_with(code_template,klass); } @@ -121,7 +121,7 @@ void inline_cache_jit::compile_inline_cache(fixnum index, /* Yes? Jump to method */ cell method = array_nth(cache_entries.untagged(),i + 1); - emit_with(parent->userenv[PIC_HIT],method); + emit_with(parent->special_objects[PIC_HIT],method); } /* Generate machine code to handle a cache miss, which ultimately results in @@ -133,7 +133,7 @@ void inline_cache_jit::compile_inline_cache(fixnum index, push(methods.value()); push(tag_fixnum(index)); push(cache_entries.value()); - word_special(parent->userenv[tail_call_p ? PIC_MISS_TAIL_WORD : PIC_MISS_WORD]); + word_special(parent->special_objects[tail_call_p ? PIC_MISS_TAIL_WORD : PIC_MISS_WORD]); } code_block *factor_vm::compile_inline_cache(fixnum index,cell generic_word_,cell methods_,cell cache_entries_,bool tail_call_p) diff --git a/vm/io.cpp b/vm/io.cpp index a75f41c5bf..bbcac0b849 100755 --- a/vm/io.cpp +++ b/vm/io.cpp @@ -16,9 +16,9 @@ normal operation. */ void factor_vm::init_c_io() { - userenv[STDIN_ENV] = allot_alien(false_object,(cell)stdin); - userenv[STDOUT_ENV] = allot_alien(false_object,(cell)stdout); - userenv[STDERR_ENV] = allot_alien(false_object,(cell)stderr); + special_objects[OBJ_STDIN] = allot_alien(false_object,(cell)stdin); + special_objects[OBJ_STDOUT] = allot_alien(false_object,(cell)stdout); + special_objects[OBJ_STDERR] = allot_alien(false_object,(cell)stderr); } void factor_vm::io_error() diff --git a/vm/jit.cpp b/vm/jit.cpp index 98212d2efe..2fa948e4d6 100644 --- a/vm/jit.cpp +++ b/vm/jit.cpp @@ -79,8 +79,8 @@ void jit::emit_with(cell code_template_, cell argument_) { void jit::emit_class_lookup(fixnum index, cell type) { - emit_with(parent->userenv[PIC_LOAD],tag_fixnum(-index * sizeof(cell))); - emit(parent->userenv[type]); + emit_with(parent->special_objects[PIC_LOAD],tag_fixnum(-index * sizeof(cell))); + emit(parent->special_objects[type]); } /* Facility to convert compiled code offsets to quotation offsets. diff --git a/vm/jit.hpp b/vm/jit.hpp index 1940da9c7c..9feade4cc1 100644 --- a/vm/jit.hpp +++ b/vm/jit.hpp @@ -21,26 +21,31 @@ struct jit { void literal(cell literal) { literals.add(literal); } void emit_with(cell code_template_, cell literal_); - void push(cell literal) { - emit_with(parent->userenv[JIT_PUSH_IMMEDIATE],literal); + void push(cell literal) + { + emit_with(parent->special_objects[JIT_PUSH_IMMEDIATE],literal); } - void word_jump(cell word_) { + void word_jump(cell word_) + { gc_root word(word_,parent); literal(tag_fixnum(xt_tail_pic_offset)); literal(word.value()); - emit(parent->userenv[JIT_WORD_JUMP]); + emit(parent->special_objects[JIT_WORD_JUMP]); } - void word_call(cell word) { - emit_with(parent->userenv[JIT_WORD_CALL],word); + void word_call(cell word) + { + emit_with(parent->special_objects[JIT_WORD_CALL],word); } - void word_special(cell word) { - emit_with(parent->userenv[JIT_WORD_SPECIAL],word); + void word_special(cell word) + { + emit_with(parent->special_objects[JIT_WORD_SPECIAL],word); } - void emit_subprimitive(cell word_) { + void emit_subprimitive(cell word_) + { gc_root word(word_,parent); gc_root code_pair(word->subprimitive,parent); literals.append(untag(array_nth(code_pair.untagged(),0))); @@ -49,7 +54,8 @@ struct jit { void emit_class_lookup(fixnum index, cell type); - fixnum get_position() { + fixnum get_position() + { if(computing_offset_p) { /* If this is still on, emit() didn't clear it, @@ -60,7 +66,8 @@ struct jit { return position; } - void set_position(fixnum position_) { + void set_position(fixnum position_) + { if(computing_offset_p) position = position_; } diff --git a/vm/os-macosx.mm b/vm/os-macosx.mm index 96f169bbcf..438957bd04 100644 --- a/vm/os-macosx.mm +++ b/vm/os-macosx.mm @@ -14,7 +14,7 @@ NS_DURING NS_VOIDRETURN; NS_HANDLER dpush(allot_alien(false_object,(cell)localException)); - quot = userenv[COCOA_EXCEPTION_ENV]; + quot = special_objects[OBJ_COCOA_EXCEPTION]; if(!tagged(quot).type_p(QUOTATION_TYPE)) { /* No Cocoa exception handler was registered, so diff --git a/vm/profiler.cpp b/vm/profiler.cpp index df9d9ee67b..5113b55cf7 100755 --- a/vm/profiler.cpp +++ b/vm/profiler.cpp @@ -14,7 +14,7 @@ code_block *factor_vm::compile_profiling_stub(cell word_) gc_root word(word_,this); jit jit(code_block_profiling,word.value(),this); - jit.emit_with(userenv[JIT_PROFILING],word.value()); + jit.emit_with(special_objects[JIT_PROFILING],word.value()); return jit.to_code_block(); } diff --git a/vm/quotations.cpp b/vm/quotations.cpp index 46087abeab..c65c0fe909 100755 --- a/vm/quotations.cpp +++ b/vm/quotations.cpp @@ -38,29 +38,29 @@ so this results in a big speedup for relatively little effort. */ bool quotation_jit::primitive_call_p(cell i, cell length) { - return (i + 2) == length && array_nth(elements.untagged(),i + 1) == parent->userenv[JIT_PRIMITIVE_WORD]; + return (i + 2) == length && array_nth(elements.untagged(),i + 1) == parent->special_objects[JIT_PRIMITIVE_WORD]; } bool quotation_jit::fast_if_p(cell i, cell length) { return (i + 3) == length && tagged(array_nth(elements.untagged(),i + 1)).type_p(QUOTATION_TYPE) - && array_nth(elements.untagged(),i + 2) == parent->userenv[JIT_IF_WORD]; + && array_nth(elements.untagged(),i + 2) == parent->special_objects[JIT_IF_WORD]; } bool quotation_jit::fast_dip_p(cell i, cell length) { - return (i + 2) <= length && array_nth(elements.untagged(),i + 1) == parent->userenv[JIT_DIP_WORD]; + return (i + 2) <= length && array_nth(elements.untagged(),i + 1) == parent->special_objects[JIT_DIP_WORD]; } bool quotation_jit::fast_2dip_p(cell i, cell length) { - return (i + 2) <= length && array_nth(elements.untagged(),i + 1) == parent->userenv[JIT_2DIP_WORD]; + return (i + 2) <= length && array_nth(elements.untagged(),i + 1) == parent->special_objects[JIT_2DIP_WORD]; } bool quotation_jit::fast_3dip_p(cell i, cell length) { - return (i + 2) <= length && array_nth(elements.untagged(),i + 1) == parent->userenv[JIT_3DIP_WORD]; + return (i + 2) <= length && array_nth(elements.untagged(),i + 1) == parent->special_objects[JIT_3DIP_WORD]; } bool quotation_jit::mega_lookup_p(cell i, cell length) @@ -68,13 +68,13 @@ bool quotation_jit::mega_lookup_p(cell i, cell length) return (i + 4) <= length && tagged(array_nth(elements.untagged(),i + 1)).type_p(FIXNUM_TYPE) && tagged(array_nth(elements.untagged(),i + 2)).type_p(ARRAY_TYPE) - && array_nth(elements.untagged(),i + 3) == parent->userenv[MEGA_LOOKUP_WORD]; + && array_nth(elements.untagged(),i + 3) == parent->special_objects[MEGA_LOOKUP_WORD]; } bool quotation_jit::declare_p(cell i, cell length) { return (i + 2) <= length - && array_nth(elements.untagged(),i + 1) == parent->userenv[JIT_DECLARE_WORD]; + && array_nth(elements.untagged(),i + 1) == parent->special_objects[JIT_DECLARE_WORD]; } bool quotation_jit::stack_frame_p() @@ -133,7 +133,7 @@ void quotation_jit::iterate_quotation() set_position(0); if(stack_frame) - emit(parent->userenv[JIT_PROLOG]); + emit(parent->special_objects[JIT_PROLOG]); cell i; cell length = array_capacity(elements.untagged()); @@ -152,23 +152,23 @@ void quotation_jit::iterate_quotation() if(parent->to_boolean(obj.as()->subprimitive)) emit_subprimitive(obj.value()); /* The (execute) primitive is special-cased */ - else if(obj.value() == parent->userenv[JIT_EXECUTE_WORD]) + else if(obj.value() == parent->special_objects[JIT_EXECUTE_WORD]) { if(i == length - 1) { - if(stack_frame) emit(parent->userenv[JIT_EPILOG]); + if(stack_frame) emit(parent->special_objects[JIT_EPILOG]); tail_call = true; - emit(parent->userenv[JIT_EXECUTE_JUMP]); + emit(parent->special_objects[JIT_EXECUTE_JUMP]); } else - emit(parent->userenv[JIT_EXECUTE_CALL]); + emit(parent->special_objects[JIT_EXECUTE_CALL]); } /* Everything else */ else { if(i == length - 1) { - if(stack_frame) emit(parent->userenv[JIT_EPILOG]); + if(stack_frame) emit(parent->special_objects[JIT_EPILOG]); tail_call = true; /* Inline cache misses are special-cased. The calling convention for tail @@ -178,8 +178,8 @@ void quotation_jit::iterate_quotation() the inline cache miss primitive, and we don't want to clobber the saved address. */ - if(obj.value() == parent->userenv[PIC_MISS_WORD] - || obj.value() == parent->userenv[PIC_MISS_TAIL_WORD]) + if(obj.value() == parent->special_objects[PIC_MISS_WORD] + || obj.value() == parent->special_objects[PIC_MISS_TAIL_WORD]) { word_special(obj.value()); } @@ -201,7 +201,7 @@ void quotation_jit::iterate_quotation() { literal(tag_fixnum(0)); literal(obj.value()); - emit(parent->userenv[JIT_PRIMITIVE]); + emit(parent->special_objects[JIT_PRIMITIVE]); i++; @@ -215,12 +215,12 @@ void quotation_jit::iterate_quotation() mutually recursive in the library, but both still work) */ if(fast_if_p(i,length)) { - if(stack_frame) emit(parent->userenv[JIT_EPILOG]); + if(stack_frame) emit(parent->special_objects[JIT_EPILOG]); tail_call = true; emit_quot(array_nth(elements.untagged(),i)); emit_quot(array_nth(elements.untagged(),i + 1)); - emit(parent->userenv[JIT_IF]); + emit(parent->special_objects[JIT_IF]); i += 2; } @@ -228,21 +228,21 @@ void quotation_jit::iterate_quotation() else if(fast_dip_p(i,length)) { emit_quot(obj.value()); - emit(parent->userenv[JIT_DIP]); + emit(parent->special_objects[JIT_DIP]); i++; } /* 2dip */ else if(fast_2dip_p(i,length)) { emit_quot(obj.value()); - emit(parent->userenv[JIT_2DIP]); + emit(parent->special_objects[JIT_2DIP]); i++; } /* 3dip */ else if(fast_3dip_p(i,length)) { emit_quot(obj.value()); - emit(parent->userenv[JIT_3DIP]); + emit(parent->special_objects[JIT_3DIP]); i++; } else @@ -276,8 +276,8 @@ void quotation_jit::iterate_quotation() set_position(length); if(stack_frame) - emit(parent->userenv[JIT_EPILOG]); - emit(parent->userenv[JIT_RETURN]); + emit(parent->special_objects[JIT_EPILOG]); + emit(parent->special_objects[JIT_RETURN]); } } diff --git a/vm/run.cpp b/vm/run.cpp index 79aca937ca..b6e3324502 100755 --- a/vm/run.cpp +++ b/vm/run.cpp @@ -6,14 +6,14 @@ namespace factor void factor_vm::primitive_getenv() { fixnum e = untag_fixnum(dpeek()); - drepl(userenv[e]); + drepl(special_objects[e]); } void factor_vm::primitive_setenv() { fixnum e = untag_fixnum(dpop()); cell value = dpop(); - userenv[e] = value; + special_objects[e] = value; } void factor_vm::primitive_exit() diff --git a/vm/run.hpp b/vm/run.hpp index 9a23979066..714ac1f64a 100755 --- a/vm/run.hpp +++ b/vm/run.hpp @@ -1,39 +1,39 @@ namespace factor { -#define USER_ENV 70 +static const cell special_object_count = 70; enum special_object { - NAMESTACK_ENV, /* used by library only */ - CATCHSTACK_ENV, /* used by library only, per-callback */ + OBJ_NAMESTACK, /* used by library only */ + OBJ_CATCHSTACK, /* used by library only, per-callback */ - CURRENT_CALLBACK_ENV = 2, /* used by library only, per-callback */ - WALKER_HOOK_ENV, /* non-local exit hook, used by library only */ - CALLCC_1_ENV, /* used to pass the value in callcc1 */ + OBJ_CURRENT_CALLBACK = 2, /* used by library only, per-callback */ + OBJ_WALKER_HOOK, /* non-local exit hook, used by library only */ + OBJ_CALLCC_1, /* used to pass the value in callcc1 */ - BREAK_ENV = 5, /* quotation called by throw primitive */ - ERROR_ENV, /* a marker consed onto kernel errors */ + OBJ_BREAK = 5, /* quotation called by throw primitive */ + OBJ_ERROR, /* a marker consed onto kernel errors */ - CELL_SIZE_ENV = 7, /* sizeof(cell) */ - CPU_ENV, /* CPU architecture */ - OS_ENV, /* operating system name */ + OBJ_CELL_SIZE = 7, /* sizeof(cell) */ + OBJ_CPU, /* CPU architecture */ + OBJ_OS, /* operating system name */ - ARGS_ENV = 10, /* command line arguments */ - STDIN_ENV, /* stdin FILE* handle */ - STDOUT_ENV, /* stdout FILE* handle */ + OBJ_ARGS = 10, /* command line arguments */ + OBJ_STDIN, /* stdin FILE* handle */ + OBJ_STDOUT, /* stdout FILE* handle */ - IMAGE_ENV = 13, /* image path name */ - EXECUTABLE_ENV, /* runtime executable path name */ + OBJ_IMAGE = 13, /* image path name */ + OBJ_EXECUTABLE, /* runtime executable path name */ - EMBEDDED_ENV = 15, /* are we embedded in another app? */ - EVAL_CALLBACK_ENV, /* used when Factor is embedded in a C app */ - YIELD_CALLBACK_ENV, /* used when Factor is embedded in a C app */ - SLEEP_CALLBACK_ENV, /* used when Factor is embedded in a C app */ + OBJ_EMBEDDED = 15, /* are we embedded in another app? */ + OBJ_EVAL_CALLBACK, /* used when Factor is embedded in a C app */ + OBJ_YIELD_CALLBACK, /* used when Factor is embedded in a C app */ + OBJ_SLEEP_CALLBACK, /* used when Factor is embedded in a C app */ - COCOA_EXCEPTION_ENV = 19, /* Cocoa exception handler quotation */ + OBJ_COCOA_EXCEPTION = 19, /* Cocoa exception handler quotation */ - BOOT_ENV = 20, /* boot quotation */ - GLOBAL_ENV, /* global namespace */ + OBJ_BOOT = 20, /* boot quotation */ + OBJ_GLOBAL, /* global namespace */ /* Quotation compilation in quotations.c */ JIT_PROLOG = 23, @@ -79,25 +79,25 @@ enum special_object { MEGA_LOOKUP_WORD, MEGA_MISS_WORD, - UNDEFINED_ENV = 60, /* default quotation for undefined words */ + OBJ_UNDEFINED = 60, /* default quotation for undefined words */ - STDERR_ENV = 61, /* stderr FILE* handle */ + OBJ_STDERR = 61, /* stderr FILE* handle */ - STAGE2_ENV = 62, /* have we bootstrapped? */ + OBJ_STAGE2 = 62, /* have we bootstrapped? */ - CURRENT_THREAD_ENV = 63, + OBJ_CURRENT_THREAD = 63, - THREADS_ENV = 64, - RUN_QUEUE_ENV = 65, - SLEEP_QUEUE_ENV = 66, + OBJ_THREADS = 64, + OBJ_RUN_QUEUE = 65, + OBJ_SLEEP_QUEUE = 66, }; -#define FIRST_SAVE_ENV BOOT_ENV -#define LAST_SAVE_ENV STAGE2_ENV +#define OBJ_FIRST_SAVE OBJ_BOOT +#define OBJ_LAST_SAVE OBJ_STAGE2 inline static bool save_env_p(cell i) { - return (i >= FIRST_SAVE_ENV && i <= LAST_SAVE_ENV); + return (i >= OBJ_FIRST_SAVE && i <= OBJ_LAST_SAVE); } } diff --git a/vm/vm.hpp b/vm/vm.hpp index 78efc915d7..e22e45f22e 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -18,7 +18,7 @@ struct factor_vm cell decks_offset; /* TAGGED user environment data; see getenv/setenv prims */ - cell userenv[USER_ENV]; + cell special_objects[special_object_count]; /* Data stack and retain stack sizes */ cell ds_size, rs_size; diff --git a/vm/words.cpp b/vm/words.cpp index 9d3ccff3c3..37a3821069 100644 --- a/vm/words.cpp +++ b/vm/words.cpp @@ -13,7 +13,7 @@ word *factor_vm::allot_word(cell name_, cell vocab_, cell hashcode_) new_word->hashcode = hashcode_; new_word->vocabulary = vocab.value(); new_word->name = name.value(); - new_word->def = userenv[UNDEFINED_ENV]; + new_word->def = special_objects[OBJ_UNDEFINED]; new_word->props = false_object; new_word->counter = tag_fixnum(0); new_word->pic_def = false_object; From fd1e992e7dccad1a744a2f8a347e51f353c79f71 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Sat, 24 Oct 2009 03:54:53 -0500 Subject: [PATCH 18/74] vm: factor out code that visits object slots and code heap blocks into slot_visitor and code_block_visitor --- vm/code_block_visitor.hpp | 79 ++++++++++++++ vm/code_heap.cpp | 85 +++------------ vm/code_heap.hpp | 1 - vm/collector.hpp | 213 +++++++++++++++----------------------- vm/full_collector.cpp | 100 ++++-------------- vm/full_collector.hpp | 5 - vm/master.hpp | 2 + vm/profiler.cpp | 2 +- vm/quotations.cpp | 2 +- vm/slot_visitor.hpp | 101 ++++++++++++++++++ vm/vm.hpp | 7 +- vm/words.cpp | 4 +- 12 files changed, 310 insertions(+), 291 deletions(-) create mode 100644 vm/code_block_visitor.hpp create mode 100644 vm/slot_visitor.hpp diff --git a/vm/code_block_visitor.hpp b/vm/code_block_visitor.hpp new file mode 100644 index 0000000000..12146a940e --- /dev/null +++ b/vm/code_block_visitor.hpp @@ -0,0 +1,79 @@ +namespace factor +{ + +template struct call_frame_code_block_visitor { + Visitor visitor; + + explicit call_frame_code_block_visitor(Visitor visitor_) : visitor(visitor_) {} + + void operator()(stack_frame *frame) + { + cell offset = (cell)FRAME_RETURN_ADDRESS(frame,parent) - (cell)frame->xt; + + code_block *new_block = visitor.visit_code_block(parent->frame_code(frame)); + frame->xt = new_block->xt(); + + FRAME_RETURN_ADDRESS(frame,parent) = (void *)((cell)frame->xt + offset); + } +}; + +template void factor_vm::visit_object_code_block(object *obj, Visitor visitor) +{ + switch(obj->h.hi_tag()) + { + case WORD_TYPE: + { + word *w = (word *)obj; + if(w->code) + w->code = visitor.visit_code_block(w->code); + if(w->profiling) + w->code = visitor.visit_code_block(w->profiling); + + update_word_xt(obj); + break; + } + case QUOTATION_TYPE: + { + quotation *q = (quotation *)obj; + if(q->code) + set_quot_xt(visitor.visit_code_block(q->code)); + break; + } + case CALLSTACK_TYPE: + { + callstack *stack = (callstack *)obj; + call_frame_code_block_visitor call_frame_visitor(visitor); + iterate_callstack_object(stack,call_frame_visitor); + break; + } + } +} + +template void factor_vm::visit_context_code_blocks(Visitor visitor) +{ + callstack *stack = (callstack *)obj; + call_frame_code_block_visitor call_frame_visitor(visitor); + iterate_active_frames(call_frame_visitor); +} + +template struct callback_code_block_visitor { + callback_heap *callbacks; + Visitor visitor; + + explicit callback_code_block_visitor(callback_heap *callbacks_, Visitor visitor_) : + callbacks(callbacks_), visitor(visitor_) {} + + void operator()(callback *stub) + { + stub->compiled = visitor.visit_code_block(stub->compiled); + callbacks->update(stub); + } +}; + +template void factor_vm::visit_callback_code_blocks(Visitor visitor) +{ + callback_code_block_visitor callback_visitor(callbacks,visitor); + callbacks->iterate(callback_visitor); +} + +} diff --git a/vm/code_heap.cpp b/vm/code_heap.cpp index 5ae55cb760..c837ec7615 100755 --- a/vm/code_heap.cpp +++ b/vm/code_heap.cpp @@ -188,7 +188,7 @@ void factor_vm::primitive_modify_code_heap() break; } - update_word_xt(word.value()); + update_word_xt(word.untagged()); } update_code_heap_words(); @@ -205,99 +205,42 @@ void factor_vm::primitive_code_room() dpush(tag_fixnum(max_free / 1024)); } -code_block *code_heap::forward_code_block(code_block *compiled) -{ - return (code_block *)allocator->state.forward_block(compiled); -} +struct code_block_forwarder { + mark_bits *forwarding_map; -struct callframe_forwarder { - factor_vm *parent; + explicit code_block_forwarder(mark_bits *forwarding_map_) : + forwarding_map(forwarding_map_) {} - explicit callframe_forwarder(factor_vm *parent_) : parent(parent_) {} - - void operator()(stack_frame *frame) + code_block *operator()(code_block *compiled) { - cell offset = (cell)FRAME_RETURN_ADDRESS(frame,parent) - (cell)frame->xt; - - code_block *forwarded = parent->code->forward_code_block(parent->frame_code(frame)); - frame->xt = forwarded->xt(); - - FRAME_RETURN_ADDRESS(frame,parent) = (void *)((cell)frame->xt + offset); + return (code_block *)forwarding_map->forward_block(compiled); } }; void factor_vm::forward_object_xts() { + code_block_forwarder forwarder(&code->allocator->state); + begin_scan(); cell obj; while(to_boolean(obj = next_object())) - { - switch(tagged(obj).type()) - { - case WORD_TYPE: - { - word *w = untag(obj); - - if(w->code) - w->code = code->forward_code_block(w->code); - if(w->profiling) - w->profiling = code->forward_code_block(w->profiling); - - update_word_xt(obj); - } - break; - case QUOTATION_TYPE: - { - quotation *quot = untag(obj); - - if(quot->code) - { - quot->code = code->forward_code_block(quot->code); - set_quot_xt(quot,quot->code); - } - } - break; - case CALLSTACK_TYPE: - { - callstack *stack = untag(obj); - callframe_forwarder forwarder(this); - iterate_callstack_object(stack,forwarder); - } - break; - default: - break; - } - } + visit_object_code_block(untag(obj),forwarder); end_scan(); } void factor_vm::forward_context_xts() { - callframe_forwarder forwarder(this); - iterate_active_frames(forwarder); + code_block_forwarder forwarder(&code->allocator->state); + visit_context_code_blocks(forwarder); } -struct callback_forwarder { - code_heap *code; - callback_heap *callbacks; - - callback_forwarder(code_heap *code_, callback_heap *callbacks_) : - code(code_), callbacks(callbacks_) {} - - void operator()(callback *stub) - { - stub->compiled = code->forward_code_block(stub->compiled); - callbacks->update(stub); - } -}; - void factor_vm::forward_callback_xts() { - callback_forwarder forwarder(code,callbacks); - callbacks->iterate(forwarder); + code_block_forwarder forwarder(&code->allocator->state); + visit_callback_code_blocks(forwarder); } /* Move all free space to the end of the code heap. Live blocks must be marked diff --git a/vm/code_heap.hpp b/vm/code_heap.hpp index 2d9961c03a..5548892d3f 100755 --- a/vm/code_heap.hpp +++ b/vm/code_heap.hpp @@ -26,7 +26,6 @@ struct code_heap { void set_marked_p(code_block *compiled); void clear_mark_bits(); void code_heap_free(code_block *compiled); - code_block *forward_code_block(code_block *compiled); }; } diff --git a/vm/collector.hpp b/vm/collector.hpp index a1a7dc5695..6f6a5e31cd 100644 --- a/vm/collector.hpp +++ b/vm/collector.hpp @@ -1,20 +1,14 @@ namespace factor { -template struct collector { +template struct collector_workhorse { factor_vm *parent; - data_heap *data; - code_heap *code; - gc_state *current_gc; generation_statistics *stats; TargetGeneration *target; Policy policy; - explicit collector(factor_vm *parent_, generation_statistics *stats_, TargetGeneration *target_, Policy policy_) : + explicit collector_workhorse(factor_vm *parent_, generation_statistics *stats_, TargetGeneration *target_, Policy policy_) : parent(parent_), - data(parent_->data), - code(parent_->code), - current_gc(parent_->current_gc), stats(stats_), target(target_), policy(policy_) {} @@ -32,52 +26,12 @@ template struct collector { return untagged; } - void trace_handle(cell *handle) - { - cell pointer = *handle; - - if(immediate_p(pointer)) return; - - object *untagged = untag(pointer); - if(!policy.should_copy_p(untagged)) - { - policy.visited_object(untagged); - return; - } - - object *forwarding = resolve_forwarding(untagged); - - if(forwarding == untagged) - untagged = promote_object(untagged); - else if(policy.should_copy_p(forwarding)) - untagged = promote_object(forwarding); - else - { - untagged = forwarding; - policy.visited_object(untagged); - } - - *handle = RETAG(untagged,TAG(pointer)); - } - - void trace_slots(object *ptr) - { - cell *slot = (cell *)ptr; - cell *end = (cell *)((cell)ptr + parent->binary_payload_start(ptr)); - - if(slot != end) - { - slot++; - for(; slot < end; slot++) trace_handle(slot); - } - } - object *promote_object(object *untagged) { cell size = untagged->size(); object *newpointer = target->allot(size); /* XXX not exception-safe */ - if(!newpointer) longjmp(current_gc->gc_unwind,1); + if(!newpointer) longjmp(parent->current_gc->gc_unwind,1); memcpy(newpointer,untagged,size); untagged->h.forward_to(newpointer); @@ -90,68 +44,90 @@ template struct collector { return newpointer; } - void trace_stack_elements(segment *region, cell *top) + object *visit_handle(object *obj) { - for(cell *ptr = (cell *)region->start; ptr <= top; ptr++) - trace_handle(ptr); - } - - void trace_registered_locals() - { - std::vector::const_iterator iter = parent->gc_locals.begin(); - std::vector::const_iterator end = parent->gc_locals.end(); - - for(; iter < end; iter++) - trace_handle((cell *)(*iter)); - } - - void trace_registered_bignums() - { - std::vector::const_iterator iter = parent->gc_bignums.begin(); - std::vector::const_iterator end = parent->gc_bignums.end(); - - for(; iter < end; iter++) + if(!policy.should_copy_p(obj)) { - cell *handle = (cell *)(*iter); + policy.visited_object(obj); + return obj; + } - if(*handle) - { - *handle |= BIGNUM_TYPE; - trace_handle(handle); - *handle &= ~BIGNUM_TYPE; - } + object *forwarding = resolve_forwarding(obj); + + if(forwarding == obj) + return promote_object(obj); + else if(policy.should_copy_p(forwarding)) + return promote_object(forwarding); + else + { + policy.visited_object(forwarding); + return forwarding; } } +}; + +template +inline static slot_visitor > make_collector_workhorse( + factor_vm *parent, + generation_statistics *stats, + TargetGeneration *target, + Policy policy) +{ + return slot_visitor >(parent, + collector_workhorse(parent,stats,target,policy)); +} + +template struct collector { + factor_vm *parent; + data_heap *data; + code_heap *code; + generation_statistics *stats; + TargetGeneration *target; + slot_visitor > workhorse; + + explicit collector(factor_vm *parent_, generation_statistics *stats_, TargetGeneration *target_, Policy policy_) : + parent(parent_), + data(parent_->data), + code(parent_->code), + stats(stats_), + target(target_), + workhorse(make_collector_workhorse(parent_,stats_,target_,policy_)) {} + + void trace_handle(cell *handle) + { + workhorse.visit_handle(handle); + } + + void trace_slots(object *ptr) + { + workhorse.visit_slots(ptr); + } - /* Copy roots over at the start of GC, namely various constants, stacks, - the user environment and extra roots registered by local_roots.hpp */ void trace_roots() { - trace_handle(&parent->true_object); - trace_handle(&parent->bignum_zero); - trace_handle(&parent->bignum_pos_one); - trace_handle(&parent->bignum_neg_one); - - trace_registered_locals(); - trace_registered_bignums(); - - for(cell i = 0; i < special_object_count; i++) - trace_handle(&parent->special_objects[i]); + workhorse.visit_roots(); } void trace_contexts() { - context *ctx = parent->ctx; + workhorse.visit_contexts(); + } - while(ctx) + /* Trace all literals referenced from a code block. Only for aging and nursery collections */ + void trace_literal_references(code_block *compiled) + { + workhorse.visit_literal_references(compiled); + } + + void trace_code_heap_roots(std::set *remembered_set) + { + std::set::const_iterator iter = remembered_set->begin(); + std::set::const_iterator end = remembered_set->end(); + + for(; iter != end; iter++) { - trace_stack_elements(ctx->datastack_region,(cell *)ctx->datastack); - trace_stack_elements(ctx->retainstack_region,(cell *)ctx->retainstack); - - trace_handle(&ctx->catchstack_save); - trace_handle(&ctx->current_callback_save); - - ctx = ctx->next; + trace_literal_references(*iter); + parent->gc_stats.code_blocks_scanned++; } } @@ -167,17 +143,17 @@ template struct collector { inline cell card_deck_for_address(cell a) { - return addr_to_deck(a - this->data->start); + return addr_to_deck(a - data->start); } inline cell card_start_address(cell card) { - return (card << card_bits) + this->data->start; + return (card << card_bits) + data->start; } inline cell card_end_address(cell card) { - return ((card + 1) << card_bits) + this->data->start; + return ((card + 1) << card_bits) + data->start; } void trace_partial_objects(cell start, cell end, cell card_start, cell card_end) @@ -195,7 +171,7 @@ template struct collector { if(slot_ptr != end_ptr) { for(; slot_ptr < end_ptr; slot_ptr++) - this->trace_handle(slot_ptr); + workhorse.visit_handle(slot_ptr); } } } @@ -205,10 +181,10 @@ template struct collector { { u64 start_time = current_micros(); - card_deck *decks = this->data->decks; - card_deck *cards = this->data->cards; + card_deck *decks = data->decks; + card_deck *cards = data->cards; - cell gen_start_card = addr_to_card(gen->start - this->data->start); + cell gen_start_card = addr_to_card(gen->start - data->start); cell first_deck = card_deck_for_address(gen->start); cell last_deck = card_deck_for_address(gen->end); @@ -219,7 +195,7 @@ template struct collector { { if(decks[deck_index] & mask) { - this->parent->gc_stats.decks_scanned++; + parent->gc_stats.decks_scanned++; cell first_card = first_card_in_deck(deck_index); cell last_card = last_card_in_deck(deck_index); @@ -228,17 +204,17 @@ template struct collector { { if(cards[card_index] & mask) { - this->parent->gc_stats.cards_scanned++; + parent->gc_stats.cards_scanned++; if(end < card_start_address(card_index)) { start = gen->starts.find_object_containing_card(card_index - gen_start_card); - binary_start = start + this->parent->binary_payload_start((object *)start); + binary_start = start + parent->binary_payload_start((object *)start); end = start + ((object *)start)->size(); } #ifdef FACTOR_DEBUG - assert(addr_to_card(start - this->data->start) <= card_index); + assert(addr_to_card(start - data->start) <= card_index); assert(start < card_end_address(card_index)); #endif @@ -253,7 +229,7 @@ scan_next_object: { start = gen->next_object_after(start); if(start) { - binary_start = start + this->parent->binary_payload_start((object *)start); + binary_start = start + parent->binary_payload_start((object *)start); end = start + ((object *)start)->size(); goto scan_next_object; } @@ -270,24 +246,7 @@ scan_next_object: { } } -end: this->parent->gc_stats.card_scan_time += (current_micros() - start_time); - } - - /* Trace all literals referenced from a code block. Only for aging and nursery collections */ - void trace_literal_references(code_block *compiled) - { - this->trace_handle(&compiled->owner); - this->trace_handle(&compiled->literals); - this->trace_handle(&compiled->relocation); - this->parent->gc_stats.code_blocks_scanned++; - } - - void trace_code_heap_roots(std::set *remembered_set) - { - std::set::const_iterator iter = remembered_set->begin(); - std::set::const_iterator end = remembered_set->end(); - - for(; iter != end; iter++) trace_literal_references(*iter); +end: parent->gc_stats.card_scan_time += (current_micros() - start_time); } }; diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index 817908ece5..fbffe5b33e 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -10,101 +10,36 @@ full_collector::full_collector(factor_vm *parent_) : parent_->data->tenured, full_policy(parent_)) {} -struct stack_frame_marker { - factor_vm *parent; +struct code_block_marker { + code_heap *code; full_collector *collector; - explicit stack_frame_marker(full_collector *collector_) : - parent(collector_->parent), collector(collector_) {} + explicit code_block_marker(code_heap *code_, full_collector *collector_) : + code(code_), collector(collector_) {} - void operator()(stack_frame *frame) + code_block *operator()(code_block *compiled) { - collector->mark_code_block(parent->frame_code(frame)); + if(!code->marked_p(compiled)) + { + code->set_marked_p(compiled); + collector->trace_literal_references(compiled); + } + + return compiled; } }; -/* Mark code blocks executing in currently active stack frames. */ -void full_collector::mark_active_blocks() -{ - stack_frame_marker marker(this); - parent->iterate_active_frames(marker); -} - -void full_collector::mark_object_code_block(object *obj) -{ - switch(obj->h.hi_tag()) - { - case WORD_TYPE: - { - word *w = (word *)obj; - if(w->code) - mark_code_block(w->code); - if(w->profiling) - mark_code_block(w->profiling); - break; - } - case QUOTATION_TYPE: - { - quotation *q = (quotation *)obj; - if(q->code) - mark_code_block(q->code); - break; - } - case CALLSTACK_TYPE: - { - callstack *stack = (callstack *)obj; - stack_frame_marker marker(this); - parent->iterate_callstack_object(stack,marker); - break; - } - } -} - -struct callback_tracer { - full_collector *collector; - - callback_tracer(full_collector *collector_) : collector(collector_) {} - - void operator()(callback *stub) - { - collector->mark_code_block(stub->compiled); - } -}; - -void full_collector::trace_callbacks() -{ - callback_tracer tracer(this); - parent->callbacks->iterate(tracer); -} - -/* Trace all literals referenced from a code block. Only for aging and nursery collections */ -void full_collector::trace_literal_references(code_block *compiled) -{ - this->trace_handle(&compiled->owner); - this->trace_handle(&compiled->literals); - this->trace_handle(&compiled->relocation); -} - -/* Mark all literals referenced from a word XT. Only for tenured -collections */ -void full_collector::mark_code_block(code_block *compiled) -{ - if(!this->code->marked_p(compiled)) - { - this->code->set_marked_p(compiled); - trace_literal_references(compiled); - } -} - void full_collector::mark_reachable_objects() { + code_block_marker marker(code,this); std::vector *mark_stack = &this->target->mark_stack; + while(!mark_stack->empty()) { object *obj = mark_stack->back(); mark_stack->pop_back(); this->trace_slots(obj); - this->mark_object_code_block(obj); + parent->visit_object_code_block(obj,marker); } } @@ -131,8 +66,9 @@ void factor_vm::collect_full_impl(bool trace_contexts_p) if(trace_contexts_p) { collector.trace_contexts(); - collector.mark_active_blocks(); - collector.trace_callbacks(); + code_block_marker marker(code,&collector); + visit_context_code_blocks(marker); + visit_callback_code_blocks(marker); } collector.mark_reachable_objects(); diff --git a/vm/full_collector.hpp b/vm/full_collector.hpp index 9aef352b4b..f613558997 100644 --- a/vm/full_collector.hpp +++ b/vm/full_collector.hpp @@ -28,11 +28,6 @@ struct full_collector : collector { bool trace_contexts_p; full_collector(factor_vm *parent_); - void mark_active_blocks(); - void mark_object_code_block(object *object); - void trace_callbacks(); - void trace_literal_references(code_block *compiled); - void mark_code_block(code_block *compiled); void mark_reachable_objects(); }; diff --git a/vm/master.hpp b/vm/master.hpp index 9168cecce4..dc01429c30 100755 --- a/vm/master.hpp +++ b/vm/master.hpp @@ -72,11 +72,13 @@ namespace factor #include "vm.hpp" #include "tagged.hpp" #include "local_roots.hpp" +#include "slot_visitor.hpp" #include "collector.hpp" #include "copying_collector.hpp" #include "nursery_collector.hpp" #include "aging_collector.hpp" #include "to_tenured_collector.hpp" +#include "code_block_visitor.hpp" #include "full_collector.hpp" #include "callstack.hpp" #include "generic_arrays.hpp" diff --git a/vm/profiler.cpp b/vm/profiler.cpp index 5113b55cf7..2f5fc6fcf4 100755 --- a/vm/profiler.cpp +++ b/vm/profiler.cpp @@ -40,7 +40,7 @@ void factor_vm::set_profiling(bool profiling) tagged word(array_nth(words.untagged(),i)); if(profiling) word->counter = tag_fixnum(0); - update_word_xt(word.value()); + update_word_xt(word.untagged()); } update_code_heap_words(); diff --git a/vm/quotations.cpp b/vm/quotations.cpp index c65c0fe909..17b7c4328b 100755 --- a/vm/quotations.cpp +++ b/vm/quotations.cpp @@ -338,7 +338,7 @@ void factor_vm::compile_all_words() if(!word->code || !word->code->optimized_p()) jit_compile_word(word.value(),word->def,false); - update_word_xt(word.value()); + update_word_xt(word.untagged()); } diff --git a/vm/slot_visitor.hpp b/vm/slot_visitor.hpp new file mode 100644 index 0000000000..a62fdd87ec --- /dev/null +++ b/vm/slot_visitor.hpp @@ -0,0 +1,101 @@ +namespace factor +{ + +template struct slot_visitor { + factor_vm *parent; + Visitor visitor; + + slot_visitor(factor_vm *parent_, Visitor visitor_) : + parent(parent_), visitor(visitor_) {} + + void visit_handle(cell *handle) + { + cell pointer = *handle; + + if(immediate_p(pointer)) return; + + object *untagged = untag(pointer); + untagged = visitor.visit_handle(untagged); + *handle = RETAG(untagged,TAG(pointer)); + } + + void visit_slots(object *ptr) + { + cell *slot = (cell *)ptr; + cell *end = (cell *)((cell)ptr + parent->binary_payload_start(ptr)); + + if(slot != end) + { + slot++; + for(; slot < end; slot++) visit_handle(slot); + } + } + + void visit_stack_elements(segment *region, cell *top) + { + for(cell *ptr = (cell *)region->start; ptr <= top; ptr++) + visit_handle(ptr); + } + + void visit_registered_locals() + { + std::vector::const_iterator iter = parent->gc_locals.begin(); + std::vector::const_iterator end = parent->gc_locals.end(); + + for(; iter < end; iter++) + visit_handle((cell *)(*iter)); + } + + void visit_registered_bignums() + { + std::vector::const_iterator iter = parent->gc_bignums.begin(); + std::vector::const_iterator end = parent->gc_bignums.end(); + + for(; iter < end; iter++) + { + cell *handle = (cell *)(*iter); + + if(*handle) + *handle = (cell)visitor.visit_handle(*(object **)handle); + } + } + + void visit_roots() + { + visit_handle(&parent->true_object); + visit_handle(&parent->bignum_zero); + visit_handle(&parent->bignum_pos_one); + visit_handle(&parent->bignum_neg_one); + + visit_registered_locals(); + visit_registered_bignums(); + + for(cell i = 0; i < special_object_count; i++) + visit_handle(&parent->special_objects[i]); + } + + void visit_contexts() + { + context *ctx = parent->ctx; + + while(ctx) + { + visit_stack_elements(ctx->datastack_region,(cell *)ctx->datastack); + visit_stack_elements(ctx->retainstack_region,(cell *)ctx->retainstack); + + visit_handle(&ctx->catchstack_save); + visit_handle(&ctx->current_callback_save); + + ctx = ctx->next; + } + } + + void visit_literal_references(code_block *compiled) + { + visit_handle(&compiled->owner); + visit_handle(&compiled->literals); + visit_handle(&compiled->relocation); + } +}; + +} diff --git a/vm/vm.hpp b/vm/vm.hpp index e22e45f22e..674f934bba 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -353,7 +353,7 @@ struct factor_vm word *allot_word(cell name_, cell vocab_, cell hashcode_); void primitive_word(); void primitive_word_xt(); - void update_word_xt(cell w_); + void update_word_xt(word *w_); void primitive_optimized_p(); void primitive_wrapper(); @@ -485,6 +485,11 @@ struct factor_vm code_block *allot_code_block(cell size, code_block_type type); code_block *add_code_block(code_block_type type, cell code_, cell labels_, cell owner_, cell relocation_, cell literals_); + //code_block_visitor + template void visit_object_code_block(object *obj, Visitor visitor); + template void visit_context_code_blocks(Visitor visitor); + template void visit_callback_code_blocks(Visitor visitor); + //code heap inline void check_code_pointer(cell ptr) { diff --git a/vm/words.cpp b/vm/words.cpp index 37a3821069..4248c14b7d 100644 --- a/vm/words.cpp +++ b/vm/words.cpp @@ -23,7 +23,7 @@ word *factor_vm::allot_word(cell name_, cell vocab_, cell hashcode_) new_word->code = NULL; jit_compile_word(new_word.value(),new_word->def,true); - update_word_xt(new_word.value()); + update_word_xt(new_word.untagged()); if(profiling_p) relocate_code_block(new_word->profiling); @@ -59,7 +59,7 @@ void factor_vm::primitive_word_xt() } /* Allocates memory */ -void factor_vm::update_word_xt(cell w_) +void factor_vm::update_word_xt(word *w_) { gc_root w(w_,this); From 29a27cfde4dc00161f4c8a02a67b07f7bf35f6ca Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Sat, 24 Oct 2009 04:18:33 -0500 Subject: [PATCH 19/74] vm: data heap compaction work in progress --- Makefile | 1 + vm/code_block_visitor.hpp | 23 +++++----- vm/code_heap.cpp | 61 ------------------------- vm/collector.hpp | 2 +- vm/compaction.cpp | 96 +++++++++++++++++++++++++++++++++++++++ vm/compaction.hpp | 4 ++ vm/full_collector.cpp | 12 ++--- vm/gc.cpp | 6 +-- vm/master.hpp | 1 + vm/slot_visitor.hpp | 4 +- vm/vm.hpp | 11 ++--- 11 files changed, 130 insertions(+), 91 deletions(-) create mode 100644 vm/compaction.cpp create mode 100644 vm/compaction.hpp diff --git a/Makefile b/Makefile index 78f59a38bb..030a278543 100755 --- a/Makefile +++ b/Makefile @@ -41,6 +41,7 @@ DLL_OBJS = $(PLAF_DLL_OBJS) \ vm/callstack.o \ vm/code_block.o \ vm/code_heap.o \ + vm/compaction.o \ vm/contexts.o \ vm/data_heap.o \ vm/debug.o \ diff --git a/vm/code_block_visitor.hpp b/vm/code_block_visitor.hpp index 12146a940e..9683c39db5 100644 --- a/vm/code_block_visitor.hpp +++ b/vm/code_block_visitor.hpp @@ -2,15 +2,17 @@ namespace factor { template struct call_frame_code_block_visitor { + factor_vm *parent; Visitor visitor; - explicit call_frame_code_block_visitor(Visitor visitor_) : visitor(visitor_) {} + explicit call_frame_code_block_visitor(factor_vm *parent_, Visitor visitor_) : + parent(parent_), visitor(visitor_) {} void operator()(stack_frame *frame) { cell offset = (cell)FRAME_RETURN_ADDRESS(frame,parent) - (cell)frame->xt; - code_block *new_block = visitor.visit_code_block(parent->frame_code(frame)); + code_block *new_block = visitor(parent->frame_code(frame)); frame->xt = new_block->xt(); FRAME_RETURN_ADDRESS(frame,parent) = (void *)((cell)frame->xt + offset); @@ -25,24 +27,24 @@ template void factor_vm::visit_object_code_block(object *obj, { word *w = (word *)obj; if(w->code) - w->code = visitor.visit_code_block(w->code); + w->code = visitor(w->code); if(w->profiling) - w->code = visitor.visit_code_block(w->profiling); + w->code = visitor(w->profiling); - update_word_xt(obj); + update_word_xt(w); break; } case QUOTATION_TYPE: { quotation *q = (quotation *)obj; if(q->code) - set_quot_xt(visitor.visit_code_block(q->code)); + set_quot_xt(q,visitor(q->code)); break; } case CALLSTACK_TYPE: { callstack *stack = (callstack *)obj; - call_frame_code_block_visitor call_frame_visitor(visitor); + call_frame_code_block_visitor call_frame_visitor(this,visitor); iterate_callstack_object(stack,call_frame_visitor); break; } @@ -51,8 +53,7 @@ template void factor_vm::visit_object_code_block(object *obj, template void factor_vm::visit_context_code_blocks(Visitor visitor) { - callstack *stack = (callstack *)obj; - call_frame_code_block_visitor call_frame_visitor(visitor); + call_frame_code_block_visitor call_frame_visitor(this,visitor); iterate_active_frames(call_frame_visitor); } @@ -65,14 +66,14 @@ template struct callback_code_block_visitor { void operator()(callback *stub) { - stub->compiled = visitor.visit_code_block(stub->compiled); + stub->compiled = visitor(stub->compiled); callbacks->update(stub); } }; template void factor_vm::visit_callback_code_blocks(Visitor visitor) { - callback_code_block_visitor callback_visitor(callbacks,visitor); + callback_code_block_visitor callback_visitor(callbacks,visitor); callbacks->iterate(callback_visitor); } diff --git a/vm/code_heap.cpp b/vm/code_heap.cpp index c837ec7615..ef257bb935 100755 --- a/vm/code_heap.cpp +++ b/vm/code_heap.cpp @@ -205,67 +205,6 @@ void factor_vm::primitive_code_room() dpush(tag_fixnum(max_free / 1024)); } -struct code_block_forwarder { - mark_bits *forwarding_map; - - explicit code_block_forwarder(mark_bits *forwarding_map_) : - forwarding_map(forwarding_map_) {} - - code_block *operator()(code_block *compiled) - { - return (code_block *)forwarding_map->forward_block(compiled); - } -}; - -void factor_vm::forward_object_xts() -{ - code_block_forwarder forwarder(&code->allocator->state); - - begin_scan(); - - cell obj; - - while(to_boolean(obj = next_object())) - visit_object_code_block(untag(obj),forwarder); - - end_scan(); -} - -void factor_vm::forward_context_xts() -{ - code_block_forwarder forwarder(&code->allocator->state); - visit_context_code_blocks(forwarder); -} - -void factor_vm::forward_callback_xts() -{ - code_block_forwarder forwarder(&code->allocator->state); - visit_callback_code_blocks(forwarder); -} - -/* Move all free space to the end of the code heap. Live blocks must be marked -on entry to this function. XTs in code blocks must be updated after this -function returns. */ -void factor_vm::compact_code_heap(bool trace_contexts_p) -{ - /* Figure out where blocks are going to go */ - code->allocator->state.compute_forwarding(); - - /* Update references to the code heap from the data heap */ - forward_object_xts(); - if(trace_contexts_p) - { - forward_context_xts(); - forward_callback_xts(); - } - - /* Move code blocks and update references amongst them (this requires - that the data heap is up to date since relocation looks up object XTs) */ - code_heap_relocator relocator(this); - code_heap_iterator iter(relocator); - code->allocator->compact(iter); -} - struct stack_trace_stripper { explicit stack_trace_stripper() {} diff --git a/vm/collector.hpp b/vm/collector.hpp index 6f6a5e31cd..86ae963774 100644 --- a/vm/collector.hpp +++ b/vm/collector.hpp @@ -44,7 +44,7 @@ template struct collector_workhorse return newpointer; } - object *visit_handle(object *obj) + object *visit_object(object *obj) { if(!policy.should_copy_p(obj)) { diff --git a/vm/compaction.cpp b/vm/compaction.cpp new file mode 100644 index 0000000000..b0ced6fcb0 --- /dev/null +++ b/vm/compaction.cpp @@ -0,0 +1,96 @@ +#include "master.hpp" + +namespace factor { + +struct object_slot_forwarder { + mark_bits *forwarding_map; + + explicit object_slot_forwarder(mark_bits *forwarding_map_) : + forwarding_map(forwarding_map_) {} + + object *visit_object(object *obj) + { + return forwarding_map->forward_block(obj); + } +}; + +struct code_block_forwarder { + mark_bits *forwarding_map; + + explicit code_block_forwarder(mark_bits *forwarding_map_) : + forwarding_map(forwarding_map_) {} + + code_block *operator()(code_block *compiled) + { + return (code_block *)forwarding_map->forward_block(compiled); + } +}; + +struct object_compaction_updater { + factor_vm *parent; + slot_visitor slot_forwarder; + code_block_forwarder code_forwarder; + + explicit object_compaction_updater(factor_vm *parent_, + slot_visitor slot_forwader_, + code_block_forwarder code_forwarder_) : + parent(parent_), + slot_forwarder(slot_forwader_), + code_forwarder(code_forwarder_) {} + + void operator()(object *obj, cell size) + { + slot_forwarder.visit_slots(obj); + parent->visit_object_code_block(obj,code_forwarder); + } +}; + +struct code_block_compaction_updater { + factor_vm *parent; + slot_visitor slot_forwarder; + + explicit code_block_compaction_updater(factor_vm *parent_, slot_visitor slot_forwader_) : + parent(parent_), slot_forwarder(slot_forwader_) {} + + void operator()(code_block *compiled, cell size) + { + slot_forwarder.visit_literal_references(compiled); + parent->relocate_code_block(compiled); + } +}; + +void factor_vm::compact_full_impl(bool trace_contexts_p) +{ + tenured_space *tenured = data->tenured; + mark_bits *data_forwarding_map = &tenured->state; + mark_bits *code_forwarding_map = &code->allocator->state; + + /* Figure out where blocks are going to go */ + data_forwarding_map->compute_forwarding(); + code_forwarding_map->compute_forwarding(); + + /* Update root pointers */ + slot_visitor slot_forwarder(this,object_slot_forwarder(data_forwarding_map)); + code_block_forwarder code_forwarder(code_forwarding_map); + + slot_forwarder.visit_roots(); + if(trace_contexts_p) + { + slot_forwarder.visit_contexts(); + visit_context_code_blocks(code_forwarder); + visit_callback_code_blocks(code_forwarder); + } + + /* Slide everything in tenured space up, and update data and code heap + pointers inside objects. */ + object_compaction_updater object_updater(this,slot_forwarder,code_forwarder); + tenured->compact(object_updater); + + /* Slide everything in the code heap up, and update data and code heap + pointers inside code blocks. */ + code_block_compaction_updater code_block_updater(this,slot_forwarder); + code_heap_iterator iter(code_block_updater); + code->allocator->compact(iter); +} + +} diff --git a/vm/compaction.hpp b/vm/compaction.hpp new file mode 100644 index 0000000000..412ef35bb4 --- /dev/null +++ b/vm/compaction.hpp @@ -0,0 +1,4 @@ +namespace factor +{ + +} diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index fbffe5b33e..531b76bf31 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -85,7 +85,7 @@ void factor_vm::collect_full_impl(bool trace_contexts_p) void factor_vm::collect_growing_heap(cell requested_bytes, bool trace_contexts_p, - bool compact_code_heap_p) + bool compact_p) { /* Grow the data heap and copy all live objects to the new heap. */ data_heap *old = data; @@ -93,18 +93,18 @@ void factor_vm::collect_growing_heap(cell requested_bytes, collect_full_impl(trace_contexts_p); delete old; - if(compact_code_heap_p) - compact_code_heap(trace_contexts_p); + if(compact_p) + compact_full_impl(trace_contexts_p); else relocate_code_heap(); } -void factor_vm::collect_full(bool trace_contexts_p, bool compact_code_heap_p) +void factor_vm::collect_full(bool trace_contexts_p, bool compact_p) { collect_full_impl(trace_contexts_p); - if(compact_code_heap_p) - compact_code_heap(trace_contexts_p); + if(compact_p) + compact_full_impl(trace_contexts_p); else update_code_heap_words_and_literals(); } diff --git a/vm/gc.cpp b/vm/gc.cpp index 6b3ec80481..1851924cd5 100755 --- a/vm/gc.cpp +++ b/vm/gc.cpp @@ -28,7 +28,7 @@ void factor_vm::record_gc_stats(generation_statistics *stats) void factor_vm::gc(gc_op op, cell requested_bytes, bool trace_contexts_p, - bool compact_code_heap_p) + bool compact_p) { assert(!gc_off); assert(!current_gc); @@ -83,11 +83,11 @@ void factor_vm::gc(gc_op op, record_gc_stats(&gc_stats.aging_stats); break; case collect_full_op: - collect_full(trace_contexts_p,compact_code_heap_p); + collect_full(trace_contexts_p,compact_p); record_gc_stats(&gc_stats.full_stats); break; case collect_growing_heap_op: - collect_growing_heap(requested_bytes,trace_contexts_p,compact_code_heap_p); + collect_growing_heap(requested_bytes,trace_contexts_p,compact_p); record_gc_stats(&gc_stats.full_stats); break; default: diff --git a/vm/master.hpp b/vm/master.hpp index dc01429c30..1947c0ad50 100755 --- a/vm/master.hpp +++ b/vm/master.hpp @@ -79,6 +79,7 @@ namespace factor #include "aging_collector.hpp" #include "to_tenured_collector.hpp" #include "code_block_visitor.hpp" +#include "compaction.hpp" #include "full_collector.hpp" #include "callstack.hpp" #include "generic_arrays.hpp" diff --git a/vm/slot_visitor.hpp b/vm/slot_visitor.hpp index a62fdd87ec..6e0f6839e2 100644 --- a/vm/slot_visitor.hpp +++ b/vm/slot_visitor.hpp @@ -15,7 +15,7 @@ template struct slot_visitor { if(immediate_p(pointer)) return; object *untagged = untag(pointer); - untagged = visitor.visit_handle(untagged); + untagged = visitor.visit_object(untagged); *handle = RETAG(untagged,TAG(pointer)); } @@ -56,7 +56,7 @@ template struct slot_visitor { cell *handle = (cell *)(*iter); if(*handle) - *handle = (cell)visitor.visit_handle(*(object **)handle); + *handle = (cell)visitor.visit_object(*(object **)handle); } } diff --git a/vm/vm.hpp b/vm/vm.hpp index 674f934bba..b596b7a50c 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -249,10 +249,11 @@ struct factor_vm void collect_aging(); void collect_to_tenured(); void collect_full_impl(bool trace_contexts_p); - void collect_growing_heap(cell requested_bytes, bool trace_contexts_p, bool compact_code_heap_p); - void collect_full(bool trace_contexts_p, bool compact_code_heap_p); + void compact_full_impl(bool trace_contexts_p); + void collect_growing_heap(cell requested_bytes, bool trace_contexts_p, bool compact_p); + void collect_full(bool trace_contexts_p, bool compact_p); void record_gc_stats(generation_statistics *stats); - void gc(gc_op op, cell requested_bytes, bool trace_contexts_p, bool compact_code_heap_p); + void gc(gc_op op, cell requested_bytes, bool trace_contexts_p, bool compact_p); void primitive_minor_gc(); void primitive_full_gc(); void primitive_compact_gc(); @@ -506,10 +507,6 @@ struct factor_vm void relocate_code_heap(); void primitive_modify_code_heap(); void primitive_code_room(); - void forward_object_xts(); - void forward_context_xts(); - void forward_callback_xts(); - void compact_code_heap(bool trace_contexts_p); void primitive_strip_stack_traces(); /* Apply a function to every code block */ From 03f4b4cdd683a259fb2fdb4cbc625602e8b1a6af Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Sat, 24 Oct 2009 04:27:45 -0500 Subject: [PATCH 20/74] vm: move binary_payload_start() method from factor_vm to object class --- vm/collector.hpp | 4 ++-- vm/data_heap.cpp | 20 ++++++++++---------- vm/generic_arrays.hpp | 2 +- vm/layouts.hpp | 1 + vm/slot_visitor.hpp | 2 +- vm/tuples.hpp | 2 +- vm/vm.hpp | 3 +-- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/vm/collector.hpp b/vm/collector.hpp index 86ae963774..e0ff53df9b 100644 --- a/vm/collector.hpp +++ b/vm/collector.hpp @@ -209,7 +209,7 @@ template struct collector { if(end < card_start_address(card_index)) { start = gen->starts.find_object_containing_card(card_index - gen_start_card); - binary_start = start + parent->binary_payload_start((object *)start); + binary_start = start + ((object *)start)->binary_payload_start(); end = start + ((object *)start)->size(); } @@ -229,7 +229,7 @@ scan_next_object: { start = gen->next_object_after(start); if(start) { - binary_start = start + parent->binary_payload_start((object *)start); + binary_start = ((object *)start)->binary_payload_start(); end = start + ((object *)start)->size(); goto scan_next_object; } diff --git a/vm/data_heap.cpp b/vm/data_heap.cpp index 6178dc8861..b210adb8e1 100755 --- a/vm/data_heap.cpp +++ b/vm/data_heap.cpp @@ -159,17 +159,12 @@ cell object::size() const } } -void factor_vm::primitive_size() -{ - box_unsigned_cell(object_size(dpop())); -} - /* The number of cells from the start of the object which should be scanned by the GC. Some types have a binary payload at the end (string, word, DLL) which we ignore. */ -cell factor_vm::binary_payload_start(object *pointer) +cell object::binary_payload_start() const { - switch(pointer->h.hi_tag()) + switch(h.hi_tag()) { /* these objects do not refer to other objects at all */ case FLOAT_TYPE: @@ -190,17 +185,22 @@ cell factor_vm::binary_payload_start(object *pointer) return sizeof(string); /* everything else consists entirely of pointers */ case ARRAY_TYPE: - return array_size(array_capacity((array*)pointer)); + return array_size(array_capacity((array*)this)); case TUPLE_TYPE: - return tuple_size(untag(((tuple *)pointer)->layout)); + return tuple_size(untag(((tuple *)this)->layout)); case WRAPPER_TYPE: return sizeof(wrapper); default: - critical_error("Invalid header",(cell)pointer); + critical_error("Invalid header",(cell)this); return 0; /* can't happen */ } } +void factor_vm::primitive_size() +{ + box_unsigned_cell(object_size(dpop())); +} + /* Push memory usage statistics in data heap */ void factor_vm::primitive_data_room() { diff --git a/vm/generic_arrays.hpp b/vm/generic_arrays.hpp index e1d2c4dc0b..89eb56a70d 100755 --- a/vm/generic_arrays.hpp +++ b/vm/generic_arrays.hpp @@ -1,7 +1,7 @@ namespace factor { -template cell array_capacity(Array *array) +template cell array_capacity(const Array *array) { #ifdef FACTOR_DEBUG assert(array->h.hi_tag() == Array::type_number); diff --git a/vm/layouts.hpp b/vm/layouts.hpp index c90be1b2dd..b3cba58495 100644 --- a/vm/layouts.hpp +++ b/vm/layouts.hpp @@ -148,6 +148,7 @@ struct object { header h; cell size() const; + cell binary_payload_start() const; cell *slots() const { return (cell *)this; } diff --git a/vm/slot_visitor.hpp b/vm/slot_visitor.hpp index 6e0f6839e2..67a51549b1 100644 --- a/vm/slot_visitor.hpp +++ b/vm/slot_visitor.hpp @@ -22,7 +22,7 @@ template struct slot_visitor { void visit_slots(object *ptr) { cell *slot = (cell *)ptr; - cell *end = (cell *)((cell)ptr + parent->binary_payload_start(ptr)); + cell *end = (cell *)((cell)ptr + ptr->binary_payload_start()); if(slot != end) { diff --git a/vm/tuples.hpp b/vm/tuples.hpp index 04b23b5857..bcd041fc65 100644 --- a/vm/tuples.hpp +++ b/vm/tuples.hpp @@ -1,7 +1,7 @@ namespace factor { -inline static cell tuple_size(tuple_layout *layout) +inline static cell tuple_size(const tuple_layout *layout) { cell size = untag_fixnum(layout->size); return sizeof(tuple) + size * sizeof(cell); diff --git a/vm/vm.hpp b/vm/vm.hpp index b596b7a50c..0b66025374 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -223,7 +223,6 @@ struct factor_vm void set_data_heap(data_heap *data_); void init_data_heap(cell young_size, cell aging_size, cell tenured_size); void primitive_size(); - cell binary_payload_start(object *pointer); void primitive_data_room(); void begin_scan(); void end_scan(); @@ -576,7 +575,7 @@ struct factor_vm template void do_slots(cell obj, Iterator &iter) { cell scan = obj; - cell payload_start = binary_payload_start((object *)obj); + cell payload_start = ((object *)obj)->binary_payload_start(); cell end = obj + payload_start; scan += sizeof(cell); From d855593f1ff4c9c4a4b7e8841be5405cb9bb2b23 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Sat, 24 Oct 2009 04:36:29 -0500 Subject: [PATCH 21/74] vm: clean up code heap visitor --- vm/code_block_visitor.hpp | 94 +++++++++++++++++++++------------------ vm/compaction.cpp | 12 ++--- vm/full_collector.cpp | 31 ++++++------- vm/full_collector.hpp | 1 - vm/vm.hpp | 5 --- 5 files changed, 70 insertions(+), 73 deletions(-) diff --git a/vm/code_block_visitor.hpp b/vm/code_block_visitor.hpp index 9683c39db5..09bbecc757 100644 --- a/vm/code_block_visitor.hpp +++ b/vm/code_block_visitor.hpp @@ -19,44 +19,6 @@ template struct call_frame_code_block_visitor { } }; -template void factor_vm::visit_object_code_block(object *obj, Visitor visitor) -{ - switch(obj->h.hi_tag()) - { - case WORD_TYPE: - { - word *w = (word *)obj; - if(w->code) - w->code = visitor(w->code); - if(w->profiling) - w->code = visitor(w->profiling); - - update_word_xt(w); - break; - } - case QUOTATION_TYPE: - { - quotation *q = (quotation *)obj; - if(q->code) - set_quot_xt(q,visitor(q->code)); - break; - } - case CALLSTACK_TYPE: - { - callstack *stack = (callstack *)obj; - call_frame_code_block_visitor call_frame_visitor(this,visitor); - iterate_callstack_object(stack,call_frame_visitor); - break; - } - } -} - -template void factor_vm::visit_context_code_blocks(Visitor visitor) -{ - call_frame_code_block_visitor call_frame_visitor(this,visitor); - iterate_active_frames(call_frame_visitor); -} - template struct callback_code_block_visitor { callback_heap *callbacks; Visitor visitor; @@ -71,10 +33,56 @@ template struct callback_code_block_visitor { } }; -template void factor_vm::visit_callback_code_blocks(Visitor visitor) -{ - callback_code_block_visitor callback_visitor(callbacks,visitor); - callbacks->iterate(callback_visitor); -} +template struct code_block_visitor { + factor_vm *parent; + Visitor visitor; + + explicit code_block_visitor(factor_vm *parent_, Visitor visitor_) : + parent(parent_), visitor(visitor_) {} + void visit_object_code_block(object *obj) + { + switch(obj->h.hi_tag()) + { + case WORD_TYPE: + { + word *w = (word *)obj; + if(w->code) + w->code = visitor(w->code); + if(w->profiling) + w->code = visitor(w->profiling); + + parent->update_word_xt(w); + break; + } + case QUOTATION_TYPE: + { + quotation *q = (quotation *)obj; + if(q->code) + parent->set_quot_xt(q,visitor(q->code)); + break; + } + case CALLSTACK_TYPE: + { + callstack *stack = (callstack *)obj; + call_frame_code_block_visitor call_frame_visitor(parent,visitor); + parent->iterate_callstack_object(stack,call_frame_visitor); + break; + } + } + } + + void visit_context_code_blocks() + { + call_frame_code_block_visitor call_frame_visitor(parent,visitor); + parent->iterate_active_frames(call_frame_visitor); + } + + void visit_callback_code_blocks() + { + callback_code_block_visitor callback_visitor(parent->callbacks,visitor); + parent->callbacks->iterate(callback_visitor); + } + +}; } diff --git a/vm/compaction.cpp b/vm/compaction.cpp index b0ced6fcb0..3cf5e5b46c 100644 --- a/vm/compaction.cpp +++ b/vm/compaction.cpp @@ -29,11 +29,11 @@ struct code_block_forwarder { struct object_compaction_updater { factor_vm *parent; slot_visitor slot_forwarder; - code_block_forwarder code_forwarder; + code_block_visitor code_forwarder; explicit object_compaction_updater(factor_vm *parent_, slot_visitor slot_forwader_, - code_block_forwarder code_forwarder_) : + code_block_visitor code_forwarder_) : parent(parent_), slot_forwarder(slot_forwader_), code_forwarder(code_forwarder_) {} @@ -41,7 +41,7 @@ struct object_compaction_updater { void operator()(object *obj, cell size) { slot_forwarder.visit_slots(obj); - parent->visit_object_code_block(obj,code_forwarder); + code_forwarder.visit_object_code_block(obj); } }; @@ -71,14 +71,14 @@ void factor_vm::compact_full_impl(bool trace_contexts_p) /* Update root pointers */ slot_visitor slot_forwarder(this,object_slot_forwarder(data_forwarding_map)); - code_block_forwarder code_forwarder(code_forwarding_map); + code_block_visitor code_forwarder(this,code_block_forwarder(code_forwarding_map)); slot_forwarder.visit_roots(); if(trace_contexts_p) { slot_forwarder.visit_contexts(); - visit_context_code_blocks(code_forwarder); - visit_callback_code_blocks(code_forwarder); + code_forwarder.visit_context_code_blocks(); + code_forwarder.visit_callback_code_blocks(); } /* Slide everything in tenured space up, and update data and code heap diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index 531b76bf31..2afb4181f7 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -29,20 +29,6 @@ struct code_block_marker { } }; -void full_collector::mark_reachable_objects() -{ - code_block_marker marker(code,this); - std::vector *mark_stack = &this->target->mark_stack; - - while(!mark_stack->empty()) - { - object *obj = mark_stack->back(); - mark_stack->pop_back(); - this->trace_slots(obj); - parent->visit_object_code_block(obj,marker); - } -} - struct object_start_map_updater { object_start_map *starts; @@ -62,16 +48,25 @@ void factor_vm::collect_full_impl(bool trace_contexts_p) data->tenured->clear_mark_bits(); data->tenured->clear_mark_stack(); + code_block_visitor code_marker(this,code_block_marker(code,&collector)); + collector.trace_roots(); if(trace_contexts_p) { collector.trace_contexts(); - code_block_marker marker(code,&collector); - visit_context_code_blocks(marker); - visit_callback_code_blocks(marker); + code_marker.visit_context_code_blocks(); + code_marker.visit_callback_code_blocks(); } - collector.mark_reachable_objects(); + std::vector *mark_stack = &data->tenured->mark_stack; + + while(!mark_stack->empty()) + { + object *obj = mark_stack->back(); + mark_stack->pop_back(); + collector.trace_slots(obj); + code_marker.visit_object_code_block(obj); + } data->tenured->starts.clear_object_start_offsets(); object_start_map_updater updater(&data->tenured->starts); diff --git a/vm/full_collector.hpp b/vm/full_collector.hpp index f613558997..c9750302d1 100644 --- a/vm/full_collector.hpp +++ b/vm/full_collector.hpp @@ -28,7 +28,6 @@ struct full_collector : collector { bool trace_contexts_p; full_collector(factor_vm *parent_); - void mark_reachable_objects(); }; } diff --git a/vm/vm.hpp b/vm/vm.hpp index 0b66025374..762a34d225 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -485,11 +485,6 @@ struct factor_vm code_block *allot_code_block(cell size, code_block_type type); code_block *add_code_block(code_block_type type, cell code_, cell labels_, cell owner_, cell relocation_, cell literals_); - //code_block_visitor - template void visit_object_code_block(object *obj, Visitor visitor); - template void visit_context_code_blocks(Visitor visitor); - template void visit_callback_code_blocks(Visitor visitor); - //code heap inline void check_code_pointer(cell ptr) { From 45a955b5bb4d691fd1d4e421876db661d14fbc50 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Sat, 24 Oct 2009 04:43:11 -0500 Subject: [PATCH 22/74] vm: fix typo in card tracing logic --- vm/collector.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/collector.hpp b/vm/collector.hpp index e0ff53df9b..54683556b1 100644 --- a/vm/collector.hpp +++ b/vm/collector.hpp @@ -229,7 +229,7 @@ scan_next_object: { start = gen->next_object_after(start); if(start) { - binary_start = ((object *)start)->binary_payload_start(); + binary_start = start + ((object *)start)->binary_payload_start(); end = start + ((object *)start)->size(); goto scan_next_object; } From 62e718eaa922b5d72aeef3a190feb1e8e57f664d Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Sat, 24 Oct 2009 21:24:06 -0500 Subject: [PATCH 23/74] vm: combine heap_block and code_block structs, eliminates some boilerplate --- vm/code_block.cpp | 7 +++--- vm/code_heap.cpp | 11 ++++----- vm/code_heap.hpp | 4 ++-- vm/compaction.cpp | 11 ++++----- vm/debug.cpp | 10 ++++---- vm/free_list_allocator.hpp | 29 ++++++++++++++++++---- vm/layouts.hpp | 49 ++++++++++---------------------------- vm/vm.hpp | 12 +--------- 8 files changed, 59 insertions(+), 74 deletions(-) diff --git a/vm/code_block.cpp b/vm/code_block.cpp index 687bbcf500..831c2388bb 100755 --- a/vm/code_block.cpp +++ b/vm/code_block.cpp @@ -439,7 +439,7 @@ void factor_vm::fixup_labels(array *labels, code_block *compiled) /* Might GC */ code_block *factor_vm::allot_code_block(cell size, code_block_type type) { - heap_block *block = code->allocator->allot(size + sizeof(code_block)); + code_block *block = code->allocator->allot(size + sizeof(code_block)); /* If allocation failed, do a full GC and compact the code heap. A full GC that occurs as a result of the data heap filling up does not @@ -465,9 +465,8 @@ code_block *factor_vm::allot_code_block(cell size, code_block_type type) } } - code_block *compiled = (code_block *)block; - compiled->set_type(type); - return compiled; + block->set_type(type); + return block; } /* Might GC */ diff --git a/vm/code_heap.cpp b/vm/code_heap.cpp index ef257bb935..bd40b1f0da 100755 --- a/vm/code_heap.cpp +++ b/vm/code_heap.cpp @@ -8,7 +8,7 @@ code_heap::code_heap(cell size) if(size > (1L << (sizeof(cell) * 8 - 6))) fatal_error("Heap too large",size); seg = new segment(align_page(size),true); if(!seg) fatal_error("Out of memory in heap allocator",size); - allocator = new free_list_allocator(size,seg->start); + allocator = new free_list_allocator(size,seg->start); } code_heap::~code_heap() @@ -36,7 +36,7 @@ bool code_heap::needs_fixup_p(code_block *compiled) return needs_fixup.count(compiled) > 0; } -bool code_heap::marked_p(heap_block *compiled) +bool code_heap::marked_p(code_block *compiled) { return allocator->state.marked_p(compiled); } @@ -109,9 +109,9 @@ struct word_and_literal_code_heap_updater { word_and_literal_code_heap_updater(factor_vm *parent_) : parent(parent_) {} - void operator()(heap_block *block, cell size) + void operator()(code_block *block, cell size) { - parent->update_code_block_words_and_literals((code_block *)block); + parent->update_code_block_words_and_literals(block); } }; @@ -137,8 +137,7 @@ struct code_heap_relocator { void factor_vm::relocate_code_heap() { code_heap_relocator relocator(this); - code_heap_iterator iter(relocator); - code->allocator->sweep(iter); + code->allocator->sweep(relocator); } void factor_vm::primitive_modify_code_heap() diff --git a/vm/code_heap.hpp b/vm/code_heap.hpp index 5548892d3f..38e53d9fbe 100755 --- a/vm/code_heap.hpp +++ b/vm/code_heap.hpp @@ -6,7 +6,7 @@ struct code_heap { segment *seg; /* Memory allocator */ - free_list_allocator *allocator; + free_list_allocator *allocator; /* Set of blocks which need full relocation. */ std::set needs_fixup; @@ -22,7 +22,7 @@ struct code_heap { void write_barrier(code_block *compiled); void clear_remembered_set(); bool needs_fixup_p(code_block *compiled); - bool marked_p(heap_block *compiled); + bool marked_p(code_block *compiled); void set_marked_p(code_block *compiled); void clear_mark_bits(); void code_heap_free(code_block *compiled); diff --git a/vm/compaction.cpp b/vm/compaction.cpp index 3cf5e5b46c..d29e8aa20d 100644 --- a/vm/compaction.cpp +++ b/vm/compaction.cpp @@ -15,14 +15,14 @@ struct object_slot_forwarder { }; struct code_block_forwarder { - mark_bits *forwarding_map; + mark_bits *forwarding_map; - explicit code_block_forwarder(mark_bits *forwarding_map_) : + explicit code_block_forwarder(mark_bits *forwarding_map_) : forwarding_map(forwarding_map_) {} code_block *operator()(code_block *compiled) { - return (code_block *)forwarding_map->forward_block(compiled); + return forwarding_map->forward_block(compiled); } }; @@ -63,7 +63,7 @@ void factor_vm::compact_full_impl(bool trace_contexts_p) { tenured_space *tenured = data->tenured; mark_bits *data_forwarding_map = &tenured->state; - mark_bits *code_forwarding_map = &code->allocator->state; + mark_bits *code_forwarding_map = &code->allocator->state; /* Figure out where blocks are going to go */ data_forwarding_map->compute_forwarding(); @@ -89,8 +89,7 @@ void factor_vm::compact_full_impl(bool trace_contexts_p) /* Slide everything in the code heap up, and update data and code heap pointers inside code blocks. */ code_block_compaction_updater code_block_updater(this,slot_forwarder); - code_heap_iterator iter(code_block_updater); - code->allocator->compact(iter); + code->allocator->compact(code_block_updater); } } diff --git a/vm/debug.cpp b/vm/debug.cpp index 91fb1ea1d3..1ee2b858fb 100755 --- a/vm/debug.cpp +++ b/vm/debug.cpp @@ -293,21 +293,21 @@ struct code_block_printer { code_block_printer(factor_vm *parent_) : parent(parent_), reloc_size(0), literal_size(0) {} - void operator()(heap_block *scan, cell size) + void operator()(code_block *scan, cell size) { const char *status; if(scan->free_p()) status = "free"; else if(parent->code->marked_p(scan)) { - reloc_size += parent->object_size(((code_block *)scan)->relocation); - literal_size += parent->object_size(((code_block *)scan)->literals); + reloc_size += parent->object_size(scan->relocation); + literal_size += parent->object_size(scan->literals); status = "marked"; } else { - reloc_size += parent->object_size(((code_block *)scan)->relocation); - literal_size += parent->object_size(((code_block *)scan)->literals); + reloc_size += parent->object_size(scan->relocation); + literal_size += parent->object_size(scan->literals); status = "allocated"; } diff --git a/vm/free_list_allocator.hpp b/vm/free_list_allocator.hpp index 8332399279..01e7ea6116 100644 --- a/vm/free_list_allocator.hpp +++ b/vm/free_list_allocator.hpp @@ -3,6 +3,27 @@ namespace factor static const cell free_list_count = 32; +struct free_heap_block +{ + cell header; + free_heap_block *next_free; + + bool free_p() const + { + return header & 1 == 1; + } + + cell size() const + { + return header >> 3; + } + + void make_free(cell size) + { + header = (size << 3) | 1; + } +}; + struct free_list { free_heap_block *small_blocks[free_list_count]; free_heap_block *large_blocks; @@ -248,7 +269,7 @@ void free_list_allocator::sweep() if(prev && prev->free_p()) { free_heap_block *free_prev = (free_heap_block *)prev; - free_prev->set_size(free_prev->size() + size); + free_prev->make_free(free_prev->size() + size); } else prev = scan; @@ -264,7 +285,7 @@ void free_list_allocator::sweep() if(prev && prev->free_p()) { free_heap_block *free_prev = (free_heap_block *)prev; - free_prev->set_size(free_prev->size() + size); + free_prev->make_free(free_prev->size() + size); } else { @@ -300,7 +321,7 @@ void free_list_allocator::sweep(Iterator &iter) if(prev && prev->free_p()) { free_heap_block *free_prev = (free_heap_block *)prev; - free_prev->set_size(free_prev->size() + size); + free_prev->make_free(free_prev->size() + size); } else prev = scan; @@ -317,7 +338,7 @@ void free_list_allocator::sweep(Iterator &iter) if(prev && prev->free_p()) { free_heap_block *free_prev = (free_heap_block *)prev; - free_prev->set_size(free_prev->size() + size); + free_prev->make_free(free_prev->size() + size); } else { diff --git a/vm/layouts.hpp b/vm/layouts.hpp index b3cba58495..f6c88064d4 100644 --- a/vm/layouts.hpp +++ b/vm/layouts.hpp @@ -221,49 +221,16 @@ struct string : public object { }; /* The compiled code heap is structured into blocks. */ -struct heap_block +struct code_block { cell header; - - bool free_p() const - { - return header & 1 == 1; - } - - cell size() const - { - cell bytes = header >> 3; -#ifdef FACTOR_DEBUG - assert(bytes > 0); -#endif - return bytes; - } - - void set_size(cell size) - { - header = ((header & 0x7) | (size << 3)); - } -}; - -struct free_heap_block : public heap_block -{ - free_heap_block *next_free; - - void make_free(cell size) - { - header = (size << 3) | 1; - } -}; - -struct code_block : public heap_block -{ cell owner; /* tagged pointer to word, quotation or f */ cell literals; /* tagged pointer to array or f */ cell relocation; /* tagged pointer to byte-array or f */ - void *xt() const + bool free_p() const { - return (void *)(this + 1); + return header & 1 == 1; } code_block_type type() const @@ -285,6 +252,16 @@ struct code_block : public heap_block { return type() == code_block_optimized; } + + cell size() const + { + return header >> 3; + } + + void *xt() const + { + return (void *)(this + 1); + } }; /* Assembly code makes assumptions about the layout of this struct */ diff --git a/vm/vm.hpp b/vm/vm.hpp index 762a34d225..29084d255d 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -504,18 +504,8 @@ struct factor_vm void primitive_strip_stack_traces(); /* Apply a function to every code block */ - template struct code_heap_iterator { - Iterator &iter; - explicit code_heap_iterator(Iterator &iter_) : iter(iter_) {} - void operator()(heap_block *block, cell size) - { - iter((code_block *)block,size); - } - }; - - template void iterate_code_heap(Iterator &iter_) + template void iterate_code_heap(Iterator &iter) { - code_heap_iterator iter(iter_); code->allocator->iterate(iter); } From fae27fb3619099b2c6518e96b87350c9c1bec39a Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Sat, 24 Oct 2009 23:02:58 -0500 Subject: [PATCH 24/74] vm: make some more ctors explicit just for kicks --- vm/aging_collector.hpp | 4 ++-- vm/aging_space.hpp | 2 +- vm/bump_allocator.hpp | 2 +- vm/code_heap.cpp | 5 +++-- vm/copying_collector.hpp | 2 +- vm/debug.cpp | 2 +- vm/full_collector.cpp | 2 +- vm/full_collector.hpp | 4 ++-- vm/image.cpp | 2 +- vm/local_roots.hpp | 2 +- vm/nursery_collector.hpp | 4 ++-- vm/nursery_space.hpp | 2 +- vm/object_start_map.hpp | 2 +- vm/slot_visitor.hpp | 2 +- vm/tenured_space.hpp | 2 +- vm/to_tenured_collector.hpp | 4 ++-- 16 files changed, 22 insertions(+), 21 deletions(-) diff --git a/vm/aging_collector.hpp b/vm/aging_collector.hpp index a04261d826..56550b211a 100644 --- a/vm/aging_collector.hpp +++ b/vm/aging_collector.hpp @@ -6,7 +6,7 @@ struct aging_policy { aging_space *aging; tenured_space *tenured; - aging_policy(factor_vm *parent_) : + explicit aging_policy(factor_vm *parent_) : parent(parent_), aging(parent->data->aging), tenured(parent->data->tenured) {} @@ -22,7 +22,7 @@ struct aging_policy { }; struct aging_collector : copying_collector { - aging_collector(factor_vm *parent_); + explicit aging_collector(factor_vm *parent_); }; } diff --git a/vm/aging_space.hpp b/vm/aging_space.hpp index 99efd44de5..7a28f54ebf 100644 --- a/vm/aging_space.hpp +++ b/vm/aging_space.hpp @@ -4,7 +4,7 @@ namespace factor struct aging_space : bump_allocator { object_start_map starts; - aging_space(cell size, cell start) : + explicit aging_space(cell size, cell start) : bump_allocator(size,start), starts(size,start) {} object *allot(cell size) diff --git a/vm/bump_allocator.hpp b/vm/bump_allocator.hpp index b41613b540..8f4fabe9a7 100644 --- a/vm/bump_allocator.hpp +++ b/vm/bump_allocator.hpp @@ -8,7 +8,7 @@ template struct bump_allocator { cell end; cell size; - bump_allocator(cell size_, cell start_) : + explicit bump_allocator(cell size_, cell start_) : here(start_), start(start_), end(start_ + size_), size(size_) {} inline bool contains_p(Block *block) diff --git a/vm/code_heap.cpp b/vm/code_heap.cpp index bd40b1f0da..2ce6d00b7c 100755 --- a/vm/code_heap.cpp +++ b/vm/code_heap.cpp @@ -88,6 +88,7 @@ struct word_updater { factor_vm *parent; explicit word_updater(factor_vm *parent_) : parent(parent_) {} + void operator()(code_block *compiled, cell size) { parent->update_word_references(compiled); @@ -107,7 +108,7 @@ to literals and other words. */ struct word_and_literal_code_heap_updater { factor_vm *parent; - word_and_literal_code_heap_updater(factor_vm *parent_) : parent(parent_) {} + explicit word_and_literal_code_heap_updater(factor_vm *parent_) : parent(parent_) {} void operator()(code_block *block, cell size) { @@ -126,7 +127,7 @@ references to card and deck arrays. */ struct code_heap_relocator { factor_vm *parent; - code_heap_relocator(factor_vm *parent_) : parent(parent_) {} + explicit code_heap_relocator(factor_vm *parent_) : parent(parent_) {} void operator()(code_block *block, cell size) { diff --git a/vm/copying_collector.hpp b/vm/copying_collector.hpp index 012aa4ec10..f79f97d34e 100644 --- a/vm/copying_collector.hpp +++ b/vm/copying_collector.hpp @@ -7,7 +7,7 @@ struct dummy_unmarker { struct simple_unmarker { card unmask; - simple_unmarker(card unmask_) : unmask(unmask_) {} + explicit simple_unmarker(card unmask_) : unmask(unmask_) {} void operator()(card *ptr) { *ptr &= ~unmask; } }; diff --git a/vm/debug.cpp b/vm/debug.cpp index 1ee2b858fb..0598d164e6 100755 --- a/vm/debug.cpp +++ b/vm/debug.cpp @@ -290,7 +290,7 @@ struct code_block_printer { factor_vm *parent; cell reloc_size, literal_size; - code_block_printer(factor_vm *parent_) : + explicit code_block_printer(factor_vm *parent_) : parent(parent_), reloc_size(0), literal_size(0) {} void operator()(code_block *scan, cell size) diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index 2afb4181f7..18e368ab2b 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -32,7 +32,7 @@ struct code_block_marker { struct object_start_map_updater { object_start_map *starts; - object_start_map_updater(object_start_map *starts_) : starts(starts_) {} + explicit object_start_map_updater(object_start_map *starts_) : starts(starts_) {} void operator()(object *obj, cell size) { diff --git a/vm/full_collector.hpp b/vm/full_collector.hpp index c9750302d1..eb125b7429 100644 --- a/vm/full_collector.hpp +++ b/vm/full_collector.hpp @@ -5,7 +5,7 @@ struct full_policy { factor_vm *parent; tenured_space *tenured; - full_policy(factor_vm *parent_) : parent(parent_), tenured(parent->data->tenured) {} + explicit full_policy(factor_vm *parent_) : parent(parent_), tenured(parent->data->tenured) {} bool should_copy_p(object *untagged) { @@ -27,7 +27,7 @@ struct full_policy { struct full_collector : collector { bool trace_contexts_p; - full_collector(factor_vm *parent_); + explicit full_collector(factor_vm *parent_); }; } diff --git a/vm/image.cpp b/vm/image.cpp index 1b7debc2b2..bee351b830 100755 --- a/vm/image.cpp +++ b/vm/image.cpp @@ -215,7 +215,7 @@ struct code_block_fixupper { factor_vm *parent; cell data_relocation_base; - code_block_fixupper(factor_vm *parent_, cell data_relocation_base_) : + explicit code_block_fixupper(factor_vm *parent_, cell data_relocation_base_) : parent(parent_), data_relocation_base(data_relocation_base_) { } void operator()(code_block *compiled, cell size) diff --git a/vm/local_roots.hpp b/vm/local_roots.hpp index 58142be8f2..442a91f350 100644 --- a/vm/local_roots.hpp +++ b/vm/local_roots.hpp @@ -7,7 +7,7 @@ struct gc_root : public tagged factor_vm *parent; void push() { parent->gc_locals.push_back((cell)this); } - + explicit gc_root(cell value_,factor_vm *vm) : tagged(value_),parent(vm) { push(); } explicit gc_root(Type *value_, factor_vm *vm) : tagged(value_),parent(vm) { push(); } diff --git a/vm/nursery_collector.hpp b/vm/nursery_collector.hpp index 778efab138..de9b38d283 100644 --- a/vm/nursery_collector.hpp +++ b/vm/nursery_collector.hpp @@ -4,7 +4,7 @@ namespace factor struct nursery_policy { factor_vm *parent; - nursery_policy(factor_vm *parent_) : parent(parent_) {} + explicit nursery_policy(factor_vm *parent_) : parent(parent_) {} bool should_copy_p(object *obj) { @@ -17,7 +17,7 @@ struct nursery_policy { }; struct nursery_collector : copying_collector { - nursery_collector(factor_vm *parent_); + explicit nursery_collector(factor_vm *parent_); }; } diff --git a/vm/nursery_space.hpp b/vm/nursery_space.hpp index 4425c1612b..c44d2a8e44 100644 --- a/vm/nursery_space.hpp +++ b/vm/nursery_space.hpp @@ -3,7 +3,7 @@ namespace factor struct nursery_space : bump_allocator { - nursery_space(cell size, cell start) : bump_allocator(size,start) {} + explicit nursery_space(cell size, cell start) : bump_allocator(size,start) {} }; } diff --git a/vm/object_start_map.hpp b/vm/object_start_map.hpp index 640e205852..69f9c11a6a 100644 --- a/vm/object_start_map.hpp +++ b/vm/object_start_map.hpp @@ -8,7 +8,7 @@ struct object_start_map { card *object_start_offsets; card *object_start_offsets_end; - object_start_map(cell size_, cell start_); + explicit object_start_map(cell size_, cell start_); ~object_start_map(); cell first_object_in_card(cell card_index); diff --git a/vm/slot_visitor.hpp b/vm/slot_visitor.hpp index 67a51549b1..b564949072 100644 --- a/vm/slot_visitor.hpp +++ b/vm/slot_visitor.hpp @@ -5,7 +5,7 @@ template struct slot_visitor { factor_vm *parent; Visitor visitor; - slot_visitor(factor_vm *parent_, Visitor visitor_) : + explicit slot_visitor(factor_vm *parent_, Visitor visitor_) : parent(parent_), visitor(visitor_) {} void visit_handle(cell *handle) diff --git a/vm/tenured_space.hpp b/vm/tenured_space.hpp index 7cc4131fa0..1b3baeaf52 100644 --- a/vm/tenured_space.hpp +++ b/vm/tenured_space.hpp @@ -5,7 +5,7 @@ struct tenured_space : free_list_allocator { object_start_map starts; std::vector mark_stack; - tenured_space(cell size, cell start) : + explicit tenured_space(cell size, cell start) : free_list_allocator(size,start), starts(size,start) {} object *allot(cell size) diff --git a/vm/to_tenured_collector.hpp b/vm/to_tenured_collector.hpp index e87ba5ee29..2f2717efd1 100644 --- a/vm/to_tenured_collector.hpp +++ b/vm/to_tenured_collector.hpp @@ -5,7 +5,7 @@ struct to_tenured_policy { factor_vm *myvm; tenured_space *tenured; - to_tenured_policy(factor_vm *myvm_) : myvm(myvm_), tenured(myvm->data->tenured) {} + explicit to_tenured_policy(factor_vm *myvm_) : myvm(myvm_), tenured(myvm->data->tenured) {} bool should_copy_p(object *untagged) { @@ -21,7 +21,7 @@ struct to_tenured_policy { }; struct to_tenured_collector : collector { - to_tenured_collector(factor_vm *myvm_); + explicit to_tenured_collector(factor_vm *myvm_); void tenure_reachable_objects(); }; From 0b65b194c9b1a5322518383a8e618fd137a29e65 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Sat, 24 Oct 2009 23:49:33 -0500 Subject: [PATCH 25/74] vm: fix 'data-room' primitive to report correct sizes for tenured space --- vm/data_heap.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/data_heap.cpp b/vm/data_heap.cpp index b210adb8e1..57f6608e6b 100755 --- a/vm/data_heap.cpp +++ b/vm/data_heap.cpp @@ -219,7 +219,7 @@ void factor_vm::primitive_data_room() cell used, total_free, max_free; data->tenured->usage(&used,&total_free,&max_free); a.add(tag_fixnum(total_free >> 10)); - a.add(tag_fixnum(used >> 10)); + a.add(tag_fixnum(data->tenured->size >> 10)); a.trim(); dpush(a.elements.value()); From 0c1e2663029d30242a3f5f55ac28060b27bb94ce Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Sat, 24 Oct 2009 23:51:14 -0500 Subject: [PATCH 26/74] vm: got data heap compaction working --- vm/compaction.cpp | 67 ++++++++++++++++++++++++++++++++------ vm/free_list_allocator.hpp | 31 ++++++++++++++---- vm/object_start_map.cpp | 1 + vm/slot_visitor.hpp | 9 +++-- 4 files changed, 89 insertions(+), 19 deletions(-) diff --git a/vm/compaction.cpp b/vm/compaction.cpp index d29e8aa20d..05d6a186db 100644 --- a/vm/compaction.cpp +++ b/vm/compaction.cpp @@ -26,21 +26,66 @@ struct code_block_forwarder { } }; +static inline cell tuple_size_with_forwarding(mark_bits *forwarding_map, object *obj) +{ + /* The tuple layout may or may not have been forwarded already. Tricky. */ + object *layout_obj = (object *)UNTAG(((tuple *)obj)->layout); + tuple_layout *layout; + + if(layout_obj < obj) + { + /* It's already been moved up; dereference through forwarding + map to get the size */ + layout = (tuple_layout *)forwarding_map->forward_block(layout_obj); + } + else + { + /* It hasn't been moved up yet; dereference directly */ + layout = (tuple_layout *)layout_obj; + } + + return tuple_size(layout); +} + +struct compaction_sizer { + mark_bits *forwarding_map; + + explicit compaction_sizer(mark_bits *forwarding_map_) : + forwarding_map(forwarding_map_) {} + + cell operator()(object *obj) + { + if(obj->free_p() || obj->h.hi_tag() != TUPLE_TYPE) + return obj->size(); + else + return align(tuple_size_with_forwarding(forwarding_map,obj),data_alignment); + } +}; + struct object_compaction_updater { factor_vm *parent; slot_visitor slot_forwarder; code_block_visitor code_forwarder; + mark_bits *data_forwarding_map; explicit object_compaction_updater(factor_vm *parent_, - slot_visitor slot_forwader_, - code_block_visitor code_forwarder_) : + slot_visitor slot_forwarder_, + code_block_visitor code_forwarder_, + mark_bits *data_forwarding_map_) : parent(parent_), - slot_forwarder(slot_forwader_), - code_forwarder(code_forwarder_) {} + slot_forwarder(slot_forwarder_), + code_forwarder(code_forwarder_), + data_forwarding_map(data_forwarding_map_) {} void operator()(object *obj, cell size) { - slot_forwarder.visit_slots(obj); + cell payload_start; + if(obj->h.hi_tag() == TUPLE_TYPE) + payload_start = tuple_size_with_forwarding(data_forwarding_map,obj); + else + payload_start = obj->binary_payload_start(); + + slot_forwarder.visit_slots(obj,payload_start); code_forwarder.visit_object_code_block(obj); } }; @@ -49,8 +94,8 @@ struct code_block_compaction_updater { factor_vm *parent; slot_visitor slot_forwarder; - explicit code_block_compaction_updater(factor_vm *parent_, slot_visitor slot_forwader_) : - parent(parent_), slot_forwarder(slot_forwader_) {} + explicit code_block_compaction_updater(factor_vm *parent_, slot_visitor slot_forwarder_) : + parent(parent_), slot_forwarder(slot_forwarder_) {} void operator()(code_block *compiled, cell size) { @@ -83,13 +128,15 @@ void factor_vm::compact_full_impl(bool trace_contexts_p) /* Slide everything in tenured space up, and update data and code heap pointers inside objects. */ - object_compaction_updater object_updater(this,slot_forwarder,code_forwarder); - tenured->compact(object_updater); + object_compaction_updater object_updater(this,slot_forwarder,code_forwarder,data_forwarding_map); + compaction_sizer object_sizer(data_forwarding_map); + tenured->compact(object_updater,object_sizer); /* Slide everything in the code heap up, and update data and code heap pointers inside code blocks. */ code_block_compaction_updater code_block_updater(this,slot_forwarder); - code->allocator->compact(code_block_updater); + standard_sizer code_block_sizer; + code->allocator->compact(code_block_updater,code_block_sizer); } } diff --git a/vm/free_list_allocator.hpp b/vm/free_list_allocator.hpp index 01e7ea6116..0f2271beac 100644 --- a/vm/free_list_allocator.hpp +++ b/vm/free_list_allocator.hpp @@ -53,7 +53,8 @@ template struct free_list_allocator { cell occupied(); void sweep(); template void sweep(Iterator &iter); - template void compact(Iterator &iter); + template void compact(Iterator &iter, Sizer &sizer); + template void iterate(Iterator &iter, Sizer &sizer); template void iterate(Iterator &iter); }; @@ -358,31 +359,47 @@ void free_list_allocator::sweep(Iterator &iter) /* The forwarding map must be computed first by calling state.compute_forwarding(). */ template -template -void free_list_allocator::compact(Iterator &iter) +template +void free_list_allocator::compact(Iterator &iter, Sizer &sizer) { heap_compactor compactor(&state,first_block(),iter); - this->iterate(compactor); + this->iterate(compactor,sizer); /* Now update the free list; there will be a single free block at the end */ this->initial_free_list((cell)compactor.address - this->start); } +/* During compaction we have to be careful and measure object sizes differently */ template -template -void free_list_allocator::iterate(Iterator &iter) +template +void free_list_allocator::iterate(Iterator &iter, Sizer &sizer) { Block *scan = first_block(); Block *end = last_block(); while(scan != end) { - cell size = scan->size(); + cell size = sizer(scan); Block *next = (Block *)((cell)scan + size); if(!scan->free_p()) iter(scan,size); scan = next; } } +template struct standard_sizer { + cell operator()(Block *block) + { + return block->size(); + } +}; + +template +template +void free_list_allocator::iterate(Iterator &iter) +{ + standard_sizer sizer; + iterate(iter,sizer); +} + } diff --git a/vm/object_start_map.cpp b/vm/object_start_map.cpp index 5f992e783e..cb4f86c6c3 100644 --- a/vm/object_start_map.cpp +++ b/vm/object_start_map.cpp @@ -8,6 +8,7 @@ object_start_map::object_start_map(cell size_, cell start_) : { object_start_offsets = new card[addr_to_card(size_)]; object_start_offsets_end = object_start_offsets + addr_to_card(size_); + clear_object_start_offsets(); } object_start_map::~object_start_map() diff --git a/vm/slot_visitor.hpp b/vm/slot_visitor.hpp index b564949072..48fb0c1af6 100644 --- a/vm/slot_visitor.hpp +++ b/vm/slot_visitor.hpp @@ -19,10 +19,10 @@ template struct slot_visitor { *handle = RETAG(untagged,TAG(pointer)); } - void visit_slots(object *ptr) + void visit_slots(object *ptr, cell payload_start) { cell *slot = (cell *)ptr; - cell *end = (cell *)((cell)ptr + ptr->binary_payload_start()); + cell *end = (cell *)((cell)ptr + payload_start); if(slot != end) { @@ -31,6 +31,11 @@ template struct slot_visitor { } } + void visit_slots(object *ptr) + { + visit_slots(ptr,ptr->binary_payload_start()); + } + void visit_stack_elements(segment *region, cell *top) { for(cell *ptr = (cell *)region->start; ptr <= top; ptr++) From b6a21b19a99c0f01631e139b436b24699fec7fb1 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Sun, 25 Oct 2009 00:06:45 -0500 Subject: [PATCH 27/74] vm: fix compaction when callback heap has entries in it --- vm/compaction.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/vm/compaction.cpp b/vm/compaction.cpp index 05d6a186db..a98876ffd4 100644 --- a/vm/compaction.cpp +++ b/vm/compaction.cpp @@ -118,14 +118,6 @@ void factor_vm::compact_full_impl(bool trace_contexts_p) slot_visitor slot_forwarder(this,object_slot_forwarder(data_forwarding_map)); code_block_visitor code_forwarder(this,code_block_forwarder(code_forwarding_map)); - slot_forwarder.visit_roots(); - if(trace_contexts_p) - { - slot_forwarder.visit_contexts(); - code_forwarder.visit_context_code_blocks(); - code_forwarder.visit_callback_code_blocks(); - } - /* Slide everything in tenured space up, and update data and code heap pointers inside objects. */ object_compaction_updater object_updater(this,slot_forwarder,code_forwarder,data_forwarding_map); @@ -137,6 +129,14 @@ void factor_vm::compact_full_impl(bool trace_contexts_p) code_block_compaction_updater code_block_updater(this,slot_forwarder); standard_sizer code_block_sizer; code->allocator->compact(code_block_updater,code_block_sizer); + + slot_forwarder.visit_roots(); + if(trace_contexts_p) + { + slot_forwarder.visit_contexts(); + code_forwarder.visit_context_code_blocks(); + code_forwarder.visit_callback_code_blocks(); + } } } From b7181d14a8340a2ad8f9db143bd7a63401c21d39 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Sun, 25 Oct 2009 08:07:21 -0500 Subject: [PATCH 28/74] vm: debugging compaction --- vm/compaction.cpp | 31 ++++++++++++++++++++----------- vm/free_list_allocator.hpp | 18 ++++++++++++++++++ vm/full_collector.cpp | 27 ++++++++++++++++++--------- vm/mark_bits.hpp | 28 ++++++++-------------------- vm/vm.hpp | 5 +++-- 5 files changed, 67 insertions(+), 42 deletions(-) diff --git a/vm/compaction.cpp b/vm/compaction.cpp index a98876ffd4..74dd8935da 100644 --- a/vm/compaction.cpp +++ b/vm/compaction.cpp @@ -67,6 +67,7 @@ struct object_compaction_updater { slot_visitor slot_forwarder; code_block_visitor code_forwarder; mark_bits *data_forwarding_map; + object_start_map *starts; explicit object_compaction_updater(factor_vm *parent_, slot_visitor slot_forwarder_, @@ -75,18 +76,22 @@ struct object_compaction_updater { parent(parent_), slot_forwarder(slot_forwarder_), code_forwarder(code_forwarder_), - data_forwarding_map(data_forwarding_map_) {} + data_forwarding_map(data_forwarding_map_), + starts(&parent->data->tenured->starts) {} - void operator()(object *obj, cell size) + void operator()(object *old_address, object *new_address, cell size) { cell payload_start; - if(obj->h.hi_tag() == TUPLE_TYPE) - payload_start = tuple_size_with_forwarding(data_forwarding_map,obj); + if(old_address->h.hi_tag() == TUPLE_TYPE) + payload_start = tuple_size_with_forwarding(data_forwarding_map,old_address); else - payload_start = obj->binary_payload_start(); + payload_start = old_address->binary_payload_start(); - slot_forwarder.visit_slots(obj,payload_start); - code_forwarder.visit_object_code_block(obj); + memmove(new_address,old_address,size); + + slot_forwarder.visit_slots(new_address,payload_start); + code_forwarder.visit_object_code_block(new_address); + starts->record_object_start_offset(new_address); } }; @@ -97,14 +102,15 @@ struct code_block_compaction_updater { explicit code_block_compaction_updater(factor_vm *parent_, slot_visitor slot_forwarder_) : parent(parent_), slot_forwarder(slot_forwarder_) {} - void operator()(code_block *compiled, cell size) + void operator()(code_block *old_address, code_block *new_address, cell size) { - slot_forwarder.visit_literal_references(compiled); - parent->relocate_code_block(compiled); + memmove(new_address,old_address,size); + slot_forwarder.visit_literal_references(new_address); + parent->relocate_code_block(new_address); } }; -void factor_vm::compact_full_impl(bool trace_contexts_p) +void factor_vm::collect_full_compact(bool trace_contexts_p) { tenured_space *tenured = data->tenured; mark_bits *data_forwarding_map = &tenured->state; @@ -118,6 +124,9 @@ void factor_vm::compact_full_impl(bool trace_contexts_p) slot_visitor slot_forwarder(this,object_slot_forwarder(data_forwarding_map)); code_block_visitor code_forwarder(this,code_block_forwarder(code_forwarding_map)); + /* Object start offsets get recomputed by the object_compaction_updater */ + data->tenured->starts.clear_object_start_offsets(); + /* Slide everything in tenured space up, and update data and code heap pointers inside objects. */ object_compaction_updater object_updater(this,slot_forwarder,code_forwarder,data_forwarding_map); diff --git a/vm/free_list_allocator.hpp b/vm/free_list_allocator.hpp index 0f2271beac..822a40f797 100644 --- a/vm/free_list_allocator.hpp +++ b/vm/free_list_allocator.hpp @@ -356,6 +356,24 @@ void free_list_allocator::sweep(Iterator &iter) this->add_to_free_list((free_heap_block *)prev); } +template struct heap_compactor { + mark_bits *state; + char *address; + Iterator &iter; + + explicit heap_compactor(mark_bits *state_, Block *address_, Iterator &iter_) : + state(state_), address((char *)address_), iter(iter_) {} + + void operator()(Block *block, cell size) + { + if(this->state->marked_p(block)) + { + iter(block,(Block *)address,size); + address += size; + } + } +}; + /* The forwarding map must be computed first by calling state.compute_forwarding(). */ template diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index 18e368ab2b..65d0edbd47 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -40,7 +40,7 @@ struct object_start_map_updater { } }; -void factor_vm::collect_full_impl(bool trace_contexts_p) +void factor_vm::collect_full_mark(bool trace_contexts_p) { full_collector collector(this); @@ -68,16 +68,19 @@ void factor_vm::collect_full_impl(bool trace_contexts_p) code_marker.visit_object_code_block(obj); } - data->tenured->starts.clear_object_start_offsets(); - object_start_map_updater updater(&data->tenured->starts); - data->tenured->sweep(updater); - data->reset_generation(data->tenured); data->reset_generation(data->aging); data->reset_generation(&nursery); code->clear_remembered_set(); } +void factor_vm::collect_full_sweep() +{ + data->tenured->starts.clear_object_start_offsets(); + object_start_map_updater updater(&data->tenured->starts); + data->tenured->sweep(updater); +} + void factor_vm::collect_growing_heap(cell requested_bytes, bool trace_contexts_p, bool compact_p) @@ -85,23 +88,29 @@ void factor_vm::collect_growing_heap(cell requested_bytes, /* Grow the data heap and copy all live objects to the new heap. */ data_heap *old = data; set_data_heap(data->grow(requested_bytes)); - collect_full_impl(trace_contexts_p); + collect_full_mark(trace_contexts_p); delete old; if(compact_p) - compact_full_impl(trace_contexts_p); + collect_full_compact(trace_contexts_p); else + { + collect_full_sweep(); relocate_code_heap(); + } } void factor_vm::collect_full(bool trace_contexts_p, bool compact_p) { - collect_full_impl(trace_contexts_p); + collect_full_mark(trace_contexts_p); if(compact_p) - compact_full_impl(trace_contexts_p); + collect_full_compact(trace_contexts_p); else + { + collect_full_sweep(); update_code_heap_words_and_literals(); + } } } diff --git a/vm/mark_bits.hpp b/vm/mark_bits.hpp index cd739346f0..8b6b0c75eb 100644 --- a/vm/mark_bits.hpp +++ b/vm/mark_bits.hpp @@ -139,32 +139,20 @@ template struct mark_bits { /* We have the popcount for every 64 entries; look up and compute the rest */ Block *forward_block(Block *original) { +#ifdef FACTOR_DEBUG + assert(marked_p(original)); +#endif std::pair pair = bitmap_deref(original); cell approx_popcount = forwarding[pair.first]; u64 mask = ((u64)1 << pair.second) - 1; cell new_line_number = approx_popcount + popcount(marked[pair.first] & mask); - return line_block(new_line_number); - } -}; - -template struct heap_compactor { - mark_bits *state; - char *address; - Iterator &iter; - - explicit heap_compactor(mark_bits *state_, Block *address_, Iterator &iter_) : - state(state_), address((char *)address_), iter(iter_) {} - - void operator()(Block *block, cell size) - { - if(this->state->marked_p(block)) - { - memmove(address,block,size); - iter((Block *)address,size); - address += size; - } + Block *new_block = line_block(new_line_number); +#ifdef FACTOR_DEBUG + assert(new_block <= original); +#endif + return new_block; } }; diff --git a/vm/vm.hpp b/vm/vm.hpp index 29084d255d..f0f37619d2 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -247,8 +247,9 @@ struct factor_vm void collect_nursery(); void collect_aging(); void collect_to_tenured(); - void collect_full_impl(bool trace_contexts_p); - void compact_full_impl(bool trace_contexts_p); + void collect_full_mark(bool trace_contexts_p); + void collect_full_sweep(); + void collect_full_compact(bool trace_contexts_p); void collect_growing_heap(cell requested_bytes, bool trace_contexts_p, bool compact_p); void collect_full(bool trace_contexts_p, bool compact_p); void record_gc_stats(generation_statistics *stats); From 49baf397f42bf870cdaa5f07faffe17ff9bcde41 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Sun, 25 Oct 2009 08:07:36 -0500 Subject: [PATCH 29/74] vm: tagged typechecks work better with DEBUG=1 --- vm/tagged.hpp | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/vm/tagged.hpp b/vm/tagged.hpp index c5325542cb..ea696c6358 100755 --- a/vm/tagged.hpp +++ b/vm/tagged.hpp @@ -16,13 +16,10 @@ struct tagged { cell value_; - cell value() const { return value_; } - Type *untagged() const { return (Type *)(UNTAG(value_)); } - cell type() const { cell tag = TAG(value_); if(tag == OBJECT_TYPE) - return untagged()->h.hi_tag(); + return ((object *)UNTAG(value_))->h.hi_tag(); else return tag; } @@ -40,23 +37,27 @@ struct tagged return type_p(Type::type_number); } + cell value() const { +#ifdef FACTOR_DEBUG + assert(type_p()); +#endif + return value_; + } + Type *untagged() const { +#ifdef FACTOR_DEBUG + assert(type_p()); +#endif + return (Type *)(UNTAG(value_)); + } + Type *untag_check(factor_vm *parent) const { if(!type_p()) parent->type_error(Type::type_number,value_); return untagged(); } - explicit tagged(cell tagged) : value_(tagged) { -#ifdef FACTOR_DEBUG - assert(type_p()); -#endif - } - - explicit tagged(Type *untagged) : value_(factor::tag(untagged)) { -#ifdef FACTOR_DEBUG - assert(type_p()); -#endif - } + explicit tagged(cell tagged) : value_(tagged) {} + explicit tagged(Type *untagged) : value_(factor::tag(untagged)) {} Type *operator->() const { return untagged(); } cell *operator&() const { return &value_; } From cf247c23a2f0e3597fa8f3bfd9826f2769de8d6a Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Sun, 25 Oct 2009 13:18:06 -0500 Subject: [PATCH 30/74] vm: room. now prints mark stack size, and total/contiguous free space --- .../known-words/known-words.factor | 4 +- basis/tools/memory/memory.factor | 78 +++++++------------ vm/code_heap.cpp | 14 +++- vm/data_heap.cpp | 18 +++-- 4 files changed, 53 insertions(+), 61 deletions(-) diff --git a/basis/stack-checker/known-words/known-words.factor b/basis/stack-checker/known-words/known-words.factor index 8cddac5a75..d064776673 100644 --- a/basis/stack-checker/known-words/known-words.factor +++ b/basis/stack-checker/known-words/known-words.factor @@ -507,10 +507,10 @@ M: bad-executable summary \ (save-image-and-exit) { byte-array } { } define-primitive -\ data-room { } { integer integer array } define-primitive +\ data-room { } { array } define-primitive \ data-room make-flushable -\ code-room { } { integer integer integer integer } define-primitive +\ code-room { } { array } define-primitive \ code-room make-flushable \ micros { } { integer } define-primitive diff --git a/basis/tools/memory/memory.factor b/basis/tools/memory/memory.factor index 81785f7ea4..2f1827a8ff 100644 --- a/basis/tools/memory/memory.factor +++ b/basis/tools/memory/memory.factor @@ -1,4 +1,4 @@ -! Copyright (C) 2005, 2008 Slava Pestov. +! Copyright (C) 2005, 2009 Slava Pestov. ! See http://factorcode.org/license.txt for BSD license. USING: kernel sequences arrays generic assocs io math namespaces parser prettyprint strings io.styles words @@ -8,48 +8,41 @@ IN: tools.memory string dup length 4 > [ 3 cut* "," glue ] when - " KB" append write-cell ; + " KB" append ; -: write-total/used/free ( free total str -- ) - [ - write-cell - dup write-size - over - write-size - write-size - ] with-row ; +: memory-table. ( sizes seq -- ) + swap [ kilobytes ] map zip simple-table. ; -: write-total ( n str -- ) - [ - write-cell - write-size - [ ] with-cell - [ ] with-cell - ] with-row ; +: young-room. ( seq -- ) + { "Total:" "Allocated:" "Free:" } memory-table. ; -: write-headings ( seq -- ) - [ [ write-cell ] each ] with-row ; +: nursery-room. ( seq -- ) "- Nursery space" print young-room. ; -: (data-room.) ( -- ) - data-room 2 [ - [ first2 ] [ number>string "Generation " prepend ] bi* - write-total/used/free - ] each-index - "Decks" write-total - "Cards" write-total ; +: aging-room. ( seq -- ) "- Aging space" print young-room. ; -: write-labeled-size ( n string -- ) - [ write-cell write-size ] with-row ; +: mark-sweep-table. ( sizes -- ) + { "Total:" "Allocated:" "Contiguous free:" "Total free:" } memory-table. ; -: (code-room.) ( -- ) - code-room { - [ "Size:" write-labeled-size ] - [ "Used:" write-labeled-size ] - [ "Total free space:" write-labeled-size ] - [ "Largest free block:" write-labeled-size ] - } spread ; +: tenured-room. ( seq -- ) "- Tenured space" print mark-sweep-table. ; + +: misc-room. ( seq -- ) + "- Miscellaneous buffers" print + { "Card array:" "Deck array:" "Mark stack:" } memory-table. ; + +: data-room. ( -- ) + "==== DATA HEAP" print nl + data-room + 3 cut [ nursery-room. nl ] dip + 3 cut [ aging-room. nl ] dip + 4 cut [ tenured-room. nl ] dip + misc-room. ; + +: code-room. ( -- ) + "==== CODE HEAP" print nl + code-room mark-sweep-table. ; : heap-stat-step ( obj counts sizes -- ) [ [ class ] dip inc-at ] @@ -57,18 +50,7 @@ IN: tools.memory PRIVATE> -: room. ( -- ) - "==== DATA HEAP" print - standard-table-style [ - { "" "Total" "Used" "Free" } write-headings - (data-room.) - ] tabular-output - nl nl - "==== CODE HEAP" print - standard-table-style [ - (code-room.) - ] tabular-output - nl ; +: room. ( -- ) data-room. nl code-room. ; : heap-stats ( -- counts sizes ) [ ] instances H{ } clone H{ } clone @@ -76,7 +58,7 @@ PRIVATE> : heap-stats. ( -- ) heap-stats dup keys natural-sort standard-table-style [ - { "Class" "Bytes" "Instances" } write-headings + [ { "Class" "Bytes" "Instances" } [ write-cell ] each ] with-row [ [ dup pprint-cell diff --git a/vm/code_heap.cpp b/vm/code_heap.cpp index 2ce6d00b7c..ae53869ef2 100755 --- a/vm/code_heap.cpp +++ b/vm/code_heap.cpp @@ -197,12 +197,18 @@ void factor_vm::primitive_modify_code_heap() /* Push the free space and total size of the code heap */ void factor_vm::primitive_code_room() { + growable_array a(this); + cell used, total_free, max_free; code->allocator->usage(&used,&total_free,&max_free); - dpush(tag_fixnum(code->seg->size / 1024)); - dpush(tag_fixnum(used / 1024)); - dpush(tag_fixnum(total_free / 1024)); - dpush(tag_fixnum(max_free / 1024)); + + a.add(tag_fixnum(code->seg->size >> 10)); + a.add(tag_fixnum(used >> 10)); + a.add(tag_fixnum(total_free >> 10)); + a.add(tag_fixnum(max_free >> 10)); + + a.trim(); + dpush(a.elements.value()); } struct stack_trace_stripper { diff --git a/vm/data_heap.cpp b/vm/data_heap.cpp index 57f6608e6b..3dd46fd848 100755 --- a/vm/data_heap.cpp +++ b/vm/data_heap.cpp @@ -204,22 +204,26 @@ void factor_vm::primitive_size() /* Push memory usage statistics in data heap */ void factor_vm::primitive_data_room() { - dpush(tag_fixnum((data->cards_end - data->cards) >> 10)); - dpush(tag_fixnum((data->decks_end - data->decks) >> 10)); - growable_array a(this); - a.add(tag_fixnum((nursery.end - nursery.here) >> 10)); a.add(tag_fixnum((nursery.size) >> 10)); + a.add(tag_fixnum((nursery.here - nursery.start) >> 10)); + a.add(tag_fixnum((nursery.end - nursery.here) >> 10)); - a.add(tag_fixnum((data->aging->end - data->aging->here) >> 10)); a.add(tag_fixnum((data->aging->size) >> 10)); + a.add(tag_fixnum((data->aging->here - data->aging->start) >> 10)); + a.add(tag_fixnum((data->aging->end - data->aging->here) >> 10)); - //XXX cell used, total_free, max_free; data->tenured->usage(&used,&total_free,&max_free); - a.add(tag_fixnum(total_free >> 10)); a.add(tag_fixnum(data->tenured->size >> 10)); + a.add(tag_fixnum(used >> 10)); + a.add(tag_fixnum(total_free >> 10)); + a.add(tag_fixnum(max_free >> 10)); + + a.add(tag_fixnum((data->cards_end - data->cards) >> 10)); + a.add(tag_fixnum((data->decks_end - data->decks) >> 10)); + a.add(tag_fixnum((data->tenured->mark_stack.capacity()) >> 10)); a.trim(); dpush(a.elements.value()); From 7d8c85443e31b1c55c3d765d9e40e49f07a84ae2 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Sun, 25 Oct 2009 13:27:40 -0500 Subject: [PATCH 31/74] vm: vm: fix large object allocation logic and change default heap image size for 2009 --- vm/factor.cpp | 2 +- vm/free_list_allocator.hpp | 22 ++++++++++++++++++++++ vm/gc.cpp | 30 ++++++++++++++---------------- vm/image.cpp | 5 +---- 4 files changed, 38 insertions(+), 21 deletions(-) diff --git a/vm/factor.cpp b/vm/factor.cpp index df27de84fd..24a3e01237 100755 --- a/vm/factor.cpp +++ b/vm/factor.cpp @@ -32,7 +32,7 @@ void factor_vm::default_parameters(vm_parameters *p) p->code_size = 8 * sizeof(cell); p->young_size = sizeof(cell) / 4; p->aging_size = sizeof(cell) / 2; - p->tenured_size = 4 * sizeof(cell); + p->tenured_size = 16 * sizeof(cell); #endif p->max_pic_size = 3; diff --git a/vm/free_list_allocator.hpp b/vm/free_list_allocator.hpp index 822a40f797..d796379799 100644 --- a/vm/free_list_allocator.hpp +++ b/vm/free_list_allocator.hpp @@ -47,6 +47,7 @@ template struct free_list_allocator { void assert_free_block(free_heap_block *block); free_heap_block *find_free_block(cell size); free_heap_block *split_free_block(free_heap_block *block, cell size); + bool can_allot_p(cell size); Block *allot(cell size); void free(Block *block); void usage(cell *used, cell *total_free, cell *max_free); @@ -180,6 +181,27 @@ template free_heap_block *free_list_allocator::split_free return block; } +template bool free_list_allocator::can_allot_p(cell size) +{ + cell attempt = size; + + while(attempt < free_list_count * block_granularity) + { + int index = attempt / block_granularity; + if(free_blocks.small_blocks[index]) return true; + attempt *= 2; + } + + free_heap_block *block = free_blocks.large_blocks; + while(block) + { + if(block->size() >= size) return true; + block = block->next_free; + } + + return false; +} + template Block *free_list_allocator::allot(cell size) { size = align(size,block_granularity); diff --git a/vm/gc.cpp b/vm/gc.cpp index 1851924cd5..4b530172c1 100755 --- a/vm/gc.cpp +++ b/vm/gc.cpp @@ -115,7 +115,7 @@ void factor_vm::primitive_full_gc() gc(collect_full_op, 0, /* requested size */ true, /* trace contexts? */ - false /* compact code heap? */); + true /* compact code heap? */); } void factor_vm::primitive_compact_gc() @@ -231,7 +231,7 @@ object *factor_vm::allot_object(header header, cell size) /* If the object is smaller than the nursery, allocate it in the nursery, after a GC if needed */ - if(nursery.size > size) + if(size < nursery.size) { /* If there is insufficient room, collect the nursery */ if(nursery.here + size > nursery.end) @@ -239,23 +239,21 @@ object *factor_vm::allot_object(header header, cell size) obj = nursery.allot(size); } - /* If the object is bigger than the nursery, allocate it in - tenured space */ else { - /* If tenured space does not have enough room, collect */ - //XXX - //if(data->tenured->here + size > data->tenured->end) - primitive_full_gc(); - - /* If it still won't fit, grow the heap */ - //XXX - //if(data->tenured->here + size > data->tenured->end) + /* If tenured space does not have enough room, collect and compact */ + if(!data->tenured->can_allot_p(size)) { - gc(collect_growing_heap_op, - size, /* requested size */ - true, /* trace contexts? */ - false /* compact code heap? */); + primitive_compact_gc(); + + /* If it still won't fit, grow the heap */ + if(!data->tenured->can_allot_p(size)) + { + gc(collect_growing_heap_op, + size, /* requested size */ + true, /* trace contexts? */ + false /* compact code heap? */); + } } obj = data->tenured->allot(size); diff --git a/vm/image.cpp b/vm/image.cpp index bee351b830..2e4433d3b5 100755 --- a/vm/image.cpp +++ b/vm/image.cpp @@ -16,10 +16,7 @@ void factor_vm::init_objects(image_header *h) void factor_vm::load_data_heap(FILE *file, image_header *h, vm_parameters *p) { - cell good_size = h->data_size + (1 << 20); - - if(good_size > p->tenured_size) - p->tenured_size = good_size; + p->tenured_size = std::max((h->data_size * 3) / 2,p->tenured_size); init_data_heap(p->young_size, p->aging_size, From c30df42e4846feae02c4220f73e4206199e902bd Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Sun, 25 Oct 2009 14:02:14 -0500 Subject: [PATCH 32/74] vm: make compaction its own gc_op --- vm/compaction.cpp | 2 +- vm/full_collector.cpp | 32 +++++--------------------------- vm/gc.cpp | 31 ++++++++++++++++--------------- vm/gc.hpp | 1 + vm/image.cpp | 5 ++--- vm/vm.hpp | 11 +++++------ 6 files changed, 30 insertions(+), 52 deletions(-) diff --git a/vm/compaction.cpp b/vm/compaction.cpp index 74dd8935da..399971805b 100644 --- a/vm/compaction.cpp +++ b/vm/compaction.cpp @@ -110,7 +110,7 @@ struct code_block_compaction_updater { } }; -void factor_vm::collect_full_compact(bool trace_contexts_p) +void factor_vm::collect_compact_impl(bool trace_contexts_p) { tenured_space *tenured = data->tenured; mark_bits *data_forwarding_map = &tenured->state; diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index 65d0edbd47..09e32574fd 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -40,7 +40,7 @@ struct object_start_map_updater { } }; -void factor_vm::collect_full_mark(bool trace_contexts_p) +void factor_vm::collect_mark_impl(bool trace_contexts_p) { full_collector collector(this); @@ -74,43 +74,21 @@ void factor_vm::collect_full_mark(bool trace_contexts_p) code->clear_remembered_set(); } -void factor_vm::collect_full_sweep() +void factor_vm::collect_sweep_impl() { data->tenured->starts.clear_object_start_offsets(); object_start_map_updater updater(&data->tenured->starts); data->tenured->sweep(updater); } -void factor_vm::collect_growing_heap(cell requested_bytes, - bool trace_contexts_p, - bool compact_p) +void factor_vm::collect_growing_heap(cell requested_bytes, bool trace_contexts_p) { /* Grow the data heap and copy all live objects to the new heap. */ data_heap *old = data; set_data_heap(data->grow(requested_bytes)); - collect_full_mark(trace_contexts_p); + collect_mark_impl(trace_contexts_p); + collect_compact_impl(trace_contexts_p); delete old; - - if(compact_p) - collect_full_compact(trace_contexts_p); - else - { - collect_full_sweep(); - relocate_code_heap(); - } -} - -void factor_vm::collect_full(bool trace_contexts_p, bool compact_p) -{ - collect_full_mark(trace_contexts_p); - - if(compact_p) - collect_full_compact(trace_contexts_p); - else - { - collect_full_sweep(); - update_code_heap_words_and_literals(); - } } } diff --git a/vm/gc.cpp b/vm/gc.cpp index 4b530172c1..2c361bcd19 100755 --- a/vm/gc.cpp +++ b/vm/gc.cpp @@ -25,10 +25,7 @@ void factor_vm::record_gc_stats(generation_statistics *stats) stats->max_gc_time = gc_elapsed; } -void factor_vm::gc(gc_op op, - cell requested_bytes, - bool trace_contexts_p, - bool compact_p) +void factor_vm::gc(gc_op op, cell requested_bytes, bool trace_contexts_p) { assert(!gc_off); assert(!current_gc); @@ -57,6 +54,7 @@ void factor_vm::gc(gc_op op, current_gc->op = collect_full_op; break; case collect_full_op: + case collect_compact_op: current_gc->op = collect_growing_heap_op; break; default: @@ -83,11 +81,18 @@ void factor_vm::gc(gc_op op, record_gc_stats(&gc_stats.aging_stats); break; case collect_full_op: - collect_full(trace_contexts_p,compact_p); + collect_mark_impl(trace_contexts_p); + collect_sweep_impl(); + update_code_heap_words_and_literals(); + record_gc_stats(&gc_stats.full_stats); + break; + case collect_compact_op: + collect_mark_impl(trace_contexts_p); + collect_compact_impl(trace_contexts_p); record_gc_stats(&gc_stats.full_stats); break; case collect_growing_heap_op: - collect_growing_heap(requested_bytes,trace_contexts_p,compact_p); + collect_growing_heap(requested_bytes,trace_contexts_p); record_gc_stats(&gc_stats.full_stats); break; default: @@ -106,24 +111,21 @@ void factor_vm::primitive_minor_gc() { gc(collect_nursery_op, 0, /* requested size */ - true, /* trace contexts? */ - false /* compact code heap? */); + true /* trace contexts? */); } void factor_vm::primitive_full_gc() { gc(collect_full_op, 0, /* requested size */ - true, /* trace contexts? */ - true /* compact code heap? */); + true /* trace contexts? */); } void factor_vm::primitive_compact_gc() { - gc(collect_full_op, + gc(collect_compact_op, 0, /* requested size */ - true, /* trace contexts? */ - true /* compact code heap? */); + true /* trace contexts? */); } void factor_vm::add_gc_stats(generation_statistics *stats, growable_array *result) @@ -251,8 +253,7 @@ object *factor_vm::allot_object(header header, cell size) { gc(collect_growing_heap_op, size, /* requested size */ - true, /* trace contexts? */ - false /* compact code heap? */); + true /* trace contexts? */); } } diff --git a/vm/gc.hpp b/vm/gc.hpp index 18b926ed8c..a4162ed620 100755 --- a/vm/gc.hpp +++ b/vm/gc.hpp @@ -6,6 +6,7 @@ enum gc_op { collect_aging_op, collect_to_tenured_op, collect_full_op, + collect_compact_op, collect_growing_heap_op }; diff --git a/vm/image.cpp b/vm/image.cpp index 2e4433d3b5..c35c0a32b8 100755 --- a/vm/image.cpp +++ b/vm/image.cpp @@ -327,10 +327,9 @@ void factor_vm::primitive_save_image_and_exit() for(cell i = 0; i < special_object_count; i++) if(!save_env_p(i)) special_objects[i] = false_object; - gc(collect_full_op, + gc(collect_compact_op, 0, /* requested size */ - false, /* discard objects only reachable from stacks */ - true /* compact the code heap */); + false /* discard objects only reachable from stacks */); /* Save the image */ if(save_image((vm_char *)(path.untagged() + 1))) diff --git a/vm/vm.hpp b/vm/vm.hpp index f0f37619d2..0124affefa 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -247,13 +247,12 @@ struct factor_vm void collect_nursery(); void collect_aging(); void collect_to_tenured(); - void collect_full_mark(bool trace_contexts_p); - void collect_full_sweep(); - void collect_full_compact(bool trace_contexts_p); - void collect_growing_heap(cell requested_bytes, bool trace_contexts_p, bool compact_p); - void collect_full(bool trace_contexts_p, bool compact_p); + void collect_mark_impl(bool trace_contexts_p); + void collect_sweep_impl(); + void collect_compact_impl(bool trace_contexts_p); + void collect_growing_heap(cell requested_bytes, bool trace_contexts_p); void record_gc_stats(generation_statistics *stats); - void gc(gc_op op, cell requested_bytes, bool trace_contexts_p, bool compact_p); + void gc(gc_op op, cell requested_bytes, bool trace_contexts_p); void primitive_minor_gc(); void primitive_full_gc(); void primitive_compact_gc(); From e793a72060c63c9b774d36e0ded080ab5c0f30ce Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Mon, 26 Oct 2009 22:08:35 -0500 Subject: [PATCH 33/74] vm: remove crummy old GC stats, split off free list code, clean up various other things --- Makefile | 1 + basis/tools/memory/memory.factor | 29 +++- core/bootstrap/primitives.factor | 2 - vm/aging_collector.cpp | 1 - vm/bump_allocator.hpp | 14 +- vm/code_block.cpp | 9 +- vm/code_heap.cpp | 12 +- vm/collector.hpp | 32 ++--- vm/copying_collector.hpp | 4 +- vm/data_heap.cpp | 29 ++-- vm/factor.cpp | 6 +- vm/free_list.cpp | 138 +++++++++++++++++++ vm/free_list.hpp | 42 ++++++ vm/free_list_allocator.hpp | 222 +++---------------------------- vm/full_collector.cpp | 1 - vm/gc.cpp | 64 +-------- vm/gc.hpp | 19 --- vm/image.cpp | 6 +- vm/image.hpp | 2 +- vm/master.hpp | 1 + vm/nursery_collector.cpp | 1 - vm/primitives.cpp | 4 - vm/to_tenured_collector.cpp | 1 - vm/vm.hpp | 10 +- 24 files changed, 286 insertions(+), 364 deletions(-) create mode 100644 vm/free_list.cpp create mode 100644 vm/free_list.hpp diff --git a/Makefile b/Makefile index 030a278543..2ea43706f4 100755 --- a/Makefile +++ b/Makefile @@ -48,6 +48,7 @@ DLL_OBJS = $(PLAF_DLL_OBJS) \ vm/dispatch.o \ vm/errors.o \ vm/factor.o \ + vm/free_list.o \ vm/full_collector.o \ vm/gc.o \ vm/image.o \ diff --git a/basis/tools/memory/memory.factor b/basis/tools/memory/memory.factor index 2f1827a8ff..a5ed0fc0c1 100644 --- a/basis/tools/memory/memory.factor +++ b/basis/tools/memory/memory.factor @@ -9,35 +9,50 @@ IN: tools.memory string + 1024 /i number>string dup length 4 > [ 3 cut* "," glue ] when " KB" append ; -: memory-table. ( sizes seq -- ) - swap [ kilobytes ] map zip simple-table. ; +: fancy-table. ( seq alist -- ) + [ [ nip first ] [ second call( obj -- str ) ] 2bi 2array ] 2map + simple-table. ; : young-room. ( seq -- ) - { "Total:" "Allocated:" "Free:" } memory-table. ; + { + { "Total:" [ kilobytes ] } + { "Allocated:" [ kilobytes ] } + { "Free:" [ kilobytes ] } + } fancy-table. ; : nursery-room. ( seq -- ) "- Nursery space" print young-room. ; : aging-room. ( seq -- ) "- Aging space" print young-room. ; : mark-sweep-table. ( sizes -- ) - { "Total:" "Allocated:" "Contiguous free:" "Total free:" } memory-table. ; + { + { "Total:" [ kilobytes ] } + { "Allocated:" [ kilobytes ] } + { "Total free:" [ kilobytes ] } + { "Contiguous free:" [ kilobytes ] } + { "Free list entries:" [ number>string ] } + } fancy-table. ; : tenured-room. ( seq -- ) "- Tenured space" print mark-sweep-table. ; : misc-room. ( seq -- ) "- Miscellaneous buffers" print - { "Card array:" "Deck array:" "Mark stack:" } memory-table. ; + { + { "Card array:" [ kilobytes ] } + { "Deck array:" [ kilobytes ] } + { "Mark stack:" [ kilobytes ] } + } fancy-table. ; : data-room. ( -- ) "==== DATA HEAP" print nl data-room 3 cut [ nursery-room. nl ] dip 3 cut [ aging-room. nl ] dip - 4 cut [ tenured-room. nl ] dip + 5 cut [ tenured-room. nl ] dip misc-room. ; : code-room. ( -- ) diff --git a/core/bootstrap/primitives.factor b/core/bootstrap/primitives.factor index ef66cc3cd6..30a41eb93d 100644 --- a/core/bootstrap/primitives.factor +++ b/core/bootstrap/primitives.factor @@ -423,7 +423,6 @@ tuple { "minor-gc" "memory" (( -- )) } { "gc" "memory" (( -- )) } { "compact-gc" "memory" (( -- )) } - { "gc-stats" "memory" f } { "(save-image)" "memory.private" (( path -- )) } { "(save-image-and-exit)" "memory.private" (( path -- )) } { "datastack" "kernel" (( -- ds )) } @@ -509,7 +508,6 @@ tuple { "resize-byte-array" "byte-arrays" (( n byte-array -- newbyte-array )) } { "dll-valid?" "alien.libraries" (( dll -- ? )) } { "unimplemented" "kernel.private" (( -- * )) } - { "gc-reset" "memory" (( -- )) } { "jit-compile" "quotations" (( quot -- )) } { "load-locals" "locals.backend" (( ... n -- )) } { "check-datastack" "kernel.private" (( array in# out# -- ? )) } diff --git a/vm/aging_collector.cpp b/vm/aging_collector.cpp index d33823b624..7b74a00f99 100644 --- a/vm/aging_collector.cpp +++ b/vm/aging_collector.cpp @@ -6,7 +6,6 @@ namespace factor aging_collector::aging_collector(factor_vm *parent_) : copying_collector( parent_, - &parent_->gc_stats.aging_stats, parent_->data->aging, aging_policy(parent_)) {} diff --git a/vm/bump_allocator.hpp b/vm/bump_allocator.hpp index 8f4fabe9a7..5488c65323 100644 --- a/vm/bump_allocator.hpp +++ b/vm/bump_allocator.hpp @@ -11,17 +11,27 @@ template struct bump_allocator { explicit bump_allocator(cell size_, cell start_) : here(start_), start(start_), end(start_ + size_), size(size_) {} - inline bool contains_p(Block *block) + bool contains_p(Block *block) { return ((cell)block - start) < size; } - inline Block *allot(cell size) + Block *allot(cell size) { cell h = here; here = h + align(size,data_alignment); return (Block *)h; } + + cell occupied_space() + { + return here - start; + } + + cell free_space() + { + return end - here; + } }; } diff --git a/vm/code_block.cpp b/vm/code_block.cpp index 831c2388bb..d3670af3c0 100755 --- a/vm/code_block.cpp +++ b/vm/code_block.cpp @@ -454,13 +454,8 @@ code_block *factor_vm::allot_code_block(cell size, code_block_type type) /* Insufficient room even after code GC, give up */ if(block == NULL) { - cell used, total_free, max_free; - code->allocator->usage(&used,&total_free,&max_free); - - std::cout << "Code heap stats:\n"; - std::cout << "Used: " << used << "\n"; - std::cout << "Total free space: " << total_free << "\n"; - std::cout << "Largest free block: " << max_free << "\n"; + std::cout << "Code heap used: " << code->allocator->occupied_space() << "\n"; + std::cout << "Code heap free: " << code->allocator->free_space() << "\n"; fatal_error("Out of memory in add-compiled-block",0); } } diff --git a/vm/code_heap.cpp b/vm/code_heap.cpp index ae53869ef2..d38ec9e3f5 100755 --- a/vm/code_heap.cpp +++ b/vm/code_heap.cpp @@ -199,13 +199,11 @@ void factor_vm::primitive_code_room() { growable_array a(this); - cell used, total_free, max_free; - code->allocator->usage(&used,&total_free,&max_free); - - a.add(tag_fixnum(code->seg->size >> 10)); - a.add(tag_fixnum(used >> 10)); - a.add(tag_fixnum(total_free >> 10)); - a.add(tag_fixnum(max_free >> 10)); + a.add(tag_fixnum(code->allocator->size)); + a.add(tag_fixnum(code->allocator->occupied_space())); + a.add(tag_fixnum(code->allocator->free_space())); + a.add(tag_fixnum(code->allocator->free_blocks.largest_free_block())); + a.add(tag_fixnum(code->allocator->free_blocks.free_block_count)); a.trim(); dpush(a.elements.value()); diff --git a/vm/collector.hpp b/vm/collector.hpp index 54683556b1..d3cfa24642 100644 --- a/vm/collector.hpp +++ b/vm/collector.hpp @@ -3,13 +3,11 @@ namespace factor template struct collector_workhorse { factor_vm *parent; - generation_statistics *stats; TargetGeneration *target; Policy policy; - explicit collector_workhorse(factor_vm *parent_, generation_statistics *stats_, TargetGeneration *target_, Policy policy_) : + explicit collector_workhorse(factor_vm *parent_, TargetGeneration *target_, Policy policy_) : parent(parent_), - stats(stats_), target(target_), policy(policy_) {} @@ -36,9 +34,6 @@ template struct collector_workhorse memcpy(newpointer,untagged,size); untagged->h.forward_to(newpointer); - stats->object_count++; - stats->bytes_copied += size; - policy.promoted_object(newpointer); return newpointer; @@ -69,29 +64,34 @@ template struct collector_workhorse template inline static slot_visitor > make_collector_workhorse( factor_vm *parent, - generation_statistics *stats, TargetGeneration *target, Policy policy) { return slot_visitor >(parent, - collector_workhorse(parent,stats,target,policy)); + collector_workhorse(parent,target,policy)); } template struct collector { factor_vm *parent; data_heap *data; code_heap *code; - generation_statistics *stats; TargetGeneration *target; slot_visitor > workhorse; + cell cards_scanned; + cell decks_scanned; + cell card_scan_time; + cell code_blocks_scanned; - explicit collector(factor_vm *parent_, generation_statistics *stats_, TargetGeneration *target_, Policy policy_) : + explicit collector(factor_vm *parent_, TargetGeneration *target_, Policy policy_) : parent(parent_), data(parent_->data), code(parent_->code), - stats(stats_), target(target_), - workhorse(make_collector_workhorse(parent_,stats_,target_,policy_)) {} + workhorse(make_collector_workhorse(parent_,target_,policy_)), + cards_scanned(0), + decks_scanned(0), + card_scan_time(0), + code_blocks_scanned(0) {} void trace_handle(cell *handle) { @@ -127,7 +127,7 @@ template struct collector { for(; iter != end; iter++) { trace_literal_references(*iter); - parent->gc_stats.code_blocks_scanned++; + code_blocks_scanned++; } } @@ -195,7 +195,7 @@ template struct collector { { if(decks[deck_index] & mask) { - parent->gc_stats.decks_scanned++; + decks_scanned++; cell first_card = first_card_in_deck(deck_index); cell last_card = last_card_in_deck(deck_index); @@ -204,7 +204,7 @@ template struct collector { { if(cards[card_index] & mask) { - parent->gc_stats.cards_scanned++; + cards_scanned++; if(end < card_start_address(card_index)) { @@ -246,7 +246,7 @@ scan_next_object: { } } -end: parent->gc_stats.card_scan_time += (current_micros() - start_time); +end: card_scan_time += (current_micros() - start_time); } }; diff --git a/vm/copying_collector.hpp b/vm/copying_collector.hpp index f79f97d34e..2264b600dd 100644 --- a/vm/copying_collector.hpp +++ b/vm/copying_collector.hpp @@ -15,8 +15,8 @@ template struct copying_collector : collector { cell scan; - explicit copying_collector(factor_vm *parent_, generation_statistics *stats_, TargetGeneration *target_, Policy policy_) : - collector(parent_,stats_,target_,policy_), scan(target_->here) {} + explicit copying_collector(factor_vm *parent_, TargetGeneration *target_, Policy policy_) : + collector(parent_,target_,policy_), scan(target_->here) {} void cheneys_algorithm() { diff --git a/vm/data_heap.cpp b/vm/data_heap.cpp index 3dd46fd848..d8b5ca3efb 100755 --- a/vm/data_heap.cpp +++ b/vm/data_heap.cpp @@ -206,24 +206,23 @@ void factor_vm::primitive_data_room() { growable_array a(this); - a.add(tag_fixnum((nursery.size) >> 10)); - a.add(tag_fixnum((nursery.here - nursery.start) >> 10)); - a.add(tag_fixnum((nursery.end - nursery.here) >> 10)); + a.add(tag_fixnum(nursery.size)); + a.add(tag_fixnum(nursery.occupied_space())); + a.add(tag_fixnum(nursery.free_space())); - a.add(tag_fixnum((data->aging->size) >> 10)); - a.add(tag_fixnum((data->aging->here - data->aging->start) >> 10)); - a.add(tag_fixnum((data->aging->end - data->aging->here) >> 10)); + a.add(tag_fixnum(data->aging->size)); + a.add(tag_fixnum(data->aging->occupied_space())); + a.add(tag_fixnum(data->aging->free_space())); - cell used, total_free, max_free; - data->tenured->usage(&used,&total_free,&max_free); - a.add(tag_fixnum(data->tenured->size >> 10)); - a.add(tag_fixnum(used >> 10)); - a.add(tag_fixnum(total_free >> 10)); - a.add(tag_fixnum(max_free >> 10)); + a.add(tag_fixnum(data->tenured->size)); + a.add(tag_fixnum(data->tenured->occupied_space())); + a.add(tag_fixnum(data->tenured->free_space())); + a.add(tag_fixnum(data->tenured->free_blocks.largest_free_block())); + a.add(tag_fixnum(data->tenured->free_blocks.free_block_count)); - a.add(tag_fixnum((data->cards_end - data->cards) >> 10)); - a.add(tag_fixnum((data->decks_end - data->decks) >> 10)); - a.add(tag_fixnum((data->tenured->mark_stack.capacity()) >> 10)); + a.add(tag_fixnum(data->cards_end - data->cards)); + a.add(tag_fixnum(data->decks_end - data->decks)); + a.add(tag_fixnum(data->tenured->mark_stack.capacity())); a.trim(); dpush(a.elements.value()); diff --git a/vm/factor.cpp b/vm/factor.cpp index 24a3e01237..6a235ec88d 100755 --- a/vm/factor.cpp +++ b/vm/factor.cpp @@ -38,7 +38,7 @@ void factor_vm::default_parameters(vm_parameters *p) p->max_pic_size = 3; p->fep = false; - p->verbosegc = false; + p->verbose_gc = false; p->signals = true; #ifdef WINDOWS @@ -87,7 +87,7 @@ void factor_vm::init_parameters_from_args(vm_parameters *p, int argc, vm_char ** else if(factor_arg(arg,STRING_LITERAL("-callbacks=%d"),&p->callback_size)); else if(STRCMP(arg,STRING_LITERAL("-fep")) == 0) p->fep = true; else if(STRCMP(arg,STRING_LITERAL("-nosignals")) == 0) p->signals = false; - else if(STRCMP(arg,STRING_LITERAL("-verbosegc")) == 0) p->verbosegc = true; + else if(STRCMP(arg,STRING_LITERAL("-verbosegc")) == 0) p->verbose_gc = true; else if(STRNCMP(arg,STRING_LITERAL("-i="),3) == 0) p->image_path = arg + 3; else if(STRCMP(arg,STRING_LITERAL("-console")) == 0) p->console = true; } @@ -142,7 +142,7 @@ void factor_vm::init_factor(vm_parameters *p) if(p->signals) init_signals(); - verbosegc = p->verbosegc; + verbose_gc = p->verbose_gc; if(p->console) open_console(); diff --git a/vm/free_list.cpp b/vm/free_list.cpp new file mode 100644 index 0000000000..e104e0b9bd --- /dev/null +++ b/vm/free_list.cpp @@ -0,0 +1,138 @@ +#include "master.hpp" + +namespace factor +{ + +void free_list::clear_free_list() +{ + memset(this,0,sizeof(free_list)); +} + +void free_list::initial_free_list(cell start, cell end, cell occupied) +{ + clear_free_list(); + if(occupied != end - start) + { + free_heap_block *last_block = (free_heap_block *)(start + occupied); + last_block->make_free(end - (cell)last_block); + add_to_free_list(last_block); + } +} + +void free_list::add_to_free_list(free_heap_block *block) +{ + cell size = block->size(); + + free_block_count++; + free_space += size; + + if(size < free_list_count * block_granularity) + { + int index = size / block_granularity; + block->next_free = small_blocks[index]; + small_blocks[index] = block; + } + else + { + block->next_free = large_blocks; + large_blocks = block; + } +} + +free_heap_block *free_list::find_free_block(cell size) +{ + cell attempt = size; + + while(attempt < free_list_count * block_granularity) + { + int index = attempt / block_granularity; + free_heap_block *block = small_blocks[index]; + if(block) + { + small_blocks[index] = block->next_free; + + free_block_count--; + free_space -= block->size(); + + return block; + } + + attempt++; + } + + free_heap_block *prev = NULL; + free_heap_block *block = large_blocks; + + while(block) + { + if(block->size() >= size) + { + if(prev) + prev->next_free = block->next_free; + else + large_blocks = block->next_free; + + free_block_count--; + free_space -= block->size(); + + return block; + } + + prev = block; + block = block->next_free; + } + + return NULL; +} + +free_heap_block *free_list::split_free_block(free_heap_block *block, cell size) +{ + if(block->size() != size) + { + /* split the block in two */ + free_heap_block *split = (free_heap_block *)((cell)block + size); + split->make_free(block->size() - size); + split->next_free = block->next_free; + block->make_free(size); + add_to_free_list(split); + } + + return block; +} + +bool free_list::can_allot_p(cell size) +{ + cell attempt = size; + + while(attempt < free_list_count * block_granularity) + { + int index = attempt / block_granularity; + if(small_blocks[index]) return true; + attempt++; + } + + free_heap_block *block = large_blocks; + while(block) + { + if(block->size() >= size) return true; + block = block->next_free; + } + + return false; +} + +cell free_list::largest_free_block() +{ + cell largest = 0; + free_heap_block *scan = large_blocks; + + while(scan) + { + largest = std::max(largest,scan->size()); + scan = scan->next_free; + } + + return largest; +} + +} diff --git a/vm/free_list.hpp b/vm/free_list.hpp new file mode 100644 index 0000000000..28bc063883 --- /dev/null +++ b/vm/free_list.hpp @@ -0,0 +1,42 @@ +namespace factor +{ + +static const cell free_list_count = 32; + +struct free_heap_block +{ + cell header; + free_heap_block *next_free; + + bool free_p() const + { + return header & 1 == 1; + } + + cell size() const + { + return header >> 3; + } + + void make_free(cell size) + { + header = (size << 3) | 1; + } +}; + +struct free_list { + free_heap_block *small_blocks[free_list_count]; + free_heap_block *large_blocks; + cell free_block_count; + cell free_space; + + void clear_free_list(); + void initial_free_list(cell start, cell end, cell occupied); + void add_to_free_list(free_heap_block *block); + free_heap_block *find_free_block(cell size); + free_heap_block *split_free_block(free_heap_block *block, cell size); + bool can_allot_p(cell size); + cell largest_free_block(); +}; + +} diff --git a/vm/free_list_allocator.hpp b/vm/free_list_allocator.hpp index d796379799..d74dd5be9a 100644 --- a/vm/free_list_allocator.hpp +++ b/vm/free_list_allocator.hpp @@ -1,34 +1,6 @@ namespace factor { -static const cell free_list_count = 32; - -struct free_heap_block -{ - cell header; - free_heap_block *next_free; - - bool free_p() const - { - return header & 1 == 1; - } - - cell size() const - { - return header >> 3; - } - - void make_free(cell size) - { - header = (size << 3) | 1; - } -}; - -struct free_list { - free_heap_block *small_blocks[free_list_count]; - free_heap_block *large_blocks; -}; - template struct free_list_allocator { cell size; cell start; @@ -37,21 +9,16 @@ template struct free_list_allocator { mark_bits state; explicit free_list_allocator(cell size, cell start); + void initial_free_list(cell occupied); bool contains_p(Block *block); Block *first_block(); Block *last_block(); Block *next_block_after(Block *block); - void clear_free_list(); - void add_to_free_list(free_heap_block *block); - void initial_free_list(cell size); - void assert_free_block(free_heap_block *block); - free_heap_block *find_free_block(cell size); - free_heap_block *split_free_block(free_heap_block *block, cell size); bool can_allot_p(cell size); Block *allot(cell size); void free(Block *block); - void usage(cell *used, cell *total_free, cell *max_free); - cell occupied(); + cell occupied_space(); + cell free_space(); void sweep(); template void sweep(Iterator &iter); template void compact(Iterator &iter, Sizer &sizer); @@ -66,9 +33,9 @@ free_list_allocator::free_list_allocator(cell size_, cell start_) : initial_free_list(0); } -template void free_list_allocator::clear_free_list() +template void free_list_allocator::initial_free_list(cell occupied) { - memset(&free_blocks,0,sizeof(free_list)); + free_blocks.initial_free_list(start,end,occupied); } template bool free_list_allocator::contains_p(Block *block) @@ -91,125 +58,19 @@ template Block *free_list_allocator::next_block_after(Blo return (Block *)((cell)block + block->size()); } -template void free_list_allocator::add_to_free_list(free_heap_block *block) -{ - if(block->size() < free_list_count * block_granularity) - { - int index = block->size() / block_granularity; - block->next_free = free_blocks.small_blocks[index]; - free_blocks.small_blocks[index] = block; - } - else - { - block->next_free = free_blocks.large_blocks; - free_blocks.large_blocks = block; - } -} - -/* Called after reading the heap from the image file, and after heap compaction. -Makes a free list consisting of one free block, at the very end. */ -template void free_list_allocator::initial_free_list(cell size) -{ - clear_free_list(); - if(size != this->size) - { - free_heap_block *last_block = (free_heap_block *)(start + size); - last_block->make_free(end - (cell)last_block); - add_to_free_list(last_block); - } -} - -template void free_list_allocator::assert_free_block(free_heap_block *block) -{ -#ifdef FACTOR_DEBUG - assert(block->free_p()); -#endif -} - -template free_heap_block *free_list_allocator::find_free_block(cell size) -{ - cell attempt = size; - - while(attempt < free_list_count * block_granularity) - { - int index = attempt / block_granularity; - free_heap_block *block = free_blocks.small_blocks[index]; - if(block) - { - assert_free_block(block); - free_blocks.small_blocks[index] = block->next_free; - return block; - } - - attempt *= 2; - } - - free_heap_block *prev = NULL; - free_heap_block *block = free_blocks.large_blocks; - - while(block) - { - assert_free_block(block); - if(block->size() >= size) - { - if(prev) - prev->next_free = block->next_free; - else - free_blocks.large_blocks = block->next_free; - return block; - } - - prev = block; - block = block->next_free; - } - - return NULL; -} - -template free_heap_block *free_list_allocator::split_free_block(free_heap_block *block, cell size) -{ - if(block->size() != size) - { - /* split the block in two */ - free_heap_block *split = (free_heap_block *)((cell)block + size); - split->make_free(block->size() - size); - split->next_free = block->next_free; - block->make_free(size); - add_to_free_list(split); - } - - return block; -} - template bool free_list_allocator::can_allot_p(cell size) { - cell attempt = size; - - while(attempt < free_list_count * block_granularity) - { - int index = attempt / block_granularity; - if(free_blocks.small_blocks[index]) return true; - attempt *= 2; - } - - free_heap_block *block = free_blocks.large_blocks; - while(block) - { - if(block->size() >= size) return true; - block = block->next_free; - } - - return false; + return free_blocks.can_allot_p(size); } template Block *free_list_allocator::allot(cell size) { size = align(size,block_granularity); - free_heap_block *block = find_free_block(size); + free_heap_block *block = free_blocks.find_free_block(size); if(block) { - block = split_free_block(block,size); + block = free_blocks.split_free_block(block,size); return (Block *)block; } else @@ -220,64 +81,23 @@ template void free_list_allocator::free(Block *block) { free_heap_block *free_block = (free_heap_block *)block; free_block->make_free(block->size()); - add_to_free_list(free_block); + free_blocks.add_to_free_list(free_block); } -/* Compute total sum of sizes of free blocks, and size of largest free block */ -template void free_list_allocator::usage(cell *used, cell *total_free, cell *max_free) +template cell free_list_allocator::free_space() { - *used = 0; - *total_free = 0; - *max_free = 0; - - Block *scan = first_block(); - Block *end = last_block(); - - while(scan != end) - { - cell size = scan->size(); - - if(scan->free_p()) - { - *total_free += size; - if(size > *max_free) - *max_free = size; - } - else - *used += size; - - scan = next_block_after(scan); - } + return free_blocks.free_space; } -/* The size of the heap after compaction */ -template cell free_list_allocator::occupied() +template cell free_list_allocator::occupied_space() { - Block *scan = first_block(); - Block *last = last_block(); - - while(scan != last) - { - if(scan->free_p()) break; - else scan = next_block_after(scan); - } - - if(scan != last) - { - free_heap_block *free_block = (free_heap_block *)scan; - assert(free_block->free_p()); - assert((cell)scan + free_block->size() == end); - - return (cell)scan - (cell)first_block(); - } - else - return size; + return size - free_blocks.free_space; } template void free_list_allocator::sweep() { - this->clear_free_list(); + free_blocks.clear_free_list(); Block *prev = NULL; Block *scan = this->first_block(); @@ -300,7 +120,7 @@ void free_list_allocator::sweep() else if(this->state.marked_p(scan)) { if(prev && prev->free_p()) - this->add_to_free_list((free_heap_block *)prev); + free_blocks.add_to_free_list((free_heap_block *)prev); prev = scan; } else @@ -322,14 +142,14 @@ void free_list_allocator::sweep() } if(prev && prev->free_p()) - this->add_to_free_list((free_heap_block *)prev); + free_blocks.add_to_free_list((free_heap_block *)prev); } template template void free_list_allocator::sweep(Iterator &iter) { - this->clear_free_list(); + free_blocks.clear_free_list(); Block *prev = NULL; Block *scan = this->first_block(); @@ -352,7 +172,7 @@ void free_list_allocator::sweep(Iterator &iter) else if(this->state.marked_p(scan)) { if(prev && prev->free_p()) - this->add_to_free_list((free_heap_block *)prev); + free_blocks.add_to_free_list((free_heap_block *)prev); prev = scan; iter(scan,size); } @@ -375,7 +195,7 @@ void free_list_allocator::sweep(Iterator &iter) } if(prev && prev->free_p()) - this->add_to_free_list((free_heap_block *)prev); + free_blocks.add_to_free_list((free_heap_block *)prev); } template struct heap_compactor { @@ -403,11 +223,11 @@ template void free_list_allocator::compact(Iterator &iter, Sizer &sizer) { heap_compactor compactor(&state,first_block(),iter); - this->iterate(compactor,sizer); + iterate(compactor,sizer); /* Now update the free list; there will be a single free block at the end */ - this->initial_free_list((cell)compactor.address - this->start); + free_blocks.initial_free_list(start,end,(cell)compactor.address - start); } /* During compaction we have to be careful and measure object sizes differently */ diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index 09e32574fd..0c90719b51 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -6,7 +6,6 @@ namespace factor full_collector::full_collector(factor_vm *parent_) : collector( parent_, - &parent_->gc_stats.full_stats, parent_->data->tenured, full_policy(parent_)) {} diff --git a/vm/gc.cpp b/vm/gc.cpp index 2c361bcd19..dd881c257b 100755 --- a/vm/gc.cpp +++ b/vm/gc.cpp @@ -16,15 +16,6 @@ void factor_vm::update_code_heap_for_minor_gc(std::set *remembered for(; iter != end; iter++) update_literal_references(*iter); } -void factor_vm::record_gc_stats(generation_statistics *stats) -{ - cell gc_elapsed = (current_micros() - current_gc->start_time); - stats->collections++; - stats->gc_time += gc_elapsed; - if(stats->max_gc_time < gc_elapsed) - stats->max_gc_time = gc_elapsed; -} - void factor_vm::gc(gc_op op, cell requested_bytes, bool trace_contexts_p) { assert(!gc_off); @@ -34,7 +25,7 @@ void factor_vm::gc(gc_op op, cell requested_bytes, bool trace_contexts_p) current_gc = new gc_state(op); - if(verbosegc) + if(verbose_gc) std::cout << "GC requested, op=" << op << std::endl; /* Keep trying to GC higher and higher generations until we don't run out @@ -62,7 +53,7 @@ void factor_vm::gc(gc_op op, cell requested_bytes, bool trace_contexts_p) break; } - if(verbosegc) + if(verbose_gc) std::cout << "GC rewind, op=" << current_gc->op << std::endl; } @@ -70,37 +61,31 @@ void factor_vm::gc(gc_op op, cell requested_bytes, bool trace_contexts_p) { case collect_nursery_op: collect_nursery(); - record_gc_stats(&gc_stats.nursery_stats); break; case collect_aging_op: collect_aging(); - record_gc_stats(&gc_stats.aging_stats); break; case collect_to_tenured_op: collect_to_tenured(); - record_gc_stats(&gc_stats.aging_stats); break; case collect_full_op: collect_mark_impl(trace_contexts_p); collect_sweep_impl(); update_code_heap_words_and_literals(); - record_gc_stats(&gc_stats.full_stats); break; case collect_compact_op: collect_mark_impl(trace_contexts_p); collect_compact_impl(trace_contexts_p); - record_gc_stats(&gc_stats.full_stats); break; case collect_growing_heap_op: collect_growing_heap(requested_bytes,trace_contexts_p); - record_gc_stats(&gc_stats.full_stats); break; default: critical_error("Bad GC op\n",current_gc->op); break; } - if(verbosegc) + if(verbose_gc) std::cout << "GC done, op=" << current_gc->op << std::endl; delete current_gc; @@ -128,49 +113,6 @@ void factor_vm::primitive_compact_gc() true /* trace contexts? */); } -void factor_vm::add_gc_stats(generation_statistics *stats, growable_array *result) -{ - result->add(allot_cell(stats->collections)); - result->add(tag(long_long_to_bignum(stats->gc_time))); - result->add(tag(long_long_to_bignum(stats->max_gc_time))); - result->add(allot_cell(stats->collections == 0 ? 0 : stats->gc_time / stats->collections)); - result->add(allot_cell(stats->object_count)); - result->add(tag(long_long_to_bignum(stats->bytes_copied))); -} - -void factor_vm::primitive_gc_stats() -{ - growable_array result(this); - - add_gc_stats(&gc_stats.nursery_stats,&result); - add_gc_stats(&gc_stats.aging_stats,&result); - add_gc_stats(&gc_stats.full_stats,&result); - - u64 total_gc_time = - gc_stats.nursery_stats.gc_time + - gc_stats.aging_stats.gc_time + - gc_stats.full_stats.gc_time; - - result.add(tag(ulong_long_to_bignum(total_gc_time))); - result.add(tag(ulong_long_to_bignum(gc_stats.cards_scanned))); - result.add(tag(ulong_long_to_bignum(gc_stats.decks_scanned))); - result.add(tag(ulong_long_to_bignum(gc_stats.card_scan_time))); - result.add(allot_cell(gc_stats.code_blocks_scanned)); - - result.trim(); - dpush(result.elements.value()); -} - -void factor_vm::clear_gc_stats() -{ - memset(&gc_stats,0,sizeof(gc_statistics)); -} - -void factor_vm::primitive_clear_gc_stats() -{ - clear_gc_stats(); -} - /* classes.tuple uses this to reshape tuples; tools.deploy.shaker uses this to coalesce equal but distinct quotations and wrappers. */ void factor_vm::primitive_become() diff --git a/vm/gc.hpp b/vm/gc.hpp index a4162ed620..dff9923f6e 100755 --- a/vm/gc.hpp +++ b/vm/gc.hpp @@ -10,25 +10,6 @@ enum gc_op { collect_growing_heap_op }; -/* statistics */ -struct generation_statistics { - cell collections; - u64 gc_time; - u64 max_gc_time; - cell object_count; - u64 bytes_copied; -}; - -struct gc_statistics { - generation_statistics nursery_stats; - generation_statistics aging_stats; - generation_statistics full_stats; - u64 cards_scanned; - u64 decks_scanned; - u64 card_scan_time; - u64 code_blocks_scanned; -}; - struct gc_state { gc_op op; u64 start_time; diff --git a/vm/image.cpp b/vm/image.cpp index c35c0a32b8..fce730df5a 100755 --- a/vm/image.cpp +++ b/vm/image.cpp @@ -22,8 +22,6 @@ void factor_vm::load_data_heap(FILE *file, image_header *h, vm_parameters *p) p->aging_size, p->tenured_size); - clear_gc_stats(); - fixnum bytes_read = fread((void*)data->tenured->start,1,h->data_size,file); if((cell)bytes_read != h->data_size) @@ -280,9 +278,9 @@ bool factor_vm::save_image(const vm_char *filename) h.magic = image_magic; h.version = image_version; h.data_relocation_base = data->tenured->start; - h.data_size = data->tenured->occupied(); + h.data_size = data->tenured->occupied_space(); h.code_relocation_base = code->seg->start; - h.code_size = code->allocator->occupied(); + h.code_size = code->allocator->occupied_space(); h.true_object = true_object; h.bignum_zero = bignum_zero; diff --git a/vm/image.hpp b/vm/image.hpp index 3a5447c63b..f740b65f68 100755 --- a/vm/image.hpp +++ b/vm/image.hpp @@ -35,7 +35,7 @@ struct vm_parameters { cell young_size, aging_size, tenured_size; cell code_size; bool fep; - bool verbosegc; + bool verbose_gc; bool console; bool signals; cell max_pic_size; diff --git a/vm/master.hpp b/vm/master.hpp index 1947c0ad50..0873480add 100755 --- a/vm/master.hpp +++ b/vm/master.hpp @@ -51,6 +51,7 @@ namespace factor #include "code_block.hpp" #include "bump_allocator.hpp" #include "mark_bits.hpp" +#include "free_list.hpp" #include "free_list_allocator.hpp" #include "write_barrier.hpp" #include "object_start_map.hpp" diff --git a/vm/nursery_collector.cpp b/vm/nursery_collector.cpp index 07f9666f37..096bf7f757 100644 --- a/vm/nursery_collector.cpp +++ b/vm/nursery_collector.cpp @@ -6,7 +6,6 @@ namespace factor nursery_collector::nursery_collector(factor_vm *parent_) : copying_collector( parent_, - &parent_->gc_stats.nursery_stats, parent_->data->aging, nursery_policy(parent_)) {} diff --git a/vm/primitives.cpp b/vm/primitives.cpp index ea02545148..5ba57ec8af 100644 --- a/vm/primitives.cpp +++ b/vm/primitives.cpp @@ -55,7 +55,6 @@ PRIMITIVE_FORWARD(existsp) PRIMITIVE_FORWARD(minor_gc) PRIMITIVE_FORWARD(full_gc) PRIMITIVE_FORWARD(compact_gc) -PRIMITIVE_FORWARD(gc_stats) PRIMITIVE_FORWARD(save_image) PRIMITIVE_FORWARD(save_image_and_exit) PRIMITIVE_FORWARD(datastack) @@ -115,7 +114,6 @@ PRIMITIVE_FORWARD(call_clear) PRIMITIVE_FORWARD(resize_byte_array) PRIMITIVE_FORWARD(dll_validp) PRIMITIVE_FORWARD(unimplemented) -PRIMITIVE_FORWARD(clear_gc_stats) PRIMITIVE_FORWARD(jit_compile) PRIMITIVE_FORWARD(load_locals) PRIMITIVE_FORWARD(check_datastack) @@ -193,7 +191,6 @@ const primitive_type primitives[] = { primitive_minor_gc, primitive_full_gc, primitive_compact_gc, - primitive_gc_stats, primitive_save_image, primitive_save_image_and_exit, primitive_datastack, @@ -279,7 +276,6 @@ const primitive_type primitives[] = { primitive_resize_byte_array, primitive_dll_validp, primitive_unimplemented, - primitive_clear_gc_stats, primitive_jit_compile, primitive_load_locals, primitive_check_datastack, diff --git a/vm/to_tenured_collector.cpp b/vm/to_tenured_collector.cpp index ea7cb8ed72..97009a5ceb 100644 --- a/vm/to_tenured_collector.cpp +++ b/vm/to_tenured_collector.cpp @@ -6,7 +6,6 @@ namespace factor to_tenured_collector::to_tenured_collector(factor_vm *myvm_) : collector( myvm_, - &myvm_->gc_stats.aging_stats, myvm_->data->tenured, to_tenured_policy(myvm_)) {} diff --git a/vm/vm.hpp b/vm/vm.hpp index 0124affefa..36f0dce049 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -47,7 +47,7 @@ struct factor_vm bool gc_off; /* GC logging */ - bool verbosegc; + bool verbose_gc; /* Data heap */ data_heap *data; @@ -61,9 +61,6 @@ struct factor_vm /* Only set if we're performing a GC */ gc_state *current_gc; - /* Statistics */ - gc_statistics gc_stats; - /* If a runtime function needs to call another function which potentially allocates memory, it must wrap any local variable references to Factor objects in gc_root instances */ @@ -251,18 +248,13 @@ struct factor_vm void collect_sweep_impl(); void collect_compact_impl(bool trace_contexts_p); void collect_growing_heap(cell requested_bytes, bool trace_contexts_p); - void record_gc_stats(generation_statistics *stats); void gc(gc_op op, cell requested_bytes, bool trace_contexts_p); void primitive_minor_gc(); void primitive_full_gc(); void primitive_compact_gc(); - void primitive_gc_stats(); - void clear_gc_stats(); void primitive_become(); void inline_gc(cell *gc_roots_base, cell gc_roots_size); object *allot_object(header header, cell size); - void add_gc_stats(generation_statistics *stats, growable_array *result); - void primitive_clear_gc_stats(); template Type *allot(cell size) { From a60bf14673529511f2dbd827d1cd1c4ef036e830 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Mon, 26 Oct 2009 22:12:44 -0500 Subject: [PATCH 34/74] vm: add gc_event --- vm/gc.cpp | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ vm/gc.hpp | 39 +++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) diff --git a/vm/gc.cpp b/vm/gc.cpp index dd881c257b..be505b4ef9 100755 --- a/vm/gc.cpp +++ b/vm/gc.cpp @@ -7,6 +7,87 @@ gc_state::gc_state(gc_op op_) : op(op_), start_time(current_micros()) {} gc_state::~gc_state() {} +gc_event::gc_event(gc_op op_, factor_vm *parent) : + op(op_), + start_time(current_micros()), + card_scan_time(0), + data_sweep_time(0), + code_sweep_time(0), + compaction_time(0) +{ + nursery_size_before = parent->nursery.occupied_space(); + aging_size_before = parent->data->aging->occupied_space(); + tenured_size_before = parent->data->tenured->occupied_space(); + tenured_free_block_count_before = parent->data->tenured->free_blocks.free_block_count; + code_size_before = parent->code->allocator->occupied_space(); + code_free_block_count_before = parent->code->allocator->free_blocks.free_block_count; + start_time = current_micros(); +} + +void gc_event::started_card_scan() +{ + card_scan_time = current_micros(); +} + +void gc_event::ended_card_scan(cell cards_scanned_, cell decks_scanned_) +{ + cards_scanned += cards_scanned_; + decks_scanned += decks_scanned_; + card_scan_time = (card_scan_time - current_micros()); +} + +void gc_event::started_code_scan() +{ + code_scan_time = current_micros(); +} + +void gc_event::ended_code_scan(cell code_blocks_scanned_) +{ + code_blocks_scanned += code_blocks_scanned_; + code_scan_time = (code_scan_time - current_micros()); +} + +void gc_event::started_data_sweep() +{ + data_sweep_time = current_micros(); +} + +void gc_event::ended_data_sweep() +{ + data_sweep_time = (data_sweep_time - current_micros()); +} + +void gc_event::started_code_sweep() +{ + code_sweep_time = current_micros(); +} + +void gc_event::ended_code_sweep() +{ + code_sweep_time = (code_sweep_time - current_micros()); +} + +void gc_event::started_compaction() +{ + compaction_time = current_micros(); +} + +void gc_event::ended_compaction() +{ + compaction_time = (compaction_time - current_micros()); +} + +void gc_event::ended_gc(factor_vm *parent) +{ + nursery_size_after = parent->nursery.occupied_space(); + aging_size_after = parent->data->aging->occupied_space(); + tenured_size_after = parent->data->tenured->occupied_space(); + tenured_free_block_count_after = parent->data->tenured->free_blocks.free_block_count; + code_size_after = parent->code->allocator->occupied_space(); + code_free_block_count_after = parent->code->allocator->free_blocks.free_block_count; + total_time = current_micros() - start_time; +} + void factor_vm::update_code_heap_for_minor_gc(std::set *remembered_set) { /* The youngest generation that any code block can now reference */ diff --git a/vm/gc.hpp b/vm/gc.hpp index dff9923f6e..e6713af52f 100755 --- a/vm/gc.hpp +++ b/vm/gc.hpp @@ -19,6 +19,45 @@ struct gc_state { ~gc_state(); }; +struct gc_event { + gc_op op; + cell nursery_size_before; + cell aging_size_before; + cell tenured_size_before; + cell tenured_free_block_count_before; + cell code_size_before; + cell code_free_block_count_before; + cell nursery_size_after; + cell aging_size_after; + cell tenured_size_after; + cell tenured_free_block_count_after; + cell code_size_after; + cell code_free_block_count_after; + cell cards_scanned; + cell decks_scanned; + cell code_blocks_scanned; + u64 start_time; + cell total_time; + cell card_scan_time; + cell code_scan_time; + cell data_sweep_time; + cell code_sweep_time; + cell compaction_time; + + explicit gc_event(gc_op op_, factor_vm *parent); + void started_card_scan(); + void ended_card_scan(cell cards_scanned_, cell decks_scanned_); + void started_code_scan(); + void ended_code_scan(cell code_blocks_scanned_); + void started_data_sweep(); + void ended_data_sweep(); + void started_code_sweep(); + void ended_code_sweep(); + void started_compaction(); + void ended_compaction(); + void ended_gc(factor_vm *parent); +}; + VM_C_API void inline_gc(cell *gc_roots_base, cell gc_roots_size, factor_vm *parent); } From 91cec17e52b6c6168e3040990149fba9d394b48b Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Mon, 26 Oct 2009 23:57:26 -0500 Subject: [PATCH 35/74] vm: get GC events working, -verbosegc switch now produces more info --- .../known-words/known-words.factor | 6 -- vm/aging_collector.cpp | 10 +++ vm/code_heap.cpp | 2 + vm/collector.hpp | 8 +- vm/compaction.cpp | 4 + vm/full_collector.cpp | 2 + vm/gc.cpp | 88 ++++++++++++++----- vm/gc.hpp | 20 +++-- vm/nursery_collector.cpp | 10 +++ vm/to_tenured_collector.cpp | 10 +++ 10 files changed, 116 insertions(+), 44 deletions(-) diff --git a/basis/stack-checker/known-words/known-words.factor b/basis/stack-checker/known-words/known-words.factor index d064776673..1f03df9642 100644 --- a/basis/stack-checker/known-words/known-words.factor +++ b/basis/stack-checker/known-words/known-words.factor @@ -501,8 +501,6 @@ M: bad-executable summary \ compact-gc { } { } define-primitive -\ gc-stats { } { array } define-primitive - \ (save-image) { byte-array } { } define-primitive \ (save-image-and-exit) { byte-array } { } define-primitive @@ -701,10 +699,6 @@ M: bad-executable summary \ unimplemented { } { } define-primitive -\ gc-reset { } { } define-primitive - -\ gc-stats { } { array } define-primitive - \ jit-compile { quotation } { } define-primitive \ lookup-method { object array } { word } define-primitive diff --git a/vm/aging_collector.cpp b/vm/aging_collector.cpp index 7b74a00f99..e39e7e7d6f 100644 --- a/vm/aging_collector.cpp +++ b/vm/aging_collector.cpp @@ -21,9 +21,13 @@ void factor_vm::collect_aging() current_gc->op = collect_to_tenured_op; to_tenured_collector collector(this); + + current_gc->event->started_code_scan(); collector.trace_cards(data->tenured, card_points_to_aging, simple_unmarker(card_mark_mask)); + current_gc->event->ended_card_scan(collector.cards_scanned,collector.decks_scanned); + collector.tenure_reachable_objects(); } { @@ -37,10 +41,16 @@ void factor_vm::collect_aging() collector.trace_roots(); collector.trace_contexts(); + + current_gc->event->started_code_scan(); collector.trace_code_heap_roots(&code->points_to_aging); + current_gc->event->ended_code_scan(collector.code_blocks_scanned); + collector.cheneys_algorithm(); + current_gc->event->started_code_sweep(); update_code_heap_for_minor_gc(&code->points_to_aging); + current_gc->event->ended_code_sweep(); data->reset_generation(&nursery); code->points_to_nursery.clear(); diff --git a/vm/code_heap.cpp b/vm/code_heap.cpp index d38ec9e3f5..444ec486f4 100755 --- a/vm/code_heap.cpp +++ b/vm/code_heap.cpp @@ -118,8 +118,10 @@ struct word_and_literal_code_heap_updater { void factor_vm::update_code_heap_words_and_literals() { + current_gc->event->started_code_sweep(); word_and_literal_code_heap_updater updater(this); code->allocator->sweep(updater); + current_gc->event->ended_code_sweep(); } /* After growing the heap, we have to perform a full relocation to update diff --git a/vm/collector.hpp b/vm/collector.hpp index d3cfa24642..4535f98dd7 100644 --- a/vm/collector.hpp +++ b/vm/collector.hpp @@ -79,7 +79,6 @@ template struct collector { slot_visitor > workhorse; cell cards_scanned; cell decks_scanned; - cell card_scan_time; cell code_blocks_scanned; explicit collector(factor_vm *parent_, TargetGeneration *target_, Policy policy_) : @@ -90,7 +89,6 @@ template struct collector { workhorse(make_collector_workhorse(parent_,target_,policy_)), cards_scanned(0), decks_scanned(0), - card_scan_time(0), code_blocks_scanned(0) {} void trace_handle(cell *handle) @@ -179,8 +177,6 @@ template struct collector { template void trace_cards(SourceGeneration *gen, card mask, Unmarker unmarker) { - u64 start_time = current_micros(); - card_deck *decks = data->decks; card_deck *cards = data->cards; @@ -238,15 +234,13 @@ scan_next_object: { unmarker(&cards[card_index]); - if(!start) goto end; + if(!start) return; } } unmarker(&decks[deck_index]); } } - -end: card_scan_time += (current_micros() - start_time); } }; diff --git a/vm/compaction.cpp b/vm/compaction.cpp index 399971805b..4540560ac4 100644 --- a/vm/compaction.cpp +++ b/vm/compaction.cpp @@ -112,6 +112,8 @@ struct code_block_compaction_updater { void factor_vm::collect_compact_impl(bool trace_contexts_p) { + current_gc->event->started_compaction(); + tenured_space *tenured = data->tenured; mark_bits *data_forwarding_map = &tenured->state; mark_bits *code_forwarding_map = &code->allocator->state; @@ -146,6 +148,8 @@ void factor_vm::collect_compact_impl(bool trace_contexts_p) code_forwarder.visit_context_code_blocks(); code_forwarder.visit_callback_code_blocks(); } + + current_gc->event->ended_compaction(); } } diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index 0c90719b51..6167c73ab5 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -75,9 +75,11 @@ void factor_vm::collect_mark_impl(bool trace_contexts_p) void factor_vm::collect_sweep_impl() { + current_gc->event->started_data_sweep(); data->tenured->starts.clear_object_start_offsets(); object_start_map_updater updater(&data->tenured->starts); data->tenured->sweep(updater); + current_gc->event->ended_data_sweep(); } void factor_vm::collect_growing_heap(cell requested_bytes, bool trace_contexts_p) diff --git a/vm/gc.cpp b/vm/gc.cpp index be505b4ef9..5d9f68e6b8 100755 --- a/vm/gc.cpp +++ b/vm/gc.cpp @@ -3,14 +3,14 @@ namespace factor { -gc_state::gc_state(gc_op op_) : op(op_), start_time(current_micros()) {} - -gc_state::~gc_state() {} - gc_event::gc_event(gc_op op_, factor_vm *parent) : op(op_), + cards_scanned(0), + decks_scanned(0), + code_blocks_scanned(0), start_time(current_micros()), card_scan_time(0), + code_scan_time(0), data_sweep_time(0), code_sweep_time(0), compaction_time(0) @@ -33,7 +33,7 @@ void gc_event::ended_card_scan(cell cards_scanned_, cell decks_scanned_) { cards_scanned += cards_scanned_; decks_scanned += decks_scanned_; - card_scan_time = (card_scan_time - current_micros()); + card_scan_time = (current_micros() - card_scan_time); } void gc_event::started_code_scan() @@ -44,7 +44,7 @@ void gc_event::started_code_scan() void gc_event::ended_code_scan(cell code_blocks_scanned_) { code_blocks_scanned += code_blocks_scanned_; - code_scan_time = (code_scan_time - current_micros()); + code_scan_time = (current_micros() - code_scan_time); } void gc_event::started_data_sweep() @@ -54,7 +54,7 @@ void gc_event::started_data_sweep() void gc_event::ended_data_sweep() { - data_sweep_time = (data_sweep_time - current_micros()); + data_sweep_time = (current_micros() - data_sweep_time); } void gc_event::started_code_sweep() @@ -64,7 +64,7 @@ void gc_event::started_code_sweep() void gc_event::ended_code_sweep() { - code_sweep_time = (code_sweep_time - current_micros()); + code_sweep_time = (current_micros() - code_sweep_time); } void gc_event::started_compaction() @@ -74,7 +74,7 @@ void gc_event::started_compaction() void gc_event::ended_compaction() { - compaction_time = (compaction_time - current_micros()); + compaction_time = (current_micros() - compaction_time); } void gc_event::ended_gc(factor_vm *parent) @@ -88,6 +88,55 @@ void gc_event::ended_gc(factor_vm *parent) total_time = current_micros() - start_time; } +std::ostream &operator<<(std::ostream &out, const gc_event *event) +{ + out << ""; + return out; +} + +gc_state::gc_state(gc_op op_, factor_vm *parent) : op(op_), start_time(current_micros()) +{ + event = new gc_event(op,parent); +} + +gc_state::~gc_state() +{ + delete event; + event = NULL; +} + +void gc_state::start_again(gc_op op_, factor_vm *parent) +{ + event->ended_gc(parent); + if(parent->verbose_gc) std::cout << event << std::endl; + delete event; + event = new gc_event(op_,parent); + op = op_; +} + void factor_vm::update_code_heap_for_minor_gc(std::set *remembered_set) { /* The youngest generation that any code block can now reference */ @@ -104,10 +153,7 @@ void factor_vm::gc(gc_op op, cell requested_bytes, bool trace_contexts_p) save_stacks(); - current_gc = new gc_state(op); - - if(verbose_gc) - std::cout << "GC requested, op=" << op << std::endl; + current_gc = new gc_state(op,this); /* Keep trying to GC higher and higher generations until we don't run out of space */ @@ -117,25 +163,22 @@ void factor_vm::gc(gc_op op, cell requested_bytes, bool trace_contexts_p) switch(current_gc->op) { case collect_nursery_op: - current_gc->op = collect_aging_op; + current_gc->start_again(collect_aging_op,this); break; case collect_aging_op: - current_gc->op = collect_to_tenured_op; + current_gc->start_again(collect_to_tenured_op,this); break; case collect_to_tenured_op: - current_gc->op = collect_full_op; + current_gc->start_again(collect_full_op,this); break; case collect_full_op: case collect_compact_op: - current_gc->op = collect_growing_heap_op; + current_gc->start_again(collect_growing_heap_op,this); break; default: critical_error("Bad GC op",current_gc->op); break; } - - if(verbose_gc) - std::cout << "GC rewind, op=" << current_gc->op << std::endl; } switch(current_gc->op) @@ -166,8 +209,9 @@ void factor_vm::gc(gc_op op, cell requested_bytes, bool trace_contexts_p) break; } - if(verbose_gc) - std::cout << "GC done, op=" << current_gc->op << std::endl; + current_gc->event->ended_gc(this); + + if(verbose_gc) std::cout << current_gc->event << std::endl; delete current_gc; current_gc = NULL; diff --git a/vm/gc.hpp b/vm/gc.hpp index e6713af52f..fe19f93ff1 100755 --- a/vm/gc.hpp +++ b/vm/gc.hpp @@ -10,15 +10,6 @@ enum gc_op { collect_growing_heap_op }; -struct gc_state { - gc_op op; - u64 start_time; - jmp_buf gc_unwind; - - explicit gc_state(gc_op op_); - ~gc_state(); -}; - struct gc_event { gc_op op; cell nursery_size_before; @@ -58,6 +49,17 @@ struct gc_event { void ended_gc(factor_vm *parent); }; +struct gc_state { + gc_op op; + u64 start_time; + jmp_buf gc_unwind; + gc_event *event; + + explicit gc_state(gc_op op_, factor_vm *parent); + ~gc_state(); + void start_again(gc_op op_, factor_vm *parent); +}; + VM_C_API void inline_gc(cell *gc_roots_base, cell gc_roots_size, factor_vm *parent); } diff --git a/vm/nursery_collector.cpp b/vm/nursery_collector.cpp index 096bf7f757..240fac60d9 100644 --- a/vm/nursery_collector.cpp +++ b/vm/nursery_collector.cpp @@ -17,15 +17,25 @@ void factor_vm::collect_nursery() collector.trace_roots(); collector.trace_contexts(); + + current_gc->event->started_card_scan(); collector.trace_cards(data->tenured, card_points_to_nursery, simple_unmarker(card_points_to_nursery)); collector.trace_cards(data->aging, card_points_to_nursery, simple_unmarker(card_mark_mask)); + current_gc->event->ended_card_scan(collector.cards_scanned,collector.decks_scanned); + + current_gc->event->started_code_scan(); collector.trace_code_heap_roots(&code->points_to_nursery); + current_gc->event->ended_code_scan(collector.code_blocks_scanned); + collector.cheneys_algorithm(); + + current_gc->event->started_code_sweep(); update_code_heap_for_minor_gc(&code->points_to_nursery); + current_gc->event->ended_code_sweep(); data->reset_generation(&nursery); code->points_to_nursery.clear(); diff --git a/vm/to_tenured_collector.cpp b/vm/to_tenured_collector.cpp index 97009a5ceb..a39d92d869 100644 --- a/vm/to_tenured_collector.cpp +++ b/vm/to_tenured_collector.cpp @@ -29,12 +29,22 @@ void factor_vm::collect_to_tenured() collector.trace_roots(); collector.trace_contexts(); + + current_gc->event->started_card_scan(); collector.trace_cards(data->tenured, card_points_to_aging, simple_unmarker(card_mark_mask)); + current_gc->event->ended_card_scan(collector.cards_scanned,collector.decks_scanned); + + current_gc->event->started_code_scan(); collector.trace_code_heap_roots(&code->points_to_aging); + current_gc->event->ended_code_scan(collector.code_blocks_scanned); + collector.tenure_reachable_objects(); + + current_gc->event->started_code_sweep(); update_code_heap_for_minor_gc(&code->points_to_aging); + current_gc->event->ended_code_sweep(); data->reset_generation(&nursery); data->reset_generation(data->aging); From d95a98eb9ce93b1cc8bef1701a195dff7163d23d Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Tue, 27 Oct 2009 03:32:28 -0500 Subject: [PATCH 36/74] vm: add primitives for getting at GC events, data-room and code-room primitives now return structs instead of arrays --- .../known-words/known-words.factor | 7 +- basis/tools/memory/memory.factor | 70 +++++++++------- basis/vm/vm.factor | 55 ++++++++++++- core/bootstrap/primitives.factor | 6 +- vm/byte_arrays.hpp | 13 +++ vm/code_heap.cpp | 16 ++-- vm/code_heap.hpp | 8 ++ vm/data_heap.cpp | 37 ++++----- vm/data_heap.hpp | 17 ++++ vm/gc.cpp | 82 ++++++++++++------- vm/primitives.cpp | 4 + vm/vm.cpp | 1 + vm/vm.hpp | 10 +++ 13 files changed, 235 insertions(+), 91 deletions(-) diff --git a/basis/stack-checker/known-words/known-words.factor b/basis/stack-checker/known-words/known-words.factor index 1f03df9642..a5ddde9409 100644 --- a/basis/stack-checker/known-words/known-words.factor +++ b/basis/stack-checker/known-words/known-words.factor @@ -505,10 +505,10 @@ M: bad-executable summary \ (save-image-and-exit) { byte-array } { } define-primitive -\ data-room { } { array } define-primitive +\ data-room { } { byte-array } define-primitive \ data-room make-flushable -\ code-room { } { array } define-primitive +\ code-room { } { byte-array } define-primitive \ code-room make-flushable \ micros { } { integer } define-primitive @@ -713,3 +713,6 @@ M: bad-executable summary \ strip-stack-traces { } { } define-primitive \ { word } { alien } define-primitive + +\ enable-gc-events { } { } define-primitive +\ disable-gc-events { } { object } define-primitive diff --git a/basis/tools/memory/memory.factor b/basis/tools/memory/memory.factor index a5ed0fc0c1..8ede74d2fe 100644 --- a/basis/tools/memory/memory.factor +++ b/basis/tools/memory/memory.factor @@ -3,7 +3,9 @@ USING: kernel sequences arrays generic assocs io math namespaces parser prettyprint strings io.styles words system sorting splitting grouping math.parser classes memory -combinators fry ; +combinators fry vm specialized-arrays accessors continuations +classes.struct ; +SPECIALIZED-ARRAY: gc-event IN: tools.memory [ 3 cut* "," glue ] when " KB" append ; -: fancy-table. ( seq alist -- ) - [ [ nip first ] [ second call( obj -- str ) ] 2bi 2array ] 2map +: fancy-table. ( obj alist -- ) + [ [ nip first ] [ second call( obj -- str ) ] 2bi 2array ] with map simple-table. ; -: young-room. ( seq -- ) +: copying-room. ( copying-sizes -- ) { - { "Total:" [ kilobytes ] } - { "Allocated:" [ kilobytes ] } - { "Free:" [ kilobytes ] } + { "Size:" [ size>> kilobytes ] } + { "Occupied:" [ occupied>> kilobytes ] } + { "Free:" [ free>> kilobytes ] } } fancy-table. ; -: nursery-room. ( seq -- ) "- Nursery space" print young-room. ; +: nursery-room. ( data-room -- ) + "- Nursery space" print nursery>> copying-room. ; -: aging-room. ( seq -- ) "- Aging space" print young-room. ; +: aging-room. ( data-room -- ) + "- Aging space" print aging>> copying-room. ; -: mark-sweep-table. ( sizes -- ) +: mark-sweep-table. ( mark-sweep-sizes -- ) { - { "Total:" [ kilobytes ] } - { "Allocated:" [ kilobytes ] } - { "Total free:" [ kilobytes ] } - { "Contiguous free:" [ kilobytes ] } - { "Free list entries:" [ number>string ] } + { "Size:" [ size>> kilobytes ] } + { "Occupied:" [ occupied>> kilobytes ] } + { "Total free:" [ total-free>> kilobytes ] } + { "Contiguous free:" [ contiguous-free>> kilobytes ] } + { "Free block count:" [ free-block-count>> number>string ] } } fancy-table. ; -: tenured-room. ( seq -- ) "- Tenured space" print mark-sweep-table. ; +: tenured-room. ( data-room -- ) + "- Tenured space" print tenured>> mark-sweep-table. ; -: misc-room. ( seq -- ) +: misc-room. ( data-room -- ) "- Miscellaneous buffers" print { - { "Card array:" [ kilobytes ] } - { "Deck array:" [ kilobytes ] } - { "Mark stack:" [ kilobytes ] } + { "Card array:" [ cards>> kilobytes ] } + { "Deck array:" [ decks>> kilobytes ] } + { "Mark stack:" [ mark-stack>> kilobytes ] } } fancy-table. ; : data-room. ( -- ) "==== DATA HEAP" print nl - data-room - 3 cut [ nursery-room. nl ] dip - 3 cut [ aging-room. nl ] dip - 5 cut [ tenured-room. nl ] dip - misc-room. ; + data-room data-heap-room memory>struct { + [ nursery-room. nl ] + [ aging-room. nl ] + [ tenured-room. nl ] + [ misc-room. ] + } cleave ; : code-room. ( -- ) "==== CODE HEAP" print nl - code-room mark-sweep-table. ; + code-room mark-sweep-sizes memory>struct mark-sweep-table. ; + +PRIVATE> + +: room. ( -- ) data-room. nl code-room. ; + + -: room. ( -- ) data-room. nl code-room. ; - : heap-stats ( -- counts sizes ) [ ] instances H{ } clone H{ } clone [ '[ _ _ heap-stat-step ] each ] 2keep ; @@ -82,3 +92,7 @@ PRIVATE> ] with-row ] each 2drop ] tabular-output nl ; + +: collect-gc-events ( quot -- events ) + enable-gc-events [ ] [ disable-gc-events drop ] cleanup + disable-gc-events byte-array>gc-event-array ; diff --git a/basis/vm/vm.factor b/basis/vm/vm.factor index 11d9dabb3d..1351ed10a3 100644 --- a/basis/vm/vm.factor +++ b/basis/vm/vm.factor @@ -3,7 +3,7 @@ USING: classes.struct alien.c-types alien.syntax ; IN: vm -TYPEDEF: void* cell +TYPEDEF: intptr_t cell C-TYPE: context STRUCT: zone @@ -20,3 +20,56 @@ STRUCT: vm { userenv cell[70] } ; : vm-field-offset ( field -- offset ) vm offset-of ; inline + +C-ENUM: +collect-nursery-op +collect-aging-op +collect-to-tenured-op +collect-full-op +collect-compact-op +collect-growing-heap-op ; + +STRUCT: gc-event +{ op uint } +{ nursery-size-before cell } +{ aging-size-before cell } +{ tenured-size-before cell } +{ tenured-free-block-count-before cell } +{ code-size-before cell } +{ code-free-block-count-before cell } +{ nursery-size-after cell } +{ aging-size-after cell } +{ tenured-size-after cell } +{ tenured-free-block-count-after cell } +{ code-size-after cell } +{ code-free-block-count-after cell } +{ cards-scanned cell } +{ decks-scanned cell } +{ code-blocks-scanned cell } +{ start-time ulonglong } +{ total-time cell } +{ card-scan-time cell } +{ code-scan-time cell } +{ data-sweep-time cell } +{ code-sweep-time cell } +{ compaction-time cell } ; + +STRUCT: copying-sizes +{ size cell } +{ occupied cell } +{ free cell } ; + +STRUCT: mark-sweep-sizes +{ size cell } +{ occupied cell } +{ total-free cell } +{ contiguous-free cell } +{ free-block-count cell } ; + +STRUCT: data-heap-room +{ nursery copying-sizes } +{ aging copying-sizes } +{ tenured mark-sweep-sizes } +{ cards cell } +{ decks cell } +{ mark-stack cell } ; diff --git a/core/bootstrap/primitives.factor b/core/bootstrap/primitives.factor index 30a41eb93d..81c09f19fa 100644 --- a/core/bootstrap/primitives.factor +++ b/core/bootstrap/primitives.factor @@ -432,8 +432,8 @@ tuple { "set-retainstack" "kernel" (( rs -- )) } { "set-callstack" "kernel" (( cs -- )) } { "exit" "system" (( n -- )) } - { "data-room" "memory" (( -- cards decks generations )) } - { "code-room" "memory" (( -- code-total code-used code-free largest-free-block )) } + { "data-room" "memory" (( -- data-room )) } + { "code-room" "memory" (( -- code-room )) } { "micros" "system" (( -- us )) } { "modify-code-heap" "compiler.units" (( alist -- )) } { "(dlopen)" "alien.libraries" (( path -- dll )) } @@ -524,6 +524,8 @@ tuple { "vm-ptr" "vm" (( -- ptr )) } { "strip-stack-traces" "kernel.private" (( -- )) } { "" "alien" (( word -- alien )) } + { "enable-gc-events" "memory" (( -- )) } + { "disable-gc-events" "memory" (( -- events )) } } [ [ first3 ] dip swap make-primitive ] each-index ! Bump build number diff --git a/vm/byte_arrays.hpp b/vm/byte_arrays.hpp index 8ca95d9d1e..93cb775b1e 100755 --- a/vm/byte_arrays.hpp +++ b/vm/byte_arrays.hpp @@ -13,4 +13,17 @@ struct growable_byte_array { void trim(); }; +template byte_array *factor_vm::byte_array_from_value(T *value) +{ + return byte_array_from_values(value,1); +} + +template byte_array *factor_vm::byte_array_from_values(T *values, cell len) +{ + cell size = sizeof(T) * len; + byte_array *data = allot_uninitialized_array(size); + memcpy(data->data(),values,size); + return data; +} + } diff --git a/vm/code_heap.cpp b/vm/code_heap.cpp index 444ec486f4..ac6669ca49 100755 --- a/vm/code_heap.cpp +++ b/vm/code_heap.cpp @@ -196,19 +196,17 @@ void factor_vm::primitive_modify_code_heap() update_code_heap_words(); } -/* Push the free space and total size of the code heap */ void factor_vm::primitive_code_room() { - growable_array a(this); + code_heap_room room; - a.add(tag_fixnum(code->allocator->size)); - a.add(tag_fixnum(code->allocator->occupied_space())); - a.add(tag_fixnum(code->allocator->free_space())); - a.add(tag_fixnum(code->allocator->free_blocks.largest_free_block())); - a.add(tag_fixnum(code->allocator->free_blocks.free_block_count)); + room.size = code->allocator->size; + room.occupied_space = code->allocator->occupied_space(); + room.total_free = code->allocator->free_space(); + room.contiguous_free = code->allocator->free_blocks.largest_free_block(); + room.free_block_count = code->allocator->free_blocks.free_block_count; - a.trim(); - dpush(a.elements.value()); + dpush(tag(byte_array_from_value(&room))); } struct stack_trace_stripper { diff --git a/vm/code_heap.hpp b/vm/code_heap.hpp index 38e53d9fbe..8f4790d2f9 100755 --- a/vm/code_heap.hpp +++ b/vm/code_heap.hpp @@ -28,4 +28,12 @@ struct code_heap { void code_heap_free(code_block *compiled); }; +struct code_heap_room { + cell size; + cell occupied_space; + cell total_free; + cell contiguous_free; + cell free_block_count; +}; + } diff --git a/vm/data_heap.cpp b/vm/data_heap.cpp index d8b5ca3efb..8f8db699f6 100755 --- a/vm/data_heap.cpp +++ b/vm/data_heap.cpp @@ -201,31 +201,26 @@ void factor_vm::primitive_size() box_unsigned_cell(object_size(dpop())); } -/* Push memory usage statistics in data heap */ void factor_vm::primitive_data_room() { - growable_array a(this); + data_heap_room room; - a.add(tag_fixnum(nursery.size)); - a.add(tag_fixnum(nursery.occupied_space())); - a.add(tag_fixnum(nursery.free_space())); + room.nursery_size = nursery.size; + room.nursery_occupied = nursery.occupied_space(); + room.nursery_free = nursery.free_space(); + room.aging_size = data->aging->size; + room.aging_occupied = data->aging->occupied_space(); + room.aging_free = data->aging->free_space(); + room.tenured_size = data->tenured->size; + room.tenured_occupied = data->tenured->occupied_space(); + room.tenured_total_free = data->tenured->free_space(); + room.tenured_contiguous_free = data->tenured->free_blocks.largest_free_block(); + room.tenured_free_block_count = data->tenured->free_blocks.free_block_count; + room.cards = data->cards_end - data->cards; + room.decks = data->decks_end - data->decks; + room.mark_stack = data->tenured->mark_stack.capacity(); - a.add(tag_fixnum(data->aging->size)); - a.add(tag_fixnum(data->aging->occupied_space())); - a.add(tag_fixnum(data->aging->free_space())); - - a.add(tag_fixnum(data->tenured->size)); - a.add(tag_fixnum(data->tenured->occupied_space())); - a.add(tag_fixnum(data->tenured->free_space())); - a.add(tag_fixnum(data->tenured->free_blocks.largest_free_block())); - a.add(tag_fixnum(data->tenured->free_blocks.free_block_count)); - - a.add(tag_fixnum(data->cards_end - data->cards)); - a.add(tag_fixnum(data->decks_end - data->decks)); - a.add(tag_fixnum(data->tenured->mark_stack.capacity())); - - a.trim(); - dpush(a.elements.value()); + dpush(tag(byte_array_from_value(&room))); } /* Disables GC and activates next-object ( -- obj ) primitive */ diff --git a/vm/data_heap.hpp b/vm/data_heap.hpp index c8d6ce0b70..a434024873 100755 --- a/vm/data_heap.hpp +++ b/vm/data_heap.hpp @@ -31,4 +31,21 @@ struct data_heap { void reset_generation(tenured_space *gen); }; +struct data_heap_room { + cell nursery_size; + cell nursery_occupied; + cell nursery_free; + cell aging_size; + cell aging_occupied; + cell aging_free; + cell tenured_size; + cell tenured_occupied; + cell tenured_total_free; + cell tenured_contiguous_free; + cell tenured_free_block_count; + cell cards; + cell decks; + cell mark_stack; +}; + } diff --git a/vm/gc.cpp b/vm/gc.cpp index 5d9f68e6b8..0c8750ad92 100755 --- a/vm/gc.cpp +++ b/vm/gc.cpp @@ -128,13 +128,40 @@ gc_state::~gc_state() event = NULL; } -void gc_state::start_again(gc_op op_, factor_vm *parent) +void factor_vm::end_gc() { - event->ended_gc(parent); - if(parent->verbose_gc) std::cout << event << std::endl; - delete event; - event = new gc_event(op_,parent); - op = op_; + current_gc->event->ended_gc(this); + if(verbose_gc) std::cout << current_gc->event << std::endl; + if(gc_events) gc_events->push_back(*current_gc->event); + delete current_gc->event; + current_gc->event = NULL; +} + +void factor_vm::start_gc_again() +{ + end_gc(); + + switch(current_gc->op) + { + case collect_nursery_op: + current_gc->op = collect_aging_op; + break; + case collect_aging_op: + current_gc->op = collect_to_tenured_op; + break; + case collect_to_tenured_op: + current_gc->op = collect_full_op; + break; + case collect_full_op: + case collect_compact_op: + current_gc->op = collect_growing_heap_op; + break; + default: + critical_error("Bad GC op",current_gc->op); + break; + } + + current_gc->event = new gc_event(current_gc->op,this); } void factor_vm::update_code_heap_for_minor_gc(std::set *remembered_set) @@ -160,25 +187,7 @@ void factor_vm::gc(gc_op op, cell requested_bytes, bool trace_contexts_p) if(setjmp(current_gc->gc_unwind)) { /* We come back here if a generation is full */ - switch(current_gc->op) - { - case collect_nursery_op: - current_gc->start_again(collect_aging_op,this); - break; - case collect_aging_op: - current_gc->start_again(collect_to_tenured_op,this); - break; - case collect_to_tenured_op: - current_gc->start_again(collect_full_op,this); - break; - case collect_full_op: - case collect_compact_op: - current_gc->start_again(collect_growing_heap_op,this); - break; - default: - critical_error("Bad GC op",current_gc->op); - break; - } + start_gc_again(); } switch(current_gc->op) @@ -209,9 +218,7 @@ void factor_vm::gc(gc_op op, cell requested_bytes, bool trace_contexts_p) break; } - current_gc->event->ended_gc(this); - - if(verbose_gc) std::cout << current_gc->event << std::endl; + end_gc(); delete current_gc; current_gc = NULL; @@ -338,4 +345,23 @@ object *factor_vm::allot_object(header header, cell size) return obj; } +void factor_vm::primitive_enable_gc_events() +{ + gc_events = new std::vector(); +} + +void factor_vm::primitive_disable_gc_events() +{ + if(gc_events) + { + byte_array *data = byte_array_from_values(&gc_events->front(),gc_events->size()); + dpush(tag(data)); + + delete gc_events; + gc_events = NULL; + } + else + dpush(false_object); +} + } diff --git a/vm/primitives.cpp b/vm/primitives.cpp index 5ba57ec8af..f950e19e5a 100644 --- a/vm/primitives.cpp +++ b/vm/primitives.cpp @@ -128,6 +128,8 @@ PRIMITIVE_FORWARD(quot_compiled_p) PRIMITIVE_FORWARD(vm_ptr) PRIMITIVE_FORWARD(strip_stack_traces) PRIMITIVE_FORWARD(callback) +PRIMITIVE_FORWARD(enable_gc_events) +PRIMITIVE_FORWARD(disable_gc_events) const primitive_type primitives[] = { primitive_bignum_to_fixnum, @@ -292,6 +294,8 @@ const primitive_type primitives[] = { primitive_vm_ptr, primitive_strip_stack_traces, primitive_callback, + primitive_enable_gc_events, + primitive_disable_gc_events, }; } diff --git a/vm/vm.cpp b/vm/vm.cpp index bcdead7da5..2a41e96dfe 100755 --- a/vm/vm.cpp +++ b/vm/vm.cpp @@ -8,6 +8,7 @@ factor_vm::factor_vm() : profiling_p(false), gc_off(false), current_gc(NULL), + gc_events(NULL), fep_disabled(false), full_output(false) { } diff --git a/vm/vm.hpp b/vm/vm.hpp index 36f0dce049..128982fb9c 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -61,6 +61,9 @@ struct factor_vm /* Only set if we're performing a GC */ gc_state *current_gc; + /* If not NULL, we push GC events here */ + std::vector *gc_events; + /* If a runtime function needs to call another function which potentially allocates memory, it must wrap any local variable references to Factor objects in gc_root instances */ @@ -240,6 +243,8 @@ struct factor_vm } // gc + void end_gc(); + void start_gc_again(); void update_code_heap_for_minor_gc(std::set *remembered_set); void collect_nursery(); void collect_aging(); @@ -255,6 +260,8 @@ struct factor_vm void primitive_become(); void inline_gc(cell *gc_roots_base, cell gc_roots_size); object *allot_object(header header, cell size); + void primitive_enable_gc_events(); + void primitive_disable_gc_events(); template Type *allot(cell size) { @@ -336,6 +343,9 @@ struct factor_vm void primitive_uninitialized_byte_array(); void primitive_resize_byte_array(); + template byte_array *byte_array_from_value(T *value); + template byte_array *byte_array_from_values(T *values, cell len); + //tuples tuple *allot_tuple(cell layout_); void primitive_tuple(); From 21f1fe7aaf763c61953f36f41e9b9bb2f6f089eb Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Tue, 27 Oct 2009 03:37:05 -0500 Subject: [PATCH 37/74] tools.time: remove crappy old GC stats --- basis/tools/time/time-docs.factor | 11 +++---- basis/tools/time/time.factor | 54 ++++++------------------------- 2 files changed, 13 insertions(+), 52 deletions(-) diff --git a/basis/tools/time/time-docs.factor b/basis/tools/time/time-docs.factor index 408592d0c6..3f8cbbf799 100644 --- a/basis/tools/time/time-docs.factor +++ b/basis/tools/time/time-docs.factor @@ -6,12 +6,9 @@ ARTICLE: "timing" "Timing code" { $subsections time } "A lower-level word puts timings on the stack, intead of printing:" { $subsections benchmark } -"You can also read the system clock and garbage collection statistics directly:" -{ $subsections - micros - gc-stats -} -{ $see-also "profiling" } ; +"You can also read the system clock directly:" +{ $subsections micros } +{ $see-also "profiling" "calendar" } ; ABOUT: "timing" @@ -23,6 +20,6 @@ HELP: benchmark HELP: time { $values { "quot" "a quotation" } } -{ $description "Runs a quotation and then prints the total run time and some garbage collection statistics." } ; +{ $description "Runs a quotation and then prints the total run time and some statistics." } ; { benchmark micros time } related-words diff --git a/basis/tools/time/time.factor b/basis/tools/time/time.factor index 948c0d482d..3d0509e87d 100644 --- a/basis/tools/time/time.factor +++ b/basis/tools/time/time.factor @@ -1,4 +1,4 @@ -! Copyright (C) 2003, 2008 Slava Pestov. +! Copyright (C) 2003, 2009 Slava Pestov. ! See http://factorcode.org/license.txt for BSD license. USING: kernel math memory io io.styles prettyprint namespaces system sequences splitting grouping assocs strings @@ -11,64 +11,28 @@ IN: tools.time : time. ( time -- ) "== Running time ==" print nl 1000000 /f pprint " seconds" print ; -: gc-stats. ( stats -- ) - 5 cut* - "== Garbage collection ==" print nl - "Times are in microseconds." print nl - [ - 6 group - { - "GC count:" - "Total GC time:" - "Longest GC pause:" - "Average GC pause:" - "Objects copied:" - "Bytes copied:" - } prefix - flip - { "" "Nursery" "Aging" "Tenured" } prefix - simple-table. - ] - [ - nl - { - "Total GC time:" - "Cards scanned:" - "Decks scanned:" - "Card scan time:" - "Code heap literal scans:" - } swap zip simple-table. - ] bi* ; - : dispatch-stats. ( stats -- ) "== Megamorphic caches ==" print nl { "Hits" "Misses" } swap zip simple-table. ; : inline-cache-stats. ( stats -- ) - nl "== Polymorphic inline caches ==" print nl + "== Polymorphic inline caches ==" print nl 3 cut [ - "Transitions:" print + "- Transitions:" print { "Cold to monomorphic" "Mono to polymorphic" "Poly to megamorphic" } swap zip simple-table. nl ] [ - "Type check stubs:" print + "- Type check stubs:" print { "Tag only" "Hi-tag" "Tuple" "Hi-tag and tuple" } swap zip simple-table. ] bi* ; : time ( quot -- ) - gc-reset reset-dispatch-stats reset-inline-cache-stats - benchmark gc-stats dispatch-stats inline-cache-stats - H{ { table-gap { 20 20 } } } [ - [ - [ [ time. ] 3dip ] with-cell - [ ] with-cell - ] with-row - [ - [ [ gc-stats. ] 2dip ] with-cell - [ [ dispatch-stats. ] [ inline-cache-stats. ] bi* ] with-cell - ] with-row - ] tabular-output nl ; inline + benchmark dispatch-stats inline-cache-stats + [ time. nl ] + [ dispatch-stats. nl ] + [ inline-cache-stats. ] + tri* ; inline From 028a7be4d6929ced938e96797a12d40b9d20c8f0 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Tue, 27 Oct 2009 16:31:45 -0500 Subject: [PATCH 38/74] tools.memory: experimenting with some GC event data mining --- basis/tools/memory/memory.factor | 52 ++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/basis/tools/memory/memory.factor b/basis/tools/memory/memory.factor index 8ede74d2fe..446f047150 100644 --- a/basis/tools/memory/memory.factor +++ b/basis/tools/memory/memory.factor @@ -4,7 +4,7 @@ USING: kernel sequences arrays generic assocs io math namespaces parser prettyprint strings io.styles words system sorting splitting grouping math.parser classes memory combinators fry vm specialized-arrays accessors continuations -classes.struct ; +classes.struct generalizations ; SPECIALIZED-ARRAY: gc-event IN: tools.memory @@ -95,4 +95,52 @@ PRIVATE> : collect-gc-events ( quot -- events ) enable-gc-events [ ] [ disable-gc-events drop ] cleanup - disable-gc-events byte-array>gc-event-array ; + disable-gc-events byte-array>gc-event-array ; inline + +: generation-sizes-before ( events -- sizes ) + [ + { + [ start-time>> ] + [ nursery-size-before>> ] + [ aging-size-before>> ] + [ tenured-size-before>> ] + [ code-size-before>> ] + } cleave 5 narray + ] { } map-as ; + +: generation-sizes-after ( events -- sizes ) + [ + { + [ start-time>> ] + [ nursery-size-after>> ] + [ aging-size-after>> ] + [ tenured-size-after>> ] + [ code-size-after>> ] + } cleave 5 narray + ] { } map-as ; + +: reclaimed-space ( events -- sizes ) + [ + [ start-time>> ] [ + { + [ [ nursery-size-before>> ] [ nursery-size-after>> ] bi - ] + [ [ aging-size-before>> ] [ aging-size-after>> ] bi - ] + [ [ tenured-size-before>> ] [ tenured-size-after>> ] bi - ] + [ [ code-size-before>> ] [ code-size-after>> ] bi - ] + } cleave + + + + + ] bi 2array + ] { } map-as ; + +: allocated-space ( events -- sizes ) + 2 [ + [ second start-time>> ] [ + { + [ [ second nursery-size-before>> ] [ first nursery-size-after>> ] bi - ] + [ [ second aging-size-before>> ] [ first aging-size-after>> ] bi - ] + [ [ second tenured-size-before>> ] [ first tenured-size-after>> ] bi - ] + [ [ second code-size-before>> ] [ first code-size-after>> ] bi - ] + } cleave + + + + + ] bi 2array + ] { } map-as ; From a7b55e70688f1a254db3ca37533dc7002cef516b Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Tue, 27 Oct 2009 18:22:08 -0500 Subject: [PATCH 39/74] vm: use STL in free list, makes finding largest contiguous free block slightly faster --- vm/free_list.cpp | 104 +++++++++++++++++++++-------------------------- vm/free_list.hpp | 14 +++++-- 2 files changed, 58 insertions(+), 60 deletions(-) diff --git a/vm/free_list.cpp b/vm/free_list.cpp index e104e0b9bd..24ffab9f2c 100644 --- a/vm/free_list.cpp +++ b/vm/free_list.cpp @@ -5,7 +5,11 @@ namespace factor void free_list::clear_free_list() { - memset(this,0,sizeof(free_list)); + for(cell i = 0; i < free_list_count; i++) + small_blocks[i].clear(); + large_blocks.clear(); + free_block_count = 0; + free_space = 0; } void free_list::initial_free_list(cell start, cell end, cell occupied) @@ -27,59 +31,40 @@ void free_list::add_to_free_list(free_heap_block *block) free_space += size; if(size < free_list_count * block_granularity) - { - int index = size / block_granularity; - block->next_free = small_blocks[index]; - small_blocks[index] = block; - } + small_blocks[size / block_granularity].push_back(block); else - { - block->next_free = large_blocks; - large_blocks = block; - } + large_blocks.insert(block); } free_heap_block *free_list::find_free_block(cell size) { - cell attempt = size; - - while(attempt < free_list_count * block_granularity) + /* Check small free lists */ + for(cell i = size / block_granularity; i < free_list_count; i++) { - int index = attempt / block_granularity; - free_heap_block *block = small_blocks[index]; - if(block) + std::vector &blocks = small_blocks[i]; + if(blocks.size()) { - small_blocks[index] = block->next_free; - - free_block_count--; - free_space -= block->size(); - + free_heap_block *block = blocks.back(); + blocks.pop_back(); return block; } - - attempt++; } - free_heap_block *prev = NULL; - free_heap_block *block = large_blocks; + /* Check large free lists */ + free_heap_block key; + key.make_free(size); + large_block_set::iterator iter = large_blocks.lower_bound(&key); + large_block_set::iterator end = large_blocks.end(); - while(block) + if(iter != end) { - if(block->size() >= size) - { - if(prev) - prev->next_free = block->next_free; - else - large_blocks = block->next_free; + free_heap_block *block = *iter; + large_blocks.erase(iter); - free_block_count--; - free_space -= block->size(); + free_block_count--; + free_space -= block->size(); - return block; - } - - prev = block; - block = block->next_free; + return block; } return NULL; @@ -92,7 +77,6 @@ free_heap_block *free_list::split_free_block(free_heap_block *block, cell size) /* split the block in two */ free_heap_block *split = (free_heap_block *)((cell)block + size); split->make_free(block->size() - size); - split->next_free = block->next_free; block->make_free(size); add_to_free_list(split); } @@ -102,20 +86,19 @@ free_heap_block *free_list::split_free_block(free_heap_block *block, cell size) bool free_list::can_allot_p(cell size) { - cell attempt = size; - - while(attempt < free_list_count * block_granularity) + /* Check small free lists */ + for(cell i = size / block_granularity; i < free_list_count; i++) { - int index = attempt / block_granularity; - if(small_blocks[index]) return true; - attempt++; + if(small_blocks[i].size()) return true; } - free_heap_block *block = large_blocks; - while(block) + /* Check large free lists */ + large_block_set::const_iterator iter = large_blocks.begin(); + large_block_set::const_iterator end = large_blocks.end(); + + for(; iter != end; iter++) { - if(block->size() >= size) return true; - block = block->next_free; + if((*iter)->size() >= size) return true; } return false; @@ -123,16 +106,23 @@ bool free_list::can_allot_p(cell size) cell free_list::largest_free_block() { - cell largest = 0; - free_heap_block *scan = large_blocks; - - while(scan) + if(large_blocks.size()) { - largest = std::max(largest,scan->size()); - scan = scan->next_free; + large_block_set::iterator end = large_blocks.end(); + free_heap_block *block = *end; + large_blocks.erase(end); + return block->size(); } + else + { + for(int i = free_list_count - 1; i >= 0; i--) + { + if(small_blocks[i].size()) + return small_blocks[i].back()->size(); + } - return largest; + return 0; + } } } diff --git a/vm/free_list.hpp b/vm/free_list.hpp index 28bc063883..305de0ac58 100644 --- a/vm/free_list.hpp +++ b/vm/free_list.hpp @@ -6,7 +6,6 @@ static const cell free_list_count = 32; struct free_heap_block { cell header; - free_heap_block *next_free; bool free_p() const { @@ -24,9 +23,18 @@ struct free_heap_block } }; +struct block_size_compare { + bool operator()(free_heap_block *a, free_heap_block *b) + { + return a->size() < b->size(); + } +}; + +typedef std::multiset large_block_set; + struct free_list { - free_heap_block *small_blocks[free_list_count]; - free_heap_block *large_blocks; + std::vector small_blocks[free_list_count]; + large_block_set large_blocks; cell free_block_count; cell free_space; From a1aac4278696d303598679561cc8f399e1610cb7 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Tue, 27 Oct 2009 21:31:28 -0500 Subject: [PATCH 40/74] vm: clean up gc events, remove -verbosegc switch, fix compaction bug --- basis/tools/memory/memory.factor | 74 +++++++++++--------------------- basis/vm/vm.factor | 42 ++++++++---------- vm/byte_arrays.hpp | 6 +-- vm/code_heap.cpp | 8 +++- vm/compaction.cpp | 8 ++-- vm/data_heap.cpp | 8 +++- vm/factor.cpp | 4 -- vm/free_list.cpp | 6 +-- vm/free_list_allocator.hpp | 15 +++++++ vm/gc.cpp | 48 +++------------------ vm/gc.hpp | 16 ++----- vm/image.hpp | 1 - vm/mark_bits.hpp | 14 ++++++ vm/master.hpp | 2 +- vm/tenured_space.hpp | 18 +------- vm/vm.hpp | 5 +-- 16 files changed, 110 insertions(+), 165 deletions(-) diff --git a/basis/tools/memory/memory.factor b/basis/tools/memory/memory.factor index 446f047150..6d0c4b7c86 100644 --- a/basis/tools/memory/memory.factor +++ b/basis/tools/memory/memory.factor @@ -10,10 +10,15 @@ IN: tools.memory string + reverse 3 group "," join reverse ; + : kilobytes ( n -- str ) - 1024 /i number>string - dup length 4 > [ 3 cut* "," glue ] when - " KB" append ; + 1024 /i commas " KB" append ; + +: micros>string ( n -- str ) + commas " microseconds" append ; : fancy-table. ( obj alist -- ) [ [ nip first ] [ second call( obj -- str ) ] 2bi 2array ] with map @@ -97,50 +102,23 @@ PRIVATE> enable-gc-events [ ] [ disable-gc-events drop ] cleanup disable-gc-events byte-array>gc-event-array ; inline -: generation-sizes-before ( events -- sizes ) - [ - { - [ start-time>> ] - [ nursery-size-before>> ] - [ aging-size-before>> ] - [ tenured-size-before>> ] - [ code-size-before>> ] - } cleave 5 narray - ] { } map-as ; +: gc-op-string ( op -- string ) + { + { collect-nursery-op [ "copying from nursery" ] } + { collect-aging-op [ "copying from aging" ] } + { collect-to-tenured-op [ "copying to tenured" ] } + { collect-full-op [ "mark and sweep" ] } + { collect-compact-op [ "mark and compact" ] } + { collect-growing-heap-op [ "grow heap" ] } + } case ; -: generation-sizes-after ( events -- sizes ) - [ - { - [ start-time>> ] - [ nursery-size-after>> ] - [ aging-size-after>> ] - [ tenured-size-after>> ] - [ code-size-after>> ] - } cleave 5 narray - ] { } map-as ; +: space-reclaimed ( event -- bytes ) + [ data-heap-before>> ] [ data-heap-after>> ] bi + [ [ nursery>> ] [ aging>> ] [ tenured>> ] tri [ occupied>> ] tri@ + + ] bi@ - ; -: reclaimed-space ( events -- sizes ) - [ - [ start-time>> ] [ - { - [ [ nursery-size-before>> ] [ nursery-size-after>> ] bi - ] - [ [ aging-size-before>> ] [ aging-size-after>> ] bi - ] - [ [ tenured-size-before>> ] [ tenured-size-after>> ] bi - ] - [ [ code-size-before>> ] [ code-size-after>> ] bi - ] - } cleave - + + + - ] bi 2array - ] { } map-as ; - -: allocated-space ( events -- sizes ) - 2 [ - [ second start-time>> ] [ - { - [ [ second nursery-size-before>> ] [ first nursery-size-after>> ] bi - ] - [ [ second aging-size-before>> ] [ first aging-size-after>> ] bi - ] - [ [ second tenured-size-before>> ] [ first tenured-size-after>> ] bi - ] - [ [ second code-size-before>> ] [ first code-size-after>> ] bi - ] - } cleave - + + + - ] bi 2array - ] { } map-as ; +: gc-event. ( event -- ) + { + { "Event type:" [ op>> gc-op-string ] } + { "Total time:" [ total-time>> micros>string ] } + { "Space reclaimed:" [ space-reclaimed kilobytes ] } + } fancy-table. ; diff --git a/basis/vm/vm.factor b/basis/vm/vm.factor index 1351ed10a3..6bcdd2862a 100644 --- a/basis/vm/vm.factor +++ b/basis/vm/vm.factor @@ -29,31 +29,6 @@ collect-full-op collect-compact-op collect-growing-heap-op ; -STRUCT: gc-event -{ op uint } -{ nursery-size-before cell } -{ aging-size-before cell } -{ tenured-size-before cell } -{ tenured-free-block-count-before cell } -{ code-size-before cell } -{ code-free-block-count-before cell } -{ nursery-size-after cell } -{ aging-size-after cell } -{ tenured-size-after cell } -{ tenured-free-block-count-after cell } -{ code-size-after cell } -{ code-free-block-count-after cell } -{ cards-scanned cell } -{ decks-scanned cell } -{ code-blocks-scanned cell } -{ start-time ulonglong } -{ total-time cell } -{ card-scan-time cell } -{ code-scan-time cell } -{ data-sweep-time cell } -{ code-sweep-time cell } -{ compaction-time cell } ; - STRUCT: copying-sizes { size cell } { occupied cell } @@ -73,3 +48,20 @@ STRUCT: data-heap-room { cards cell } { decks cell } { mark-stack cell } ; + +STRUCT: gc-event +{ op uint } +{ data-heap-before data-heap-room } +{ code-heap-before mark-sweep-sizes } +{ data-heap-after data-heap-room } +{ code-heap-after mark-sweep-sizes } +{ cards-scanned cell } +{ decks-scanned cell } +{ code-blocks-scanned cell } +{ start-time ulonglong } +{ total-time cell } +{ card-scan-time cell } +{ code-scan-time cell } +{ data-sweep-time cell } +{ code-sweep-time cell } +{ compaction-time cell } ; diff --git a/vm/byte_arrays.hpp b/vm/byte_arrays.hpp index 93cb775b1e..0817f7dcdc 100755 --- a/vm/byte_arrays.hpp +++ b/vm/byte_arrays.hpp @@ -13,14 +13,14 @@ struct growable_byte_array { void trim(); }; -template byte_array *factor_vm::byte_array_from_value(T *value) +template byte_array *factor_vm::byte_array_from_value(Type *value) { return byte_array_from_values(value,1); } -template byte_array *factor_vm::byte_array_from_values(T *values, cell len) +template byte_array *factor_vm::byte_array_from_values(Type *values, cell len) { - cell size = sizeof(T) * len; + cell size = sizeof(Type) * len; byte_array *data = allot_uninitialized_array(size); memcpy(data->data(),values,size); return data; diff --git a/vm/code_heap.cpp b/vm/code_heap.cpp index ac6669ca49..77ae5180c8 100755 --- a/vm/code_heap.cpp +++ b/vm/code_heap.cpp @@ -196,7 +196,7 @@ void factor_vm::primitive_modify_code_heap() update_code_heap_words(); } -void factor_vm::primitive_code_room() +code_heap_room factor_vm::code_room() { code_heap_room room; @@ -206,6 +206,12 @@ void factor_vm::primitive_code_room() room.contiguous_free = code->allocator->free_blocks.largest_free_block(); room.free_block_count = code->allocator->free_blocks.free_block_count; + return room; +} + +void factor_vm::primitive_code_room() +{ + code_heap_room room = code_room(); dpush(tag(byte_array_from_value(&room))); } diff --git a/vm/compaction.cpp b/vm/compaction.cpp index 4540560ac4..bda8035858 100644 --- a/vm/compaction.cpp +++ b/vm/compaction.cpp @@ -55,10 +55,12 @@ struct compaction_sizer { cell operator()(object *obj) { - if(obj->free_p() || obj->h.hi_tag() != TUPLE_TYPE) - return obj->size(); - else + if(!forwarding_map->marked_p(obj)) + return forwarding_map->unmarked_space_starting_at(obj); + else if(obj->h.hi_tag() == TUPLE_TYPE) return align(tuple_size_with_forwarding(forwarding_map,obj),data_alignment); + else + return obj->size(); } }; diff --git a/vm/data_heap.cpp b/vm/data_heap.cpp index 8f8db699f6..b38674fe1f 100755 --- a/vm/data_heap.cpp +++ b/vm/data_heap.cpp @@ -201,7 +201,7 @@ void factor_vm::primitive_size() box_unsigned_cell(object_size(dpop())); } -void factor_vm::primitive_data_room() +data_heap_room factor_vm::data_room() { data_heap_room room; @@ -220,6 +220,12 @@ void factor_vm::primitive_data_room() room.decks = data->decks_end - data->decks; room.mark_stack = data->tenured->mark_stack.capacity(); + return room; +} + +void factor_vm::primitive_data_room() +{ + data_heap_room room = data_room(); dpush(tag(byte_array_from_value(&room))); } diff --git a/vm/factor.cpp b/vm/factor.cpp index 6a235ec88d..510900a036 100755 --- a/vm/factor.cpp +++ b/vm/factor.cpp @@ -38,7 +38,6 @@ void factor_vm::default_parameters(vm_parameters *p) p->max_pic_size = 3; p->fep = false; - p->verbose_gc = false; p->signals = true; #ifdef WINDOWS @@ -87,7 +86,6 @@ void factor_vm::init_parameters_from_args(vm_parameters *p, int argc, vm_char ** else if(factor_arg(arg,STRING_LITERAL("-callbacks=%d"),&p->callback_size)); else if(STRCMP(arg,STRING_LITERAL("-fep")) == 0) p->fep = true; else if(STRCMP(arg,STRING_LITERAL("-nosignals")) == 0) p->signals = false; - else if(STRCMP(arg,STRING_LITERAL("-verbosegc")) == 0) p->verbose_gc = true; else if(STRNCMP(arg,STRING_LITERAL("-i="),3) == 0) p->image_path = arg + 3; else if(STRCMP(arg,STRING_LITERAL("-console")) == 0) p->console = true; } @@ -142,8 +140,6 @@ void factor_vm::init_factor(vm_parameters *p) if(p->signals) init_signals(); - verbose_gc = p->verbose_gc; - if(p->console) open_console(); diff --git a/vm/free_list.cpp b/vm/free_list.cpp index 24ffab9f2c..644f62d870 100644 --- a/vm/free_list.cpp +++ b/vm/free_list.cpp @@ -108,10 +108,8 @@ cell free_list::largest_free_block() { if(large_blocks.size()) { - large_block_set::iterator end = large_blocks.end(); - free_heap_block *block = *end; - large_blocks.erase(end); - return block->size(); + large_block_set::reverse_iterator last = large_blocks.rbegin(); + return (*last)->size(); } else { diff --git a/vm/free_list_allocator.hpp b/vm/free_list_allocator.hpp index d74dd5be9a..dad54b644d 100644 --- a/vm/free_list_allocator.hpp +++ b/vm/free_list_allocator.hpp @@ -14,6 +14,7 @@ template struct free_list_allocator { Block *first_block(); Block *last_block(); Block *next_block_after(Block *block); + Block *next_allocated_block_after(Block *block); bool can_allot_p(cell size); Block *allot(cell size); void free(Block *block); @@ -58,6 +59,20 @@ template Block *free_list_allocator::next_block_after(Blo return (Block *)((cell)block + block->size()); } +template Block *free_list_allocator::next_allocated_block_after(Block *block) +{ + while(block != this->last_block() && block->free_p()) + { + free_heap_block *free_block = (free_heap_block *)block; + block = (object *)((cell)free_block + free_block->size()); + } + + if(block == this->last_block()) + return NULL; + else + return block; +} + template bool free_list_allocator::can_allot_p(cell size) { return free_blocks.can_allot_p(size); diff --git a/vm/gc.cpp b/vm/gc.cpp index 0c8750ad92..cce8298921 100755 --- a/vm/gc.cpp +++ b/vm/gc.cpp @@ -15,12 +15,8 @@ gc_event::gc_event(gc_op op_, factor_vm *parent) : code_sweep_time(0), compaction_time(0) { - nursery_size_before = parent->nursery.occupied_space(); - aging_size_before = parent->data->aging->occupied_space(); - tenured_size_before = parent->data->tenured->occupied_space(); - tenured_free_block_count_before = parent->data->tenured->free_blocks.free_block_count; - code_size_before = parent->code->allocator->occupied_space(); - code_free_block_count_before = parent->code->allocator->free_blocks.free_block_count; + data_heap_before = parent->data_room(); + code_heap_before = parent->code_room(); start_time = current_micros(); } @@ -79,44 +75,11 @@ void gc_event::ended_compaction() void gc_event::ended_gc(factor_vm *parent) { - nursery_size_after = parent->nursery.occupied_space(); - aging_size_after = parent->data->aging->occupied_space(); - tenured_size_after = parent->data->tenured->occupied_space(); - tenured_free_block_count_after = parent->data->tenured->free_blocks.free_block_count; - code_size_after = parent->code->allocator->occupied_space(); - code_free_block_count_after = parent->code->allocator->free_blocks.free_block_count; + data_heap_after = parent->data_room(); + code_heap_after = parent->code_room(); total_time = current_micros() - start_time; } -std::ostream &operator<<(std::ostream &out, const gc_event *event) -{ - out << ""; - return out; -} - gc_state::gc_state(gc_op op_, factor_vm *parent) : op(op_), start_time(current_micros()) { event = new gc_event(op,parent); @@ -131,7 +94,6 @@ gc_state::~gc_state() void factor_vm::end_gc() { current_gc->event->ended_gc(this); - if(verbose_gc) std::cout << current_gc->event << std::endl; if(gc_events) gc_events->push_back(*current_gc->event); delete current_gc->event; current_gc->event = NULL; @@ -354,7 +316,7 @@ void factor_vm::primitive_disable_gc_events() { if(gc_events) { - byte_array *data = byte_array_from_values(&gc_events->front(),gc_events->size()); + byte_array *data = byte_array_from_values(&gc_events->first(),gc_events->size()); dpush(tag(data)); delete gc_events; diff --git a/vm/gc.hpp b/vm/gc.hpp index fe19f93ff1..90b5ce6032 100755 --- a/vm/gc.hpp +++ b/vm/gc.hpp @@ -12,18 +12,10 @@ enum gc_op { struct gc_event { gc_op op; - cell nursery_size_before; - cell aging_size_before; - cell tenured_size_before; - cell tenured_free_block_count_before; - cell code_size_before; - cell code_free_block_count_before; - cell nursery_size_after; - cell aging_size_after; - cell tenured_size_after; - cell tenured_free_block_count_after; - cell code_size_after; - cell code_free_block_count_after; + data_heap_room data_heap_before; + code_heap_room code_heap_before; + data_heap_room data_heap_after; + code_heap_room code_heap_after; cell cards_scanned; cell decks_scanned; cell code_blocks_scanned; diff --git a/vm/image.hpp b/vm/image.hpp index f740b65f68..cca0e9e378 100755 --- a/vm/image.hpp +++ b/vm/image.hpp @@ -35,7 +35,6 @@ struct vm_parameters { cell young_size, aging_size, tenured_size; cell code_size; bool fep; - bool verbose_gc; bool console; bool signals; cell max_pic_size; diff --git a/vm/mark_bits.hpp b/vm/mark_bits.hpp index 8b6b0c75eb..1b8fa913a6 100644 --- a/vm/mark_bits.hpp +++ b/vm/mark_bits.hpp @@ -154,6 +154,20 @@ template struct mark_bits { #endif return new_block; } + + /* Find the next allocated block without calling size() on unmarked + objects. */ + cell unmarked_space_starting_at(Block *original) + { + char *start = (char *)original; + char *scan = start; + char *end = (char *)(this->start + this->size); + + while(scan != end && !marked_p((Block *)scan)) + scan += block_granularity; + + return scan - start; + } }; } diff --git a/vm/master.hpp b/vm/master.hpp index 0873480add..9b525bd22e 100755 --- a/vm/master.hpp +++ b/vm/master.hpp @@ -59,6 +59,7 @@ namespace factor #include "aging_space.hpp" #include "tenured_space.hpp" #include "data_heap.hpp" +#include "code_heap.hpp" #include "gc.hpp" #include "debug.hpp" #include "strings.hpp" @@ -68,7 +69,6 @@ namespace factor #include "io.hpp" #include "image.hpp" #include "alien.hpp" -#include "code_heap.hpp" #include "callbacks.hpp" #include "vm.hpp" #include "tagged.hpp" diff --git a/vm/tenured_space.hpp b/vm/tenured_space.hpp index 1b3baeaf52..f77f17a56a 100644 --- a/vm/tenured_space.hpp +++ b/vm/tenured_space.hpp @@ -20,30 +20,16 @@ struct tenured_space : free_list_allocator { return NULL; } - object *first_allocated_block_after(object *block) - { - while(block != this->last_block() && block->free_p()) - { - free_heap_block *free_block = (free_heap_block *)block; - block = (object *)((cell)free_block + free_block->size()); - } - - if(block == this->last_block()) - return NULL; - else - return block; - } - cell first_object() { - return (cell)first_allocated_block_after(this->first_block()); + return (cell)next_allocated_block_after(this->first_block()); } cell next_object_after(cell scan) { cell size = ((object *)scan)->size(); object *next = (object *)(scan + size); - return (cell)first_allocated_block_after(next); + return (cell)next_allocated_block_after(next); } void clear_mark_bits() diff --git a/vm/vm.hpp b/vm/vm.hpp index 128982fb9c..0844b47e51 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -46,9 +46,6 @@ struct factor_vm /* GC is off during heap walking */ bool gc_off; - /* GC logging */ - bool verbose_gc; - /* Data heap */ data_heap *data; @@ -223,6 +220,7 @@ struct factor_vm void set_data_heap(data_heap *data_); void init_data_heap(cell young_size, cell aging_size, cell tenured_size); void primitive_size(); + data_heap_room data_room(); void primitive_data_room(); void begin_scan(); void end_scan(); @@ -502,6 +500,7 @@ struct factor_vm void update_code_heap_words_and_literals(); void relocate_code_heap(); void primitive_modify_code_heap(); + code_heap_room code_room(); void primitive_code_room(); void primitive_strip_stack_traces(); From 5536003cf887a47ccc9f08701d6ca8adc66b8301 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Fri, 30 Oct 2009 02:26:57 -0500 Subject: [PATCH 41/74] vm: new GC policy: mark sweep after N megabytes promoted, mark compact if tenured space cannot fit nursery+aging in one contiguous block. N = 32 on 32-bit and 64 on 64-bit --- vm/code_heap.cpp | 4 ++-- vm/data_heap.cpp | 25 ++++++++++++++----------- vm/data_heap.hpp | 4 +++- vm/factor.cpp | 17 ++++------------- vm/free_list.cpp | 4 ++++ vm/free_list_allocator.hpp | 23 ++++++++++++++++++++++- vm/gc.cpp | 15 ++++++++++++++- vm/image.cpp | 3 ++- vm/image.hpp | 1 + vm/vm.hpp | 2 +- 10 files changed, 67 insertions(+), 31 deletions(-) diff --git a/vm/code_heap.cpp b/vm/code_heap.cpp index 77ae5180c8..f3d36527b5 100755 --- a/vm/code_heap.cpp +++ b/vm/code_heap.cpp @@ -203,8 +203,8 @@ code_heap_room factor_vm::code_room() room.size = code->allocator->size; room.occupied_space = code->allocator->occupied_space(); room.total_free = code->allocator->free_space(); - room.contiguous_free = code->allocator->free_blocks.largest_free_block(); - room.free_block_count = code->allocator->free_blocks.free_block_count; + room.contiguous_free = code->allocator->largest_free_block(); + room.free_block_count = code->allocator->free_block_count(); return room; } diff --git a/vm/data_heap.cpp b/vm/data_heap.cpp index b38674fe1f..5ae11fd486 100755 --- a/vm/data_heap.cpp +++ b/vm/data_heap.cpp @@ -9,7 +9,10 @@ void factor_vm::init_card_decks() decks_offset = (cell)data->decks - addr_to_deck(data->start); } -data_heap::data_heap(cell young_size_, cell aging_size_, cell tenured_size_) +data_heap::data_heap(cell young_size_, + cell aging_size_, + cell tenured_size_, + cell promotion_threshold_) { young_size_ = align(young_size_,deck_size); aging_size_ = align(aging_size_,deck_size); @@ -18,15 +21,12 @@ data_heap::data_heap(cell young_size_, cell aging_size_, cell tenured_size_) young_size = young_size_; aging_size = aging_size_; tenured_size = tenured_size_; + promotion_threshold = promotion_threshold_; - cell total_size = young_size + 2 * aging_size + tenured_size; - - total_size += deck_size; - + cell total_size = young_size + 2 * aging_size + tenured_size + deck_size; seg = new segment(total_size,false); cell cards_size = addr_to_card(total_size); - cards = new card[cards_size]; cards_end = cards + cards_size; memset(cards,0,cards_size); @@ -62,7 +62,10 @@ data_heap::~data_heap() data_heap *data_heap::grow(cell requested_bytes) { cell new_tenured_size = (tenured_size * 2) + requested_bytes; - return new data_heap(young_size,aging_size,new_tenured_size); + return new data_heap(young_size, + aging_size, + new_tenured_size, + promotion_threshold * 1.25); } template void data_heap::clear_cards(Generation *gen) @@ -105,9 +108,9 @@ void factor_vm::set_data_heap(data_heap *data_) init_card_decks(); } -void factor_vm::init_data_heap(cell young_size, cell aging_size, cell tenured_size) +void factor_vm::init_data_heap(cell young_size, cell aging_size, cell tenured_size, cell promotion_threshold) { - set_data_heap(new data_heap(young_size,aging_size,tenured_size)); + set_data_heap(new data_heap(young_size,aging_size,tenured_size,promotion_threshold)); } /* Size of the object pointed to by a tagged pointer */ @@ -214,8 +217,8 @@ data_heap_room factor_vm::data_room() room.tenured_size = data->tenured->size; room.tenured_occupied = data->tenured->occupied_space(); room.tenured_total_free = data->tenured->free_space(); - room.tenured_contiguous_free = data->tenured->free_blocks.largest_free_block(); - room.tenured_free_block_count = data->tenured->free_blocks.free_block_count; + room.tenured_contiguous_free = data->tenured->largest_free_block(); + room.tenured_free_block_count = data->tenured->free_block_count(); room.cards = data->cards_end - data->cards; room.decks = data->decks_end - data->decks; room.mark_stack = data->tenured->mark_stack.capacity(); diff --git a/vm/data_heap.hpp b/vm/data_heap.hpp index a434024873..0cb9b9254b 100755 --- a/vm/data_heap.hpp +++ b/vm/data_heap.hpp @@ -8,6 +8,8 @@ struct data_heap { cell aging_size; cell tenured_size; + cell promotion_threshold; + segment *seg; nursery_space *nursery; @@ -21,7 +23,7 @@ struct data_heap { card_deck *decks; card_deck *decks_end; - explicit data_heap(cell young_size, cell aging_size, cell tenured_size); + explicit data_heap(cell young_size, cell aging_size, cell tenured_size, cell promotion_threshold); ~data_heap(); data_heap *grow(cell requested_size); template void clear_cards(Generation *gen); diff --git a/vm/factor.cpp b/vm/factor.cpp index 510900a036..50a3070922 100755 --- a/vm/factor.cpp +++ b/vm/factor.cpp @@ -15,25 +15,14 @@ void factor_vm::default_parameters(vm_parameters *p) { p->image_path = NULL; - /* We make a wild guess here that if we're running on ARM, we don't - have a lot of memory. */ -#ifdef FACTOR_ARM - p->ds_size = 8 * sizeof(cell); - p->rs_size = 8 * sizeof(cell); - - p->code_size = 4; - p->young_size = 1; - p->aging_size = 1; - p->tenured_size = 6; -#else p->ds_size = 32 * sizeof(cell); p->rs_size = 32 * sizeof(cell); p->code_size = 8 * sizeof(cell); p->young_size = sizeof(cell) / 4; p->aging_size = sizeof(cell) / 2; - p->tenured_size = 16 * sizeof(cell); -#endif + p->tenured_size = 24 * sizeof(cell); + p->promotion_threshold = 8 * sizeof(cell); p->max_pic_size = 3; @@ -81,6 +70,7 @@ void factor_vm::init_parameters_from_args(vm_parameters *p, int argc, vm_char ** else if(factor_arg(arg,STRING_LITERAL("-young=%d"),&p->young_size)); else if(factor_arg(arg,STRING_LITERAL("-aging=%d"),&p->aging_size)); else if(factor_arg(arg,STRING_LITERAL("-tenured=%d"),&p->tenured_size)); + else if(factor_arg(arg,STRING_LITERAL("-promote=%d"),&p->promotion_threshold)); else if(factor_arg(arg,STRING_LITERAL("-codeheap=%d"),&p->code_size)); else if(factor_arg(arg,STRING_LITERAL("-pic=%d"),&p->max_pic_size)); else if(factor_arg(arg,STRING_LITERAL("-callbacks=%d"),&p->callback_size)); @@ -114,6 +104,7 @@ void factor_vm::init_factor(vm_parameters *p) p->young_size <<= 20; p->aging_size <<= 20; p->tenured_size <<= 20; + p->promotion_threshold <<= 20; p->code_size <<= 20; /* Disable GC during init as a sanity check */ diff --git a/vm/free_list.cpp b/vm/free_list.cpp index 644f62d870..cde6a2d781 100644 --- a/vm/free_list.cpp +++ b/vm/free_list.cpp @@ -46,6 +46,10 @@ free_heap_block *free_list::find_free_block(cell size) { free_heap_block *block = blocks.back(); blocks.pop_back(); + + free_block_count--; + free_space -= block->size(); + return block; } } diff --git a/vm/free_list_allocator.hpp b/vm/free_list_allocator.hpp index dad54b644d..6515078100 100644 --- a/vm/free_list_allocator.hpp +++ b/vm/free_list_allocator.hpp @@ -5,6 +5,7 @@ template struct free_list_allocator { cell size; cell start; cell end; + cell high_water_mark; free_list free_blocks; mark_bits state; @@ -20,6 +21,8 @@ template struct free_list_allocator { void free(Block *block); cell occupied_space(); cell free_space(); + cell largest_free_block(); + cell free_block_count(); void sweep(); template void sweep(Iterator &iter); template void compact(Iterator &iter, Sizer &sizer); @@ -29,7 +32,10 @@ template struct free_list_allocator { template free_list_allocator::free_list_allocator(cell size_, cell start_) : - size(size_), start(start_), end(start_ + size_), state(mark_bits(size_,start_)) + size(size_), + start(start_), + end(start_ + size_), + state(mark_bits(size_,start_)) { initial_free_list(0); } @@ -37,6 +43,7 @@ free_list_allocator::free_list_allocator(cell size_, cell start_) : template void free_list_allocator::initial_free_list(cell occupied) { free_blocks.initial_free_list(start,end,occupied); + high_water_mark = free_blocks.free_space; } template bool free_list_allocator::contains_p(Block *block) @@ -109,6 +116,16 @@ template cell free_list_allocator::occupied_space() return size - free_blocks.free_space; } +template cell free_list_allocator::largest_free_block() +{ + return free_blocks.largest_free_block(); +} + +template cell free_list_allocator::free_block_count() +{ + return free_blocks.free_block_count; +} + template void free_list_allocator::sweep() { @@ -158,6 +175,8 @@ void free_list_allocator::sweep() if(prev && prev->free_p()) free_blocks.add_to_free_list((free_heap_block *)prev); + + high_water_mark = free_blocks.free_space; } template @@ -211,6 +230,8 @@ void free_list_allocator::sweep(Iterator &iter) if(prev && prev->free_p()) free_blocks.add_to_free_list((free_heap_block *)prev); + + high_water_mark = free_blocks.free_space; } template struct heap_compactor { diff --git a/vm/gc.cpp b/vm/gc.cpp index cce8298921..6b1536bcde 100755 --- a/vm/gc.cpp +++ b/vm/gc.cpp @@ -152,6 +152,19 @@ void factor_vm::gc(gc_op op, cell requested_bytes, bool trace_contexts_p) start_gc_again(); } + if(current_gc->op == collect_aging_op + || current_gc->op == collect_to_tenured_op + || current_gc->op == collect_full_op) + { + if(data->tenured->largest_free_block() <= data->nursery->size + data->aging->size) + current_gc->op = collect_compact_op; + + if(data->tenured->high_water_mark - data->tenured->free_space() >= data->promotion_threshold) + current_gc->op = collect_full_op; + } + + current_gc->event->op = current_gc->op; + switch(current_gc->op) { case collect_nursery_op: @@ -316,7 +329,7 @@ void factor_vm::primitive_disable_gc_events() { if(gc_events) { - byte_array *data = byte_array_from_values(&gc_events->first(),gc_events->size()); + byte_array *data = byte_array_from_values(&gc_events->front(),gc_events->size()); dpush(tag(data)); delete gc_events; diff --git a/vm/image.cpp b/vm/image.cpp index fce730df5a..152a39bdf3 100755 --- a/vm/image.cpp +++ b/vm/image.cpp @@ -20,7 +20,8 @@ void factor_vm::load_data_heap(FILE *file, image_header *h, vm_parameters *p) init_data_heap(p->young_size, p->aging_size, - p->tenured_size); + p->tenured_size, + p->promotion_threshold); fixnum bytes_read = fread((void*)data->tenured->start,1,h->data_size,file); diff --git a/vm/image.hpp b/vm/image.hpp index cca0e9e378..753d11987d 100755 --- a/vm/image.hpp +++ b/vm/image.hpp @@ -33,6 +33,7 @@ struct vm_parameters { const vm_char *executable_path; cell ds_size, rs_size; cell young_size, aging_size, tenured_size; + cell promotion_threshold; cell code_size; bool fep; bool console; diff --git a/vm/vm.hpp b/vm/vm.hpp index 0844b47e51..7860d4f237 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -218,7 +218,7 @@ struct factor_vm //data heap void init_card_decks(); void set_data_heap(data_heap *data_); - void init_data_heap(cell young_size, cell aging_size, cell tenured_size); + void init_data_heap(cell young_size, cell aging_size, cell tenured_size, cell promotion_threshold); void primitive_size(); data_heap_room data_room(); void primitive_data_room(); From a3522554f3583b3a7c50e1fc744b3a33c2a0f282 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Fri, 30 Oct 2009 02:45:16 -0500 Subject: [PATCH 42/74] tools.memory: add gc-stats. for viewing event statistics --- basis/tools/memory/memory.factor | 112 ++++++++++++++++++++++++++----- 1 file changed, 96 insertions(+), 16 deletions(-) diff --git a/basis/tools/memory/memory.factor b/basis/tools/memory/memory.factor index 6d0c4b7c86..479c16989c 100644 --- a/basis/tools/memory/memory.factor +++ b/basis/tools/memory/memory.factor @@ -1,24 +1,26 @@ ! Copyright (C) 2005, 2009 Slava Pestov. ! See http://factorcode.org/license.txt for BSD license. -USING: kernel sequences arrays generic assocs io math -namespaces parser prettyprint strings io.styles words -system sorting splitting grouping math.parser classes memory -combinators fry vm specialized-arrays accessors continuations -classes.struct generalizations ; +USING: accessors arrays assocs classes classes.struct +combinators combinators.smart continuations fry generalizations +generic grouping io io.styles kernel make math math.parser +math.statistics memory namespaces parser prettyprint sequences +sorting specialized-arrays splitting strings system vm words ; SPECIALIZED-ARRAY: gc-event IN: tools.memory string - reverse 3 group "," join reverse ; + dup 0 < [ neg commas "-" prepend ] [ + number>string + reverse 3 group "," join reverse + ] if ; : kilobytes ( n -- str ) 1024 /i commas " KB" append ; : micros>string ( n -- str ) - commas " microseconds" append ; + commas " µs" append ; : fancy-table. ( obj alist -- ) [ [ nip first ] [ second call( obj -- str ) ] 2bi 2array ] with map @@ -102,19 +104,88 @@ PRIVATE> enable-gc-events [ ] [ disable-gc-events drop ] cleanup disable-gc-events byte-array>gc-event-array ; inline +> ] [ aging>> ] [ tenured>> ] tri [ occupied>> ] tri@ ] + [ occupied>> ] + bi* + ] sum-outputs ; + +: space-occupied-before ( event -- bytes ) + [ data-heap-before>> ] [ code-heap-before>> ] bi (space-occupied) ; + +: space-occupied-after ( event -- bytes ) + [ data-heap-after>> ] [ code-heap-after>> ] bi (space-occupied) ; + : space-reclaimed ( event -- bytes ) - [ data-heap-before>> ] [ data-heap-after>> ] bi - [ [ nursery>> ] [ aging>> ] [ tenured>> ] tri [ occupied>> ] tri@ + + ] bi@ - ; + [ space-occupied-before ] [ space-occupied-after ] bi - ; + +TUPLE: gc-stats collections times ; + +: ( -- stats ) + gc-stats new + 0 >>collections + V{ } clone >>times ; inline + +: compute-gc-stats ( events -- stats ) + V{ } clone [ + '[ + dup op>> _ [ drop ] cache + [ 1 + ] change-collections + [ total-time>> ] dip times>> push + ] each + ] keep sort-keys ; + +: gc-stats-table-row ( pair -- row ) + [ + [ first gc-op-string ] [ + second + [ collections>> ] + [ + times>> { + [ sum micros>string ] + [ mean >integer micros>string ] + [ median >integer micros>string ] + [ infimum micros>string ] + [ supremum micros>string ] + } cleave + ] bi + ] bi + ] output>array ; + +: gc-stats-table ( stats -- table ) + [ gc-stats-table-row ] map + { "" "Number" "Total" "Mean" "Median" "Min" "Max" } prefix ; + +: heap-sizes ( events -- seq ) + [ + [ + [ [ start-time>> ] [ space-occupied-before ] bi 2array , ] + [ [ [ start-time>> ] [ total-time>> ] bi + ] [ space-occupied-after ] bi 2array , ] + bi + ] each + ] { } make ; + +: aggregate-stats-table ( stats -- table ) + [ { "Total collections:" "Total GC time:" } ] dip + values + [ [ collections>> ] sigma ] + [ [ times>> sum ] sigma micros>string ] + bi 2array zip ; + +PRIVATE> : gc-event. ( event -- ) { @@ -122,3 +193,12 @@ PRIVATE> { "Total time:" [ total-time>> micros>string ] } { "Space reclaimed:" [ space-reclaimed kilobytes ] } } fancy-table. ; + +: gc-stats. ( events -- ) + compute-gc-stats + [ gc-stats-table simple-table. nl ] + [ aggregate-stats-table simple-table. ] + bi ; + +: heap-sizes. ( events -- ) + heap-sizes simple-table. ; From f102d1073eb0ed0101e51e39a2b73d043cbe0f2a Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Fri, 30 Oct 2009 03:15:07 -0500 Subject: [PATCH 43/74] tools.memory: update for language changes --- basis/tools/memory/memory.factor | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/basis/tools/memory/memory.factor b/basis/tools/memory/memory.factor index 479c16989c..9b1d6284bb 100644 --- a/basis/tools/memory/memory.factor +++ b/basis/tools/memory/memory.factor @@ -181,8 +181,8 @@ TUPLE: gc-stats collections times ; : aggregate-stats-table ( stats -- table ) [ { "Total collections:" "Total GC time:" } ] dip values - [ [ collections>> ] sigma ] - [ [ times>> sum ] sigma micros>string ] + [ [ collections>> ] map-sum ] + [ [ times>> sum ] map-sum micros>string ] bi 2array zip ; PRIVATE> From 6f7607eed70a09731f952243868f65791e93dd9c Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Fri, 30 Oct 2009 03:15:50 -0500 Subject: [PATCH 44/74] vm: clean up card unmarking --- vm/aging_collector.cpp | 2 +- vm/collector.hpp | 15 +++++++++++++++ vm/copying_collector.hpp | 10 ---------- vm/nursery_collector.cpp | 2 +- vm/to_tenured_collector.cpp | 2 +- 5 files changed, 18 insertions(+), 13 deletions(-) diff --git a/vm/aging_collector.cpp b/vm/aging_collector.cpp index e39e7e7d6f..1979769b81 100644 --- a/vm/aging_collector.cpp +++ b/vm/aging_collector.cpp @@ -25,7 +25,7 @@ void factor_vm::collect_aging() current_gc->event->started_code_scan(); collector.trace_cards(data->tenured, card_points_to_aging, - simple_unmarker(card_mark_mask)); + full_unmarker()); current_gc->event->ended_card_scan(collector.cards_scanned,collector.decks_scanned); collector.tenure_reachable_objects(); diff --git a/vm/collector.hpp b/vm/collector.hpp index 4535f98dd7..31d830256d 100644 --- a/vm/collector.hpp +++ b/vm/collector.hpp @@ -71,6 +71,21 @@ inline static slot_visitor > make_c collector_workhorse(parent,target,policy)); } +struct dummy_unmarker { + void operator()(card *ptr) {} +}; + +struct simple_unmarker { + card unmask; + explicit simple_unmarker(card unmask_) : unmask(unmask_) {} + void operator()(card *ptr) { *ptr &= ~unmask; } +}; + +struct full_unmarker { + explicit full_unmarker() {} + void operator()(card *ptr) { *ptr = 0; } +}; + template struct collector { factor_vm *parent; data_heap *data; diff --git a/vm/copying_collector.hpp b/vm/copying_collector.hpp index 2264b600dd..a21147ff0c 100644 --- a/vm/copying_collector.hpp +++ b/vm/copying_collector.hpp @@ -1,16 +1,6 @@ namespace factor { -struct dummy_unmarker { - void operator()(card *ptr) {} -}; - -struct simple_unmarker { - card unmask; - explicit simple_unmarker(card unmask_) : unmask(unmask_) {} - void operator()(card *ptr) { *ptr &= ~unmask; } -}; - template struct copying_collector : collector { cell scan; diff --git a/vm/nursery_collector.cpp b/vm/nursery_collector.cpp index 240fac60d9..155da243d4 100644 --- a/vm/nursery_collector.cpp +++ b/vm/nursery_collector.cpp @@ -24,7 +24,7 @@ void factor_vm::collect_nursery() simple_unmarker(card_points_to_nursery)); collector.trace_cards(data->aging, card_points_to_nursery, - simple_unmarker(card_mark_mask)); + full_unmarker()); current_gc->event->ended_card_scan(collector.cards_scanned,collector.decks_scanned); current_gc->event->started_code_scan(); diff --git a/vm/to_tenured_collector.cpp b/vm/to_tenured_collector.cpp index a39d92d869..6067bf1bf4 100644 --- a/vm/to_tenured_collector.cpp +++ b/vm/to_tenured_collector.cpp @@ -33,7 +33,7 @@ void factor_vm::collect_to_tenured() current_gc->event->started_card_scan(); collector.trace_cards(data->tenured, card_points_to_aging, - simple_unmarker(card_mark_mask)); + full_unmarker()); current_gc->event->ended_card_scan(collector.cards_scanned,collector.decks_scanned); current_gc->event->started_code_scan(); From 7cbaf3e0a34c8253e90673b1f28495429d494f51 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Sat, 31 Oct 2009 21:06:34 -0500 Subject: [PATCH 45/74] vm: don't do a data compaction when growing the data heap, speeds up data heap grow by a bit --- vm/code_block_visitor.hpp | 1 + vm/compaction.cpp | 48 +++++++++++++++++++++++++++++++--- vm/data_heap.cpp | 25 ++++++++++-------- vm/debug.cpp | 55 +++++++++++++++++++++++---------------- vm/full_collector.cpp | 2 +- vm/gc.cpp | 20 ++++++++------ vm/vm.hpp | 11 +++++++- 7 files changed, 115 insertions(+), 47 deletions(-) diff --git a/vm/code_block_visitor.hpp b/vm/code_block_visitor.hpp index 09bbecc757..33e889c940 100644 --- a/vm/code_block_visitor.hpp +++ b/vm/code_block_visitor.hpp @@ -39,6 +39,7 @@ template struct code_block_visitor { explicit code_block_visitor(factor_vm *parent_, Visitor visitor_) : parent(parent_), visitor(visitor_) {} + void visit_object_code_block(object *obj) { switch(obj->h.hi_tag()) diff --git a/vm/compaction.cpp b/vm/compaction.cpp index bda8035858..dee922d2f4 100644 --- a/vm/compaction.cpp +++ b/vm/compaction.cpp @@ -97,11 +97,11 @@ struct object_compaction_updater { } }; -struct code_block_compaction_updater { +template struct code_block_compaction_updater { factor_vm *parent; - slot_visitor slot_forwarder; + SlotForwarder slot_forwarder; - explicit code_block_compaction_updater(factor_vm *parent_, slot_visitor slot_forwarder_) : + explicit code_block_compaction_updater(factor_vm *parent_, SlotForwarder slot_forwarder_) : parent(parent_), slot_forwarder(slot_forwarder_) {} void operator()(code_block *old_address, code_block *new_address, cell size) @@ -112,6 +112,7 @@ struct code_block_compaction_updater { } }; +/* Compact data and code heaps */ void factor_vm::collect_compact_impl(bool trace_contexts_p) { current_gc->event->started_compaction(); @@ -139,7 +140,7 @@ void factor_vm::collect_compact_impl(bool trace_contexts_p) /* Slide everything in the code heap up, and update data and code heap pointers inside code blocks. */ - code_block_compaction_updater code_block_updater(this,slot_forwarder); + code_block_compaction_updater > code_block_updater(this,slot_forwarder); standard_sizer code_block_sizer; code->allocator->compact(code_block_updater,code_block_sizer); @@ -154,4 +155,43 @@ void factor_vm::collect_compact_impl(bool trace_contexts_p) current_gc->event->ended_compaction(); } +struct object_code_block_updater { + code_block_visitor *forwarder; + + explicit object_code_block_updater(code_block_visitor *forwarder_) : + forwarder(forwarder_) {} + + void operator()(cell obj) + { + forwarder->visit_object_code_block(tagged(obj).untagged()); + } +}; + +struct dummy_slot_forwarder { + void visit_literal_references(code_block *compiled) {} +}; + +/* Compact just the code heap */ +void factor_vm::collect_compact_code_impl() +{ + mark_bits *code_forwarding_map = &code->allocator->state; + + /* Figure out where blocks are going to go */ + code_forwarding_map->compute_forwarding(); + + /* Update root pointers */ + code_block_visitor code_forwarder(this,code_block_forwarder(code_forwarding_map)); + + /* Slide everything in the code heap up, and update code heap + pointers inside code blocks. */ + dummy_slot_forwarder slot_forwarder; + code_block_compaction_updater code_block_updater(this,slot_forwarder); + standard_sizer code_block_sizer; + code->allocator->compact(code_block_updater,code_block_sizer); + + /* Update code heap references in data heap */ + object_code_block_updater updater(&code_forwarder); + each_object(updater); +} + } diff --git a/vm/data_heap.cpp b/vm/data_heap.cpp index 5ae11fd486..2ddc520668 100755 --- a/vm/data_heap.cpp +++ b/vm/data_heap.cpp @@ -276,25 +276,28 @@ void factor_vm::primitive_end_scan() gc_off = false; } -template void factor_vm::each_object(Iterator &iterator) -{ - begin_scan(); - cell obj; - while(to_boolean(obj = next_object())) - iterator(tagged(obj)); - end_scan(); -} - struct word_counter { cell count; + explicit word_counter() : count(0) {} - void operator()(tagged obj) { if(obj.type_p(WORD_TYPE)) count++; } + + void operator()(cell obj) + { + if(tagged(obj).type_p(WORD_TYPE)) + count++; + } }; struct word_accumulator { growable_array words; + explicit word_accumulator(int count,factor_vm *vm) : words(vm,count) {} - void operator()(tagged obj) { if(obj.type_p(WORD_TYPE)) words.add(obj.value()); } + + void operator()(cell obj) + { + if(tagged(obj).type_p(WORD_TYPE)) + words.add(obj); + } }; cell factor_vm::find_all_words() diff --git a/vm/debug.cpp b/vm/debug.cpp index 0598d164e6..fee3e6a257 100755 --- a/vm/debug.cpp +++ b/vm/debug.cpp @@ -234,31 +234,37 @@ void factor_vm::dump_generations() std::cout << "size=" << (cell)(data->cards_end - data->cards) << std::endl; } -void factor_vm::dump_objects(cell type) -{ - primitive_full_gc(); - begin_scan(); +struct object_dumper { + factor_vm *parent; + cell type; - cell obj; - while(to_boolean(obj = next_object())) + explicit object_dumper(factor_vm *parent_, cell type_) : + parent(parent_), type(type_) {} + + void operator()(cell obj) { if(type == TYPE_COUNT || tagged(obj).type_p(type)) { std::cout << padded_address(obj) << " "; - print_nested_obj(obj,2); + parent->print_nested_obj(obj,2); std::cout << std::endl; } } +}; - end_scan(); +void factor_vm::dump_objects(cell type) +{ + primitive_full_gc(); + object_dumper dumper(this,type); + each_object(dumper); } -struct data_references_finder { +struct data_reference_slot_visitor { cell look_for, obj; factor_vm *parent; - explicit data_references_finder(cell look_for_, cell obj_, factor_vm *parent_) - : look_for(look_for_), obj(obj_), parent(parent_) { } + explicit data_reference_slot_visitor(cell look_for_, cell obj_, factor_vm *parent_) : + look_for(look_for_), obj(obj_), parent(parent_) { } void operator()(cell *scan) { @@ -271,19 +277,24 @@ struct data_references_finder { } }; +struct data_reference_object_visitor { + cell look_for; + factor_vm *parent; + + explicit data_reference_object_visitor(cell look_for_, factor_vm *parent_) : + look_for(look_for_), parent(parent_) {} + + void operator()(cell obj) + { + data_reference_slot_visitor visitor(look_for,obj,parent); + parent->do_slots(UNTAG(obj),visitor); + } +}; + void factor_vm::find_data_references(cell look_for) { - begin_scan(); - - cell obj; - - while(to_boolean(obj = next_object())) - { - data_references_finder finder(look_for,obj,this); - do_slots(UNTAG(obj),finder); - } - - end_scan(); + data_reference_object_visitor visitor(look_for,this); + each_object(visitor); } struct code_block_printer { diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index 6167c73ab5..397b210c6d 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -88,7 +88,7 @@ void factor_vm::collect_growing_heap(cell requested_bytes, bool trace_contexts_p data_heap *old = data; set_data_heap(data->grow(requested_bytes)); collect_mark_impl(trace_contexts_p); - collect_compact_impl(trace_contexts_p); + collect_compact_code_impl(); delete old; } diff --git a/vm/gc.cpp b/vm/gc.cpp index c382f3df54..f358a9fe4a 100755 --- a/vm/gc.cpp +++ b/vm/gc.cpp @@ -152,15 +152,13 @@ void factor_vm::gc(gc_op op, cell requested_bytes, bool trace_contexts_p) start_gc_again(); } - if(current_gc->op == collect_aging_op - || current_gc->op == collect_to_tenured_op - || current_gc->op == collect_full_op) + if(current_gc->op == collect_aging_op || current_gc->op == collect_to_tenured_op) { - if(data->tenured->largest_free_block() <= data->nursery->size + data->aging->size) - current_gc->op = collect_compact_op; - - if(data->tenured->high_water_mark - data->tenured->free_space() >= data->promotion_threshold) + if(data->tenured->free_space() <= data->nursery->size + data->aging->size) + { + printf("upgrade\n"); current_gc->op = collect_full_op; + } } current_gc->event->op = current_gc->op; @@ -179,7 +177,13 @@ void factor_vm::gc(gc_op op, cell requested_bytes, bool trace_contexts_p) case collect_full_op: collect_mark_impl(trace_contexts_p); collect_sweep_impl(); - update_code_heap_words_and_literals(); + if(data->tenured->largest_free_block() <= data->nursery->size + data->aging->size) + { + printf("forced compaction\n"); + collect_compact_impl(trace_contexts_p); + } + else + update_code_heap_words_and_literals(); break; case collect_compact_op: collect_mark_impl(trace_contexts_p); diff --git a/vm/vm.hpp b/vm/vm.hpp index b5bbca525b..f04030761b 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -228,10 +228,18 @@ struct factor_vm cell next_object(); void primitive_next_object(); void primitive_end_scan(); - template void each_object(Iterator &iterator); cell find_all_words(); cell object_size(cell tagged); + template inline void each_object(Iterator &iterator) + { + begin_scan(); + cell obj; + while(to_boolean(obj = next_object())) + iterator(obj); + end_scan(); + } + /* the write barrier must be called any time we are potentially storing a pointer from an older generation to a younger one */ inline void write_barrier(cell *slot_ptr) @@ -250,6 +258,7 @@ struct factor_vm void collect_mark_impl(bool trace_contexts_p); void collect_sweep_impl(); void collect_compact_impl(bool trace_contexts_p); + void collect_compact_code_impl(); void collect_growing_heap(cell requested_bytes, bool trace_contexts_p); void gc(gc_op op, cell requested_bytes, bool trace_contexts_p); void primitive_minor_gc(); From 5ae40e26ee7f8f2a1a9c58c0b8c95f448001321e Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Sun, 1 Nov 2009 03:47:09 -0600 Subject: [PATCH 46/74] vm: remove promotion strategy, clean up compaction code --- vm/collector.hpp | 2 +- vm/compaction.cpp | 49 ++++++++-------------- vm/data_heap.cpp | 11 ++--- vm/data_heap.hpp | 4 +- vm/factor.cpp | 3 -- vm/free_list_allocator.hpp | 59 +++++++------------------- vm/image.cpp | 3 +- vm/image.hpp | 1 - vm/mark_bits.hpp | 84 ++++++++++++++++++++++++++++++++------ vm/slot_visitor.hpp | 4 +- vm/vm.hpp | 6 +-- 11 files changed, 117 insertions(+), 109 deletions(-) diff --git a/vm/collector.hpp b/vm/collector.hpp index 31d830256d..a52d5f97b1 100644 --- a/vm/collector.hpp +++ b/vm/collector.hpp @@ -39,7 +39,7 @@ template struct collector_workhorse return newpointer; } - object *visit_object(object *obj) + object *operator()(object *obj) { if(!policy.should_copy_p(obj)) { diff --git a/vm/compaction.cpp b/vm/compaction.cpp index dee922d2f4..eb11ec9f3d 100644 --- a/vm/compaction.cpp +++ b/vm/compaction.cpp @@ -2,27 +2,15 @@ namespace factor { -struct object_slot_forwarder { - mark_bits *forwarding_map; +template struct forwarder { + mark_bits *forwarding_map; - explicit object_slot_forwarder(mark_bits *forwarding_map_) : + explicit forwarder(mark_bits *forwarding_map_) : forwarding_map(forwarding_map_) {} - object *visit_object(object *obj) + Block *operator()(Block *block) { - return forwarding_map->forward_block(obj); - } -}; - -struct code_block_forwarder { - mark_bits *forwarding_map; - - explicit code_block_forwarder(mark_bits *forwarding_map_) : - forwarding_map(forwarding_map_) {} - - code_block *operator()(code_block *compiled) - { - return forwarding_map->forward_block(compiled); + return forwarding_map->forward_block(block); } }; @@ -56,7 +44,7 @@ struct compaction_sizer { cell operator()(object *obj) { if(!forwarding_map->marked_p(obj)) - return forwarding_map->unmarked_space_starting_at(obj); + return forwarding_map->unmarked_block_size(obj); else if(obj->h.hi_tag() == TUPLE_TYPE) return align(tuple_size_with_forwarding(forwarding_map,obj),data_alignment); else @@ -66,14 +54,14 @@ struct compaction_sizer { struct object_compaction_updater { factor_vm *parent; - slot_visitor slot_forwarder; - code_block_visitor code_forwarder; + slot_visitor > slot_forwarder; + code_block_visitor > code_forwarder; mark_bits *data_forwarding_map; object_start_map *starts; explicit object_compaction_updater(factor_vm *parent_, - slot_visitor slot_forwarder_, - code_block_visitor code_forwarder_, + slot_visitor > slot_forwarder_, + code_block_visitor > code_forwarder_, mark_bits *data_forwarding_map_) : parent(parent_), slot_forwarder(slot_forwarder_), @@ -125,9 +113,8 @@ void factor_vm::collect_compact_impl(bool trace_contexts_p) data_forwarding_map->compute_forwarding(); code_forwarding_map->compute_forwarding(); - /* Update root pointers */ - slot_visitor slot_forwarder(this,object_slot_forwarder(data_forwarding_map)); - code_block_visitor code_forwarder(this,code_block_forwarder(code_forwarding_map)); + slot_visitor > slot_forwarder(this,forwarder(data_forwarding_map)); + code_block_visitor > code_forwarder(this,forwarder(code_forwarding_map)); /* Object start offsets get recomputed by the object_compaction_updater */ data->tenured->starts.clear_object_start_offsets(); @@ -140,7 +127,7 @@ void factor_vm::collect_compact_impl(bool trace_contexts_p) /* Slide everything in the code heap up, and update data and code heap pointers inside code blocks. */ - code_block_compaction_updater > code_block_updater(this,slot_forwarder); + code_block_compaction_updater > > code_block_updater(this,slot_forwarder); standard_sizer code_block_sizer; code->allocator->compact(code_block_updater,code_block_sizer); @@ -156,14 +143,14 @@ void factor_vm::collect_compact_impl(bool trace_contexts_p) } struct object_code_block_updater { - code_block_visitor *forwarder; + code_block_visitor > *visitor; - explicit object_code_block_updater(code_block_visitor *forwarder_) : - forwarder(forwarder_) {} + explicit object_code_block_updater(code_block_visitor > *visitor_) : + visitor(visitor_) {} void operator()(cell obj) { - forwarder->visit_object_code_block(tagged(obj).untagged()); + visitor->visit_object_code_block(tagged(obj).untagged()); } }; @@ -180,7 +167,7 @@ void factor_vm::collect_compact_code_impl() code_forwarding_map->compute_forwarding(); /* Update root pointers */ - code_block_visitor code_forwarder(this,code_block_forwarder(code_forwarding_map)); + code_block_visitor > code_forwarder(this,forwarder(code_forwarding_map)); /* Slide everything in the code heap up, and update code heap pointers inside code blocks. */ diff --git a/vm/data_heap.cpp b/vm/data_heap.cpp index 2ddc520668..21008af5cb 100755 --- a/vm/data_heap.cpp +++ b/vm/data_heap.cpp @@ -11,8 +11,7 @@ void factor_vm::init_card_decks() data_heap::data_heap(cell young_size_, cell aging_size_, - cell tenured_size_, - cell promotion_threshold_) + cell tenured_size_) { young_size_ = align(young_size_,deck_size); aging_size_ = align(aging_size_,deck_size); @@ -21,7 +20,6 @@ data_heap::data_heap(cell young_size_, young_size = young_size_; aging_size = aging_size_; tenured_size = tenured_size_; - promotion_threshold = promotion_threshold_; cell total_size = young_size + 2 * aging_size + tenured_size + deck_size; seg = new segment(total_size,false); @@ -64,8 +62,7 @@ data_heap *data_heap::grow(cell requested_bytes) cell new_tenured_size = (tenured_size * 2) + requested_bytes; return new data_heap(young_size, aging_size, - new_tenured_size, - promotion_threshold * 1.25); + new_tenured_size); } template void data_heap::clear_cards(Generation *gen) @@ -108,9 +105,9 @@ void factor_vm::set_data_heap(data_heap *data_) init_card_decks(); } -void factor_vm::init_data_heap(cell young_size, cell aging_size, cell tenured_size, cell promotion_threshold) +void factor_vm::init_data_heap(cell young_size, cell aging_size, cell tenured_size) { - set_data_heap(new data_heap(young_size,aging_size,tenured_size,promotion_threshold)); + set_data_heap(new data_heap(young_size,aging_size,tenured_size)); } /* Size of the object pointed to by a tagged pointer */ diff --git a/vm/data_heap.hpp b/vm/data_heap.hpp index 0cb9b9254b..a434024873 100755 --- a/vm/data_heap.hpp +++ b/vm/data_heap.hpp @@ -8,8 +8,6 @@ struct data_heap { cell aging_size; cell tenured_size; - cell promotion_threshold; - segment *seg; nursery_space *nursery; @@ -23,7 +21,7 @@ struct data_heap { card_deck *decks; card_deck *decks_end; - explicit data_heap(cell young_size, cell aging_size, cell tenured_size, cell promotion_threshold); + explicit data_heap(cell young_size, cell aging_size, cell tenured_size); ~data_heap(); data_heap *grow(cell requested_size); template void clear_cards(Generation *gen); diff --git a/vm/factor.cpp b/vm/factor.cpp index 50a3070922..d382745da8 100755 --- a/vm/factor.cpp +++ b/vm/factor.cpp @@ -22,7 +22,6 @@ void factor_vm::default_parameters(vm_parameters *p) p->young_size = sizeof(cell) / 4; p->aging_size = sizeof(cell) / 2; p->tenured_size = 24 * sizeof(cell); - p->promotion_threshold = 8 * sizeof(cell); p->max_pic_size = 3; @@ -70,7 +69,6 @@ void factor_vm::init_parameters_from_args(vm_parameters *p, int argc, vm_char ** else if(factor_arg(arg,STRING_LITERAL("-young=%d"),&p->young_size)); else if(factor_arg(arg,STRING_LITERAL("-aging=%d"),&p->aging_size)); else if(factor_arg(arg,STRING_LITERAL("-tenured=%d"),&p->tenured_size)); - else if(factor_arg(arg,STRING_LITERAL("-promote=%d"),&p->promotion_threshold)); else if(factor_arg(arg,STRING_LITERAL("-codeheap=%d"),&p->code_size)); else if(factor_arg(arg,STRING_LITERAL("-pic=%d"),&p->max_pic_size)); else if(factor_arg(arg,STRING_LITERAL("-callbacks=%d"),&p->callback_size)); @@ -104,7 +102,6 @@ void factor_vm::init_factor(vm_parameters *p) p->young_size <<= 20; p->aging_size <<= 20; p->tenured_size <<= 20; - p->promotion_threshold <<= 20; p->code_size <<= 20; /* Disable GC during init as a sanity check */ diff --git a/vm/free_list_allocator.hpp b/vm/free_list_allocator.hpp index 6515078100..a4801daa72 100644 --- a/vm/free_list_allocator.hpp +++ b/vm/free_list_allocator.hpp @@ -5,7 +5,6 @@ template struct free_list_allocator { cell size; cell start; cell end; - cell high_water_mark; free_list free_blocks; mark_bits state; @@ -43,7 +42,6 @@ free_list_allocator::free_list_allocator(cell size_, cell start_) : template void free_list_allocator::initial_free_list(cell occupied) { free_blocks.initial_free_list(start,end,occupied); - high_water_mark = free_blocks.free_space; } template bool free_list_allocator::contains_p(Block *block) @@ -131,52 +129,27 @@ void free_list_allocator::sweep() { free_blocks.clear_free_list(); - Block *prev = NULL; - Block *scan = this->first_block(); + Block *start = this->first_block(); Block *end = this->last_block(); - while(scan != end) + while(start != end) { - cell size = scan->size(); + /* find next unmarked block */ + start = state.next_unmarked_block_after(start); + + if(start != end) + { + /* find size */ + cell size = state.unmarked_block_size(start); + assert(size > 0); - if(scan->free_p()) - { - if(prev && prev->free_p()) - { - free_heap_block *free_prev = (free_heap_block *)prev; - free_prev->make_free(free_prev->size() + size); - } - else - prev = scan; - } - else if(this->state.marked_p(scan)) - { - if(prev && prev->free_p()) - free_blocks.add_to_free_list((free_heap_block *)prev); - prev = scan; - } - else - { - if(prev && prev->free_p()) - { - free_heap_block *free_prev = (free_heap_block *)prev; - free_prev->make_free(free_prev->size() + size); - } - else - { - free_heap_block *free_block = (free_heap_block *)scan; - free_block->make_free(size); - prev = scan; - } - } + free_heap_block *free_block = (free_heap_block *)start; + free_block->make_free(size); + free_blocks.add_to_free_list(free_block); - scan = (Block *)((cell)scan + size); + start = (Block *)((char *)start + size); + } } - - if(prev && prev->free_p()) - free_blocks.add_to_free_list((free_heap_block *)prev); - - high_water_mark = free_blocks.free_space; } template @@ -230,8 +203,6 @@ void free_list_allocator::sweep(Iterator &iter) if(prev && prev->free_p()) free_blocks.add_to_free_list((free_heap_block *)prev); - - high_water_mark = free_blocks.free_space; } template struct heap_compactor { diff --git a/vm/image.cpp b/vm/image.cpp index 152a39bdf3..fce730df5a 100755 --- a/vm/image.cpp +++ b/vm/image.cpp @@ -20,8 +20,7 @@ void factor_vm::load_data_heap(FILE *file, image_header *h, vm_parameters *p) init_data_heap(p->young_size, p->aging_size, - p->tenured_size, - p->promotion_threshold); + p->tenured_size); fixnum bytes_read = fread((void*)data->tenured->start,1,h->data_size,file); diff --git a/vm/image.hpp b/vm/image.hpp index 753d11987d..cca0e9e378 100755 --- a/vm/image.hpp +++ b/vm/image.hpp @@ -33,7 +33,6 @@ struct vm_parameters { const vm_char *executable_path; cell ds_size, rs_size; cell young_size, aging_size, tenured_size; - cell promotion_threshold; cell code_size; bool fep; bool console; diff --git a/vm/mark_bits.hpp b/vm/mark_bits.hpp index 1b8fa913a6..97d4bd4fa3 100644 --- a/vm/mark_bits.hpp +++ b/vm/mark_bits.hpp @@ -60,8 +60,8 @@ template struct mark_bits { bool bitmap_elt(u64 *bits, Block *address) { - std::pair pair = bitmap_deref(address); - return (bits[pair.first] & ((u64)1 << pair.second)) != 0; + std::pair position = bitmap_deref(address); + return (bits[position.first] & ((u64)1 << position.second)) != 0; } Block *next_block_after(Block *block) @@ -142,12 +142,12 @@ template struct mark_bits { #ifdef FACTOR_DEBUG assert(marked_p(original)); #endif - std::pair pair = bitmap_deref(original); + std::pair position = bitmap_deref(original); - cell approx_popcount = forwarding[pair.first]; - u64 mask = ((u64)1 << pair.second) - 1; + cell approx_popcount = forwarding[position.first]; + u64 mask = ((u64)1 << position.second) - 1; - cell new_line_number = approx_popcount + popcount(marked[pair.first] & mask); + cell new_line_number = approx_popcount + popcount(marked[position.first] & mask); Block *new_block = line_block(new_line_number); #ifdef FACTOR_DEBUG assert(new_block <= original); @@ -155,18 +155,78 @@ template struct mark_bits { return new_block; } - /* Find the next allocated block without calling size() on unmarked - objects. */ - cell unmarked_space_starting_at(Block *original) + cell rightmost_clear_bit(u64 x) { - char *start = (char *)original; - char *scan = start; + cell n = 0; + while(x & 1) + { + n++; + x >>= 1; + } + return n; + } + + Block *next_unmarked_block_after_slow(Block *original) + { + char *scan = (char *)original; + char *end = (char *)(this->start + this->size); + + while(scan != end && marked_p((Block *)scan)) + scan += block_granularity; + + return (Block *)scan; + } + + Block *next_unmarked_block_after_fast(Block *original) + { + std::pair position = bitmap_deref(original); + cell bit_index = position.second; + + for(cell index = position.first; index < bits_size; index++) + { + u64 mask = ((s64)marked[index] >> bit_index); + if(~mask) + { + /* Found an unmarked block on this page. + Stop, it's hammer time */ + cell clear_bit = rightmost_clear_bit(mask); + return line_block(index * 64 + bit_index + clear_bit); + } + else + { + /* No unmarked blocks on this page. + Keep looking */ + bit_index = 0; + } + } + + /* No unmarked blocks were found */ + return (Block *)(this->start + this->size); + } + + Block *next_unmarked_block_after(Block *original) + { + Block *first_result = next_unmarked_block_after_slow(original); + Block *second_result = next_unmarked_block_after_fast(original); + assert(first_result == second_result); + return second_result; + } + + Block *next_marked_block_after(Block *original) + { + char *scan = (char *)original; char *end = (char *)(this->start + this->size); while(scan != end && !marked_p((Block *)scan)) scan += block_granularity; - return scan - start; + return (Block *)scan; + } + + cell unmarked_block_size(Block *original) + { + Block *next_marked = next_marked_block_after(original); + return ((char *)next_marked - (char *)original); } }; diff --git a/vm/slot_visitor.hpp b/vm/slot_visitor.hpp index 48fb0c1af6..38d0081c0a 100644 --- a/vm/slot_visitor.hpp +++ b/vm/slot_visitor.hpp @@ -15,7 +15,7 @@ template struct slot_visitor { if(immediate_p(pointer)) return; object *untagged = untag(pointer); - untagged = visitor.visit_object(untagged); + untagged = visitor(untagged); *handle = RETAG(untagged,TAG(pointer)); } @@ -61,7 +61,7 @@ template struct slot_visitor { cell *handle = (cell *)(*iter); if(*handle) - *handle = (cell)visitor.visit_object(*(object **)handle); + *handle = (cell)visitor(*(object **)handle); } } diff --git a/vm/vm.hpp b/vm/vm.hpp index f04030761b..139c13bb41 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -218,7 +218,7 @@ struct factor_vm //data heap void init_card_decks(); void set_data_heap(data_heap *data_); - void init_data_heap(cell young_size, cell aging_size, cell tenured_size, cell promotion_threshold); + void init_data_heap(cell young_size, cell aging_size, cell tenured_size); void primitive_size(); data_heap_room data_room(); void primitive_data_room(); @@ -351,8 +351,8 @@ struct factor_vm void primitive_uninitialized_byte_array(); void primitive_resize_byte_array(); - template byte_array *byte_array_from_value(T *value); - template byte_array *byte_array_from_values(T *values, cell len); + template byte_array *byte_array_from_value(Type *value); + template byte_array *byte_array_from_values(Type *values, cell len); //tuples void primitive_tuple(); From 882daae06c59db2518b1d58e88faa82a30aa1cab Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Sun, 1 Nov 2009 04:40:15 -0600 Subject: [PATCH 47/74] vm: more sweep phase optimizations --- vm/compaction.cpp | 21 +++++++++-------- vm/full_collector.cpp | 6 ++--- vm/mark_bits.hpp | 52 +++++++++++++++++++++++++------------------ vm/vm.hpp | 2 +- 4 files changed, 45 insertions(+), 36 deletions(-) diff --git a/vm/compaction.cpp b/vm/compaction.cpp index eb11ec9f3d..86d3efd3d6 100644 --- a/vm/compaction.cpp +++ b/vm/compaction.cpp @@ -159,26 +159,29 @@ struct dummy_slot_forwarder { }; /* Compact just the code heap */ -void factor_vm::collect_compact_code_impl() +void factor_vm::collect_compact_code_impl(bool trace_contexts_p) { - mark_bits *code_forwarding_map = &code->allocator->state; - /* Figure out where blocks are going to go */ + mark_bits *code_forwarding_map = &code->allocator->state; code_forwarding_map->compute_forwarding(); - - /* Update root pointers */ code_block_visitor > code_forwarder(this,forwarder(code_forwarding_map)); + if(trace_contexts_p) + { + code_forwarder.visit_context_code_blocks(); + code_forwarder.visit_callback_code_blocks(); + } + + /* Update code heap references in data heap */ + object_code_block_updater updater(&code_forwarder); + each_object(updater); + /* Slide everything in the code heap up, and update code heap pointers inside code blocks. */ dummy_slot_forwarder slot_forwarder; code_block_compaction_updater code_block_updater(this,slot_forwarder); standard_sizer code_block_sizer; code->allocator->compact(code_block_updater,code_block_sizer); - - /* Update code heap references in data heap */ - object_code_block_updater updater(&code_forwarder); - each_object(updater); } } diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index 397b210c6d..9d725f67d9 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -76,9 +76,7 @@ void factor_vm::collect_mark_impl(bool trace_contexts_p) void factor_vm::collect_sweep_impl() { current_gc->event->started_data_sweep(); - data->tenured->starts.clear_object_start_offsets(); - object_start_map_updater updater(&data->tenured->starts); - data->tenured->sweep(updater); + data->tenured->sweep(); current_gc->event->ended_data_sweep(); } @@ -88,7 +86,7 @@ void factor_vm::collect_growing_heap(cell requested_bytes, bool trace_contexts_p data_heap *old = data; set_data_heap(data->grow(requested_bytes)); collect_mark_impl(trace_contexts_p); - collect_compact_code_impl(); + collect_compact_code_impl(trace_contexts_p); delete old; } diff --git a/vm/mark_bits.hpp b/vm/mark_bits.hpp index 97d4bd4fa3..98b9bdae3e 100644 --- a/vm/mark_bits.hpp +++ b/vm/mark_bits.hpp @@ -166,18 +166,7 @@ template struct mark_bits { return n; } - Block *next_unmarked_block_after_slow(Block *original) - { - char *scan = (char *)original; - char *end = (char *)(this->start + this->size); - - while(scan != end && marked_p((Block *)scan)) - scan += block_granularity; - - return (Block *)scan; - } - - Block *next_unmarked_block_after_fast(Block *original) + Block *next_unmarked_block_after(Block *original) { std::pair position = bitmap_deref(original); cell bit_index = position.second; @@ -204,23 +193,42 @@ template struct mark_bits { return (Block *)(this->start + this->size); } - Block *next_unmarked_block_after(Block *original) + cell rightmost_set_bit(u64 x) { - Block *first_result = next_unmarked_block_after_slow(original); - Block *second_result = next_unmarked_block_after_fast(original); - assert(first_result == second_result); - return second_result; + cell n = 0; + while(!(x & 1)) + { + n++; + x >>= 1; + } + return n; } Block *next_marked_block_after(Block *original) { - char *scan = (char *)original; - char *end = (char *)(this->start + this->size); + std::pair position = bitmap_deref(original); + cell bit_index = position.second; - while(scan != end && !marked_p((Block *)scan)) - scan += block_granularity; + for(cell index = position.first; index < bits_size; index++) + { + u64 mask = (marked[index] >> bit_index); + if(mask) + { + /* Found an marked block on this page. + Stop, it's hammer time */ + cell set_bit = rightmost_set_bit(mask); + return line_block(index * 64 + bit_index + set_bit); + } + else + { + /* No marked blocks on this page. + Keep looking */ + bit_index = 0; + } + } - return (Block *)scan; + /* No marked blocks were found */ + return (Block *)(this->start + this->size); } cell unmarked_block_size(Block *original) diff --git a/vm/vm.hpp b/vm/vm.hpp index 139c13bb41..5cb11c12f7 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -258,7 +258,7 @@ struct factor_vm void collect_mark_impl(bool trace_contexts_p); void collect_sweep_impl(); void collect_compact_impl(bool trace_contexts_p); - void collect_compact_code_impl(); + void collect_compact_code_impl(bool trace_contexts_p); void collect_growing_heap(cell requested_bytes, bool trace_contexts_p); void gc(gc_op op, cell requested_bytes, bool trace_contexts_p); void primitive_minor_gc(); From e28580dd0f7365e65c7b55e95e3574c8a46f5714 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Sun, 1 Nov 2009 20:15:42 -0600 Subject: [PATCH 48/74] vm: faster sweep algorithm --- basis/tools/memory/memory.factor | 26 +++++++------ basis/vm/vm.factor | 3 +- vm/bitwise_hacks.hpp | 67 ++++++++++++++++++++++++++++++++ vm/full_collector.cpp | 11 ------ vm/gc.cpp | 20 +++++----- vm/gc.hpp | 1 + vm/mark_bits.hpp | 37 ------------------ vm/master.hpp | 1 + vm/object_start_map.cpp | 32 +++++++++++++++ vm/object_start_map.hpp | 2 + vm/tenured_space.hpp | 6 +++ 11 files changed, 136 insertions(+), 70 deletions(-) create mode 100644 vm/bitwise_hacks.hpp diff --git a/basis/tools/memory/memory.factor b/basis/tools/memory/memory.factor index 9b1d6284bb..5c6e910108 100644 --- a/basis/tools/memory/memory.factor +++ b/basis/tools/memory/memory.factor @@ -178,13 +178,6 @@ TUPLE: gc-stats collections times ; ] each ] { } make ; -: aggregate-stats-table ( stats -- table ) - [ { "Total collections:" "Total GC time:" } ] dip - values - [ [ collections>> ] map-sum ] - [ [ times>> sum ] map-sum micros>string ] - bi 2array zip ; - PRIVATE> : gc-event. ( event -- ) @@ -195,10 +188,21 @@ PRIVATE> } fancy-table. ; : gc-stats. ( events -- ) - compute-gc-stats - [ gc-stats-table simple-table. nl ] - [ aggregate-stats-table simple-table. ] - bi ; + compute-gc-stats gc-stats-table simple-table. ; + +: gc-summary. ( events -- ) + { + { "Collections:" [ length commas ] } + { "Cards scanned:" [ [ cards-scanned>> ] map-sum commas ] } + { "Decks scanned:" [ [ decks-scanned>> ] map-sum commas ] } + { "Code blocks scanned:" [ [ code-blocks-scanned>> ] map-sum commas ] } + { "Total time:" [ [ total-time>> ] map-sum micros>string ] } + { "Card scan time:" [ [ card-scan-time>> ] map-sum micros>string ] } + { "Code block scan time:" [ [ code-scan-time>> ] map-sum micros>string ] } + { "Data heap sweep time:" [ [ data-sweep-time>> ] map-sum micros>string ] } + { "Code heap sweep time:" [ [ code-sweep-time>> ] map-sum micros>string ] } + { "Compaction time:" [ [ compaction-time>> ] map-sum micros>string ] } + } fancy-table. ; : heap-sizes. ( events -- ) heap-sizes simple-table. ; diff --git a/basis/vm/vm.factor b/basis/vm/vm.factor index 6bcdd2862a..94fa0fcaf5 100644 --- a/basis/vm/vm.factor +++ b/basis/vm/vm.factor @@ -64,4 +64,5 @@ STRUCT: gc-event { code-scan-time cell } { data-sweep-time cell } { code-sweep-time cell } -{ compaction-time cell } ; +{ compaction-time cell } +{ temp-time cell } ; diff --git a/vm/bitwise_hacks.hpp b/vm/bitwise_hacks.hpp new file mode 100644 index 0000000000..dc685bb28c --- /dev/null +++ b/vm/bitwise_hacks.hpp @@ -0,0 +1,67 @@ +namespace factor +{ + +/* These algorithms were snarfed from various places. I did not come up with them myself */ + +inline cell popcount(u64 x) +{ + u64 k1 = 0x5555555555555555ll; + u64 k2 = 0x3333333333333333ll; + u64 k4 = 0x0f0f0f0f0f0f0f0fll; + u64 kf = 0x0101010101010101ll; + x = x - ((x >> 1) & k1); // put count of each 2 bits into those 2 bits + x = (x & k2) + ((x >> 2) & k2); // put count of each 4 bits into those 4 bits + x = (x + (x >> 4)) & k4 ; // put count of each 8 bits into those 8 bits + x = (x * kf) >> 56; // returns 8 most significant bits of x + (x<<8) + (x<<16) + (x<<24) + ... + + return (cell)x; +} + +inline cell log2(u64 x) +{ +#ifdef FACTOR_AMD64 + cell n; + asm ("bsr %1, %0;":"=r"(n):"r"((cell)x)); +#else + cell n = 0; + if (x >= (u64)1 << 32) { x >>= 32; n += 32; } + if (x >= (u64)1 << 16) { x >>= 16; n += 16; } + if (x >= (u64)1 << 8) { x >>= 8; n += 8; } + if (x >= (u64)1 << 4) { x >>= 4; n += 4; } + if (x >= (u64)1 << 2) { x >>= 2; n += 2; } + if (x >= (u64)1 << 1) { n += 1; } +#endif + return n; +} + +inline cell log2(u16 x) +{ +#if defined(FACTOR_X86) || defined(FACTOR_AMD64) + cell n; + asm ("bsr %1, %0;":"=r"(n):"r"((cell)x)); +#else + cell n = 0; + if (x >= 1 << 8) { x >>= 8; n += 8; } + if (x >= 1 << 4) { x >>= 4; n += 4; } + if (x >= 1 << 2) { x >>= 2; n += 2; } + if (x >= 1 << 1) { n += 1; } +#endif + return n; +} + +inline cell rightmost_clear_bit(u64 x) +{ + return log2(~x & (x + 1)); +} + +inline cell rightmost_set_bit(u64 x) +{ + return log2(x & -x); +} + +inline cell rightmost_set_bit(u16 x) +{ + return log2((u16)(x & -x)); +} + +} diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index 9d725f67d9..bbce01be76 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -28,17 +28,6 @@ struct code_block_marker { } }; -struct object_start_map_updater { - object_start_map *starts; - - explicit object_start_map_updater(object_start_map *starts_) : starts(starts_) {} - - void operator()(object *obj, cell size) - { - starts->record_object_start_offset(obj); - } -}; - void factor_vm::collect_mark_impl(bool trace_contexts_p) { full_collector collector(this); diff --git a/vm/gc.cpp b/vm/gc.cpp index f358a9fe4a..1c3d90e66d 100755 --- a/vm/gc.cpp +++ b/vm/gc.cpp @@ -22,55 +22,55 @@ gc_event::gc_event(gc_op op_, factor_vm *parent) : void gc_event::started_card_scan() { - card_scan_time = current_micros(); + temp_time = current_micros(); } void gc_event::ended_card_scan(cell cards_scanned_, cell decks_scanned_) { cards_scanned += cards_scanned_; decks_scanned += decks_scanned_; - card_scan_time = (current_micros() - card_scan_time); + card_scan_time = (current_micros() - temp_time); } void gc_event::started_code_scan() { - code_scan_time = current_micros(); + temp_time = current_micros(); } void gc_event::ended_code_scan(cell code_blocks_scanned_) { code_blocks_scanned += code_blocks_scanned_; - code_scan_time = (current_micros() - code_scan_time); + code_scan_time = (current_micros() - temp_time); } void gc_event::started_data_sweep() { - data_sweep_time = current_micros(); + temp_time = current_micros(); } void gc_event::ended_data_sweep() { - data_sweep_time = (current_micros() - data_sweep_time); + data_sweep_time = (current_micros() - temp_time); } void gc_event::started_code_sweep() { - code_sweep_time = current_micros(); + temp_time = current_micros(); } void gc_event::ended_code_sweep() { - code_sweep_time = (current_micros() - code_sweep_time); + code_sweep_time = (current_micros() - temp_time); } void gc_event::started_compaction() { - compaction_time = current_micros(); + temp_time = current_micros(); } void gc_event::ended_compaction() { - compaction_time = (current_micros() - compaction_time); + compaction_time = (current_micros() - temp_time); } void gc_event::ended_gc(factor_vm *parent) diff --git a/vm/gc.hpp b/vm/gc.hpp index 90b5ce6032..7ad7e71524 100755 --- a/vm/gc.hpp +++ b/vm/gc.hpp @@ -26,6 +26,7 @@ struct gc_event { cell data_sweep_time; cell code_sweep_time; cell compaction_time; + cell temp_time; explicit gc_event(gc_op op_, factor_vm *parent); void started_card_scan(); diff --git a/vm/mark_bits.hpp b/vm/mark_bits.hpp index 98b9bdae3e..b54a2c9d46 100644 --- a/vm/mark_bits.hpp +++ b/vm/mark_bits.hpp @@ -109,21 +109,6 @@ template struct mark_bits { set_bitmap_range(marked,address); } - /* From http://chessprogramming.wikispaces.com/Population+Count */ - cell popcount(u64 x) - { - u64 k1 = 0x5555555555555555ll; - u64 k2 = 0x3333333333333333ll; - u64 k4 = 0x0f0f0f0f0f0f0f0fll; - u64 kf = 0x0101010101010101ll; - x = x - ((x >> 1) & k1); // put count of each 2 bits into those 2 bits - x = (x & k2) + ((x >> 2) & k2); // put count of each 4 bits into those 4 bits - x = (x + (x >> 4)) & k4 ; // put count of each 8 bits into those 8 bits - x = (x * kf) >> 56; // returns 8 most significant bits of x + (x<<8) + (x<<16) + (x<<24) + ... - - return (cell)x; - } - /* The eventual destination of a block after compaction is just the number of marked blocks before it. Live blocks must be marked on entry. */ void compute_forwarding() @@ -155,17 +140,6 @@ template struct mark_bits { return new_block; } - cell rightmost_clear_bit(u64 x) - { - cell n = 0; - while(x & 1) - { - n++; - x >>= 1; - } - return n; - } - Block *next_unmarked_block_after(Block *original) { std::pair position = bitmap_deref(original); @@ -193,17 +167,6 @@ template struct mark_bits { return (Block *)(this->start + this->size); } - cell rightmost_set_bit(u64 x) - { - cell n = 0; - while(!(x & 1)) - { - n++; - x >>= 1; - } - return n; - } - Block *next_marked_block_after(Block *original) { std::pair position = bitmap_deref(original); diff --git a/vm/master.hpp b/vm/master.hpp index e4f478e7c1..0f8276cf66 100755 --- a/vm/master.hpp +++ b/vm/master.hpp @@ -50,6 +50,7 @@ namespace factor #include "bignum.hpp" #include "code_block.hpp" #include "bump_allocator.hpp" +#include "bitwise_hacks.hpp" #include "mark_bits.hpp" #include "free_list.hpp" #include "free_list_allocator.hpp" diff --git a/vm/object_start_map.cpp b/vm/object_start_map.cpp index cb4f86c6c3..724f365e79 100644 --- a/vm/object_start_map.cpp +++ b/vm/object_start_map.cpp @@ -55,4 +55,36 @@ void object_start_map::clear_object_start_offsets() memset(object_start_offsets,card_starts_inside_object,addr_to_card(size)); } +void object_start_map::update_card_for_sweep(cell index, u16 mask) +{ + cell offset = object_start_offsets[index]; + if(offset != card_starts_inside_object) + { + mask >>= (offset / block_granularity); + + if(mask == 0) + { + /* The rest of the block after the old object start is free */ + object_start_offsets[index] = card_starts_inside_object; + } + else + { + /* Move the object start forward if necessary */ + object_start_offsets[index] = offset + (rightmost_set_bit(mask) * block_granularity); + } + } +} + +void object_start_map::update_for_sweep(mark_bits *state) +{ + for(cell index = 0; index < state->bits_size; index++) + { + u64 mask = state->marked[index]; + update_card_for_sweep(index * 4, mask & 0xffff); + update_card_for_sweep(index * 4 + 1, (mask >> 16) & 0xffff); + update_card_for_sweep(index * 4 + 2, (mask >> 32) & 0xffff); + update_card_for_sweep(index * 4 + 3, (mask >> 48) & 0xffff); + } +} + } diff --git a/vm/object_start_map.hpp b/vm/object_start_map.hpp index 69f9c11a6a..a2e24eeda6 100644 --- a/vm/object_start_map.hpp +++ b/vm/object_start_map.hpp @@ -15,6 +15,8 @@ struct object_start_map { cell find_object_containing_card(cell card_index); void record_object_start_offset(object *obj); void clear_object_start_offsets(); + void update_card_for_sweep(cell index, u16 mask); + void update_for_sweep(mark_bits *state); }; } diff --git a/vm/tenured_space.hpp b/vm/tenured_space.hpp index f77f17a56a..baab47e383 100644 --- a/vm/tenured_space.hpp +++ b/vm/tenured_space.hpp @@ -52,6 +52,12 @@ struct tenured_space : free_list_allocator { this->state.set_marked_p(obj); this->mark_stack.push_back(obj); } + + void sweep() + { + free_list_allocator::sweep(); + starts.update_for_sweep(&this->state); + } }; } From b2d96e5ef17288945ed44fa8b4b91fa9b76d76c7 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Sun, 1 Nov 2009 20:34:54 -0600 Subject: [PATCH 49/74] vm: remove debug messages --- vm/gc.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/vm/gc.cpp b/vm/gc.cpp index 1c3d90e66d..afc1798911 100755 --- a/vm/gc.cpp +++ b/vm/gc.cpp @@ -155,10 +155,7 @@ void factor_vm::gc(gc_op op, cell requested_bytes, bool trace_contexts_p) if(current_gc->op == collect_aging_op || current_gc->op == collect_to_tenured_op) { if(data->tenured->free_space() <= data->nursery->size + data->aging->size) - { - printf("upgrade\n"); current_gc->op = collect_full_op; - } } current_gc->event->op = current_gc->op; @@ -178,10 +175,7 @@ void factor_vm::gc(gc_op op, cell requested_bytes, bool trace_contexts_p) collect_mark_impl(trace_contexts_p); collect_sweep_impl(); if(data->tenured->largest_free_block() <= data->nursery->size + data->aging->size) - { - printf("forced compaction\n"); collect_compact_impl(trace_contexts_p); - } else update_code_heap_words_and_literals(); break; From 94e9230ba34120853178bab0bc51839967e572a4 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Sun, 1 Nov 2009 23:14:34 -0600 Subject: [PATCH 50/74] vm: promote objects referenced from compiled code blocks directly to tenured in an aging collection --- vm/aging_collector.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/vm/aging_collector.cpp b/vm/aging_collector.cpp index 1979769b81..3572677aa6 100644 --- a/vm/aging_collector.cpp +++ b/vm/aging_collector.cpp @@ -28,7 +28,15 @@ void factor_vm::collect_aging() full_unmarker()); current_gc->event->ended_card_scan(collector.cards_scanned,collector.decks_scanned); + current_gc->event->started_code_scan(); + collector.trace_code_heap_roots(&code->points_to_aging); + current_gc->event->ended_code_scan(collector.code_blocks_scanned); + collector.tenure_reachable_objects(); + + current_gc->event->started_code_sweep(); + update_code_heap_for_minor_gc(&code->points_to_aging); + current_gc->event->ended_code_sweep(); } { /* If collection fails here, do a to_tenured collection. */ @@ -42,18 +50,11 @@ void factor_vm::collect_aging() collector.trace_roots(); collector.trace_contexts(); - current_gc->event->started_code_scan(); - collector.trace_code_heap_roots(&code->points_to_aging); - current_gc->event->ended_code_scan(collector.code_blocks_scanned); - collector.cheneys_algorithm(); - current_gc->event->started_code_sweep(); - update_code_heap_for_minor_gc(&code->points_to_aging); - current_gc->event->ended_code_sweep(); - data->reset_generation(&nursery); code->points_to_nursery.clear(); + code->points_to_aging.clear(); } } From 7e17c3077c7df8792cd0a9b6279691b99ca0c8ef Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Mon, 2 Nov 2009 01:37:58 -0600 Subject: [PATCH 51/74] vm: new megamorphic hashcode algorithm improves bootstrap speed --- basis/cpu/ppc/bootstrap.factor | 9 +++++++-- basis/cpu/x86/bootstrap.factor | 11 +++++++++-- vm/dispatch.cpp | 2 +- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/basis/cpu/ppc/bootstrap.factor b/basis/cpu/ppc/bootstrap.factor index cd877cfafe..f7a7e58d7d 100644 --- a/basis/cpu/ppc/bootstrap.factor +++ b/basis/cpu/ppc/bootstrap.factor @@ -224,8 +224,13 @@ CONSTANT: rs-reg 14 [ ! cache = ... 0 3 LOAD32 rc-absolute-ppc-2/2 rt-immediate jit-rel - ! key = class - 5 4 MR + ! key = hashcode(class) + 5 4 3 SRAWI + 6 4 8 SRAWI + 5 5 6 ADD + 6 4 13 SRAWI + 5 5 6 ADD + 5 5 3 SLWI ! key &= cache.length - 1 5 5 mega-cache-size get 1 - bootstrap-cell * ANDI ! cache += array-start-offset diff --git a/basis/cpu/x86/bootstrap.factor b/basis/cpu/x86/bootstrap.factor index 7930970193..25a826cde4 100644 --- a/basis/cpu/x86/bootstrap.factor +++ b/basis/cpu/x86/bootstrap.factor @@ -222,9 +222,16 @@ big-endian off [ ! cache = ... temp0 0 MOV rc-absolute-cell rt-immediate jit-rel - ! key = class + ! key = hashcode(class) temp2 temp1 MOV - bootstrap-cell 8 = [ temp2 1 SHL ] when + temp2 3 SHR + temp3 temp1 MOV + temp3 8 SHR + temp2 temp3 ADD + temp3 temp1 MOV + temp3 13 SHR + temp2 temp3 ADD + temp2 bootstrap-cell 4 = 3 4 ? SHL ! key &= cache.length - 1 temp2 mega-cache-size get 1 - bootstrap-cell * AND ! cache += array-start-offset diff --git a/vm/dispatch.cpp b/vm/dispatch.cpp index e636012126..333a49bfbe 100755 --- a/vm/dispatch.cpp +++ b/vm/dispatch.cpp @@ -126,7 +126,7 @@ cell factor_vm::object_class(cell obj) cell factor_vm::method_cache_hashcode(cell klass, array *array) { cell capacity = (array_capacity(array) >> 1) - 1; - return ((klass >> TAG_BITS) & capacity) << 1; + return (((klass >> 3) + (klass >> 8) + (klass >> 13)) & capacity) << 1; } void factor_vm::update_method_cache(cell cache, cell klass, cell method) From e4ad6421345b60f75ecf2a0302c3a085e2cae816 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Mon, 2 Nov 2009 03:25:39 -0600 Subject: [PATCH 52/74] vm: 4 bit tags, new representation of alien objects makes unbox-any-c-ptr more efficient (work in progress) --- basis/bootstrap/image/image.factor | 19 +++---- .../cfg/alias-analysis/alias-analysis.factor | 2 +- .../compiler/cfg/builder/builder-tests.factor | 5 +- basis/compiler/cfg/builder/builder.factor | 6 +- basis/compiler/cfg/gc-checks/gc-checks.factor | 10 ++-- basis/compiler/cfg/hats/hats.factor | 4 +- .../cfg/instructions/instructions.factor | 2 +- .../cfg/intrinsics/allot/allot.factor | 6 +- .../cfg/intrinsics/fixnum/fixnum.factor | 2 +- .../cfg/intrinsics/slots/slots.factor | 2 +- .../representations/representations.factor | 2 +- .../value-numbering/rewrite/rewrite.factor | 6 +- basis/compiler/constants/constants.factor | 25 ++++----- basis/compiler/tests/codegen.factor | 14 ----- basis/compiler/tests/intrinsics.factor | 2 +- basis/compiler/tests/low-level-ir.factor | 8 +-- .../known-words/known-words.factor | 2 +- .../tree/propagation/propagation-tests.factor | 4 +- basis/cpu/ppc/bootstrap.factor | 39 +++---------- basis/cpu/ppc/ppc.factor | 31 +++++----- basis/cpu/x86/32/bootstrap.factor | 2 +- basis/cpu/x86/64/bootstrap.factor | 2 +- basis/cpu/x86/bootstrap.factor | 49 +++------------- basis/cpu/x86/x86.factor | 56 ++++++++----------- basis/io/buffers/buffers.factor | 2 +- .../known-words/known-words.factor | 2 +- basis/tools/time/time.factor | 6 +- core/alien/alien.factor | 16 +----- core/bootstrap/layouts/layouts.factor | 30 ++++------ core/bootstrap/primitives.factor | 4 -- core/classes/algebra/algebra-docs.factor | 1 - core/classes/algebra/algebra-tests.factor | 2 - core/classes/algebra/algebra.factor | 10 +--- core/classes/builtin/builtin.factor | 18 +----- core/classes/classes-tests.factor | 1 - core/generic/single/single.factor | 27 +-------- core/kernel/kernel-docs.factor | 2 +- core/kernel/kernel.factor | 2 - core/layouts/layouts-docs.factor | 15 +---- core/layouts/layouts.factor | 7 --- vm/alien.cpp | 13 ++--- vm/collector.hpp | 4 +- vm/copying_collector.hpp | 2 +- vm/cpu-ppc.S | 2 +- vm/cpu-x86.S | 2 +- vm/dispatch.cpp | 33 ++--------- vm/full_collector.cpp | 2 +- vm/image.cpp | 7 ++- vm/inline_cache.cpp | 41 ++++---------- vm/layouts.hpp | 32 +++++------ vm/run.hpp | 6 +- vm/tagged.hpp | 10 +--- vm/to_tenured_collector.cpp | 2 +- vm/vm.hpp | 5 +- 54 files changed, 196 insertions(+), 410 deletions(-) diff --git a/basis/bootstrap/image/image.factor b/basis/bootstrap/image/image.factor index 6d2dfe332e..2178b5d4cb 100644 --- a/basis/bootstrap/image/image.factor +++ b/basis/bootstrap/image/image.factor @@ -176,14 +176,12 @@ USERENV: callback-stub 45 ! PIC stubs USERENV: pic-load 47 USERENV: pic-tag 48 -USERENV: pic-hi-tag 49 -USERENV: pic-tuple 50 -USERENV: pic-hi-tag-tuple 51 -USERENV: pic-check-tag 52 -USERENV: pic-check 53 -USERENV: pic-hit 54 -USERENV: pic-miss-word 55 -USERENV: pic-miss-tail-word 56 +USERENV: pic-tuple 49 +USERENV: pic-check-tag 50 +USERENV: pic-check-tuple 51 +USERENV: pic-hit 52 +USERENV: pic-miss-word 53 +USERENV: pic-miss-tail-word 54 ! Megamorphic dispatch USERENV: mega-lookup 57 @@ -227,7 +225,8 @@ USERENV: undefined-quot 60 : emit-fixnum ( n -- ) tag-fixnum emit ; : emit-object ( class quot -- addr ) - over tag-number here-as [ swap type-number tag-fixnum emit call align-here ] dip ; + [ type-number ] dip over here-as + [ swap tag-fixnum emit call align-here ] dip ; inline ! Write an object to the image. @@ -308,7 +307,7 @@ M: float ' M: f ' #! f is #define F RETAG(0,F_TYPE) - drop \ f tag-number ; + drop \ f type-number ; : 0, ( -- ) 0 >bignum ' 0-offset fixup ; : 1, ( -- ) 1 >bignum ' 1-offset fixup ; diff --git a/basis/compiler/cfg/alias-analysis/alias-analysis.factor b/basis/compiler/cfg/alias-analysis/alias-analysis.factor index 2303b98aed..9fffa0eed2 100644 --- a/basis/compiler/cfg/alias-analysis/alias-analysis.factor +++ b/basis/compiler/cfg/alias-analysis/alias-analysis.factor @@ -284,7 +284,7 @@ M: ##copy analyze-aliases* M: ##compare analyze-aliases* call-next-method dup useless-compare? [ - dst>> \ f tag-number \ ##load-immediate new-insn + dst>> \ f type-number \ ##load-immediate new-insn analyze-aliases* ] when ; diff --git a/basis/compiler/cfg/builder/builder-tests.factor b/basis/compiler/cfg/builder/builder-tests.factor index 9d1502d3f0..7f1b6aa6f2 100644 --- a/basis/compiler/cfg/builder/builder-tests.factor +++ b/basis/compiler/cfg/builder/builder-tests.factor @@ -119,7 +119,6 @@ IN: compiler.cfg.builder.tests { byte-array - simple-alien alien POSTPONE: f } [| class | @@ -192,7 +191,7 @@ IN: compiler.cfg.builder.tests ] unit-test [ f t ] [ - [ { fixnum simple-alien } declare 0 alien-cell ] + [ { fixnum alien } declare 0 alien-cell ] [ [ ##unbox-any-c-ptr? ] contains-insn? ] [ [ ##unbox-alien? ] contains-insn? ] bi ] unit-test @@ -205,7 +204,7 @@ IN: compiler.cfg.builder.tests ] unit-test [ f t ] [ - [ { byte-array fixnum } declare alien-cell { simple-alien } declare 4 alien-float ] + [ { byte-array fixnum } declare alien-cell { alien } declare 4 alien-float ] [ [ ##box-alien? ] contains-insn? ] [ [ ##allot? ] contains-insn? ] bi ] unit-test diff --git a/basis/compiler/cfg/builder/builder.factor b/basis/compiler/cfg/builder/builder.factor index 11aae28bf3..cf6215c5cd 100755 --- a/basis/compiler/cfg/builder/builder.factor +++ b/basis/compiler/cfg/builder/builder.factor @@ -117,7 +117,7 @@ M: #recursive emit-node and ; : emit-trivial-if ( -- ) - ds-pop \ f tag-number cc/= ^^compare-imm ds-push ; + ds-pop \ f type-number cc/= ^^compare-imm ds-push ; : trivial-not-if? ( #if -- ? ) children>> first2 @@ -126,12 +126,12 @@ M: #recursive emit-node and ; : emit-trivial-not-if ( -- ) - ds-pop \ f tag-number cc= ^^compare-imm ds-push ; + ds-pop \ f type-number cc= ^^compare-imm ds-push ; : emit-actual-if ( #if -- ) ! Inputs to the final instruction need to be copied because of ! loc>vreg sync - ds-pop any-rep ^^copy \ f tag-number cc/= ##compare-imm-branch emit-if ; + ds-pop any-rep ^^copy \ f type-number cc/= ##compare-imm-branch emit-if ; M: #if emit-node { diff --git a/basis/compiler/cfg/gc-checks/gc-checks.factor b/basis/compiler/cfg/gc-checks/gc-checks.factor index 7285685b48..29616aaf8f 100644 --- a/basis/compiler/cfg/gc-checks/gc-checks.factor +++ b/basis/compiler/cfg/gc-checks/gc-checks.factor @@ -1,6 +1,6 @@ ! Copyright (C) 2009 Slava Pestov. ! See http://factorcode.org/license.txt for BSD license. -USING: accessors kernel sequences assocs fry +USING: accessors kernel sequences assocs fry math cpu.architecture layouts compiler.cfg.rpo compiler.cfg.registers @@ -21,12 +21,14 @@ GENERIC: allocation-size* ( insn -- n ) M: ##allot allocation-size* size>> ; -M: ##box-alien allocation-size* drop 4 cells ; +M: ##box-alien allocation-size* drop 5 cells ; -M: ##box-displaced-alien allocation-size* drop 4 cells ; +M: ##box-displaced-alien allocation-size* drop 5 cells ; : allocation-size ( bb -- n ) - instructions>> [ ##allocation? ] filter [ allocation-size* ] map-sum ; + instructions>> + [ ##allocation? ] filter + [ allocation-size* data-alignment align ] map-sum ; : insert-gc-check ( bb -- ) dup dup '[ diff --git a/basis/compiler/cfg/hats/hats.factor b/basis/compiler/cfg/hats/hats.factor index 783df0678c..9d1945c525 100644 --- a/basis/compiler/cfg/hats/hats.factor +++ b/basis/compiler/cfg/hats/hats.factor @@ -43,14 +43,14 @@ insn-classes get [ : ^^load-literal ( obj -- dst ) [ next-vreg dup ] dip { - { [ dup not ] [ drop \ f tag-number ##load-immediate ] } + { [ dup not ] [ drop \ f type-number ##load-immediate ] } { [ dup fixnum? ] [ tag-fixnum ##load-immediate ] } { [ dup float? ] [ ##load-constant ] } [ ##load-reference ] } cond ; : ^^offset>slot ( slot -- vreg' ) - cell 4 = [ 1 ^^shr-imm ] [ any-rep ^^copy ] if ; + cell 4 = 2 1 ? ^^shr-imm ; : ^^tag-fixnum ( src -- dst ) tag-bits get ^^shl-imm ; diff --git a/basis/compiler/cfg/instructions/instructions.factor b/basis/compiler/cfg/instructions/instructions.factor index d4d84a088a..fecc087dae 100644 --- a/basis/compiler/cfg/instructions/instructions.factor +++ b/basis/compiler/cfg/instructions/instructions.factor @@ -530,7 +530,7 @@ use: src/int-rep ; : ##unbox-c-ptr ( dst src class temp -- ) { { [ over \ f class<= ] [ 2drop ##unbox-f ] } - { [ over simple-alien class<= ] [ 2drop ##unbox-alien ] } + { [ over alien class<= ] [ 2drop ##unbox-alien ] } { [ over byte-array class<= ] [ 2drop ##unbox-byte-array ] } [ nip ##unbox-any-c-ptr ] } cond ; diff --git a/basis/compiler/cfg/intrinsics/allot/allot.factor b/basis/compiler/cfg/intrinsics/allot/allot.factor index 044b839f4d..43747f88c9 100644 --- a/basis/compiler/cfg/intrinsics/allot/allot.factor +++ b/basis/compiler/cfg/intrinsics/allot/allot.factor @@ -8,7 +8,7 @@ compiler.cfg.utilities compiler.cfg.builder.blocks ; IN: compiler.cfg.intrinsics.allot : ##set-slots ( regs obj class -- ) - '[ _ swap 1 + _ tag-number ##set-slot-imm ] each-index ; + '[ _ swap 1 + _ type-number ##set-slot-imm ] each-index ; : emit-simple-allot ( node -- ) [ in-d>> length ] [ node-output-infos first class>> ] bi @@ -31,10 +31,10 @@ IN: compiler.cfg.intrinsics.allot ] [ drop emit-primitive ] if ; : store-length ( len reg class -- ) - [ [ ^^load-literal ] dip 1 ] dip tag-number ##set-slot-imm ; + [ [ ^^load-literal ] dip 1 ] dip type-number ##set-slot-imm ; :: store-initial-element ( len reg elt class -- ) - len [ [ elt reg ] dip 2 + class tag-number ##set-slot-imm ] each ; + len [ [ elt reg ] dip 2 + class type-number ##set-slot-imm ] each ; : expand-? ( obj -- ? ) dup integer? [ 0 8 between? ] [ drop f ] if ; diff --git a/basis/compiler/cfg/intrinsics/fixnum/fixnum.factor b/basis/compiler/cfg/intrinsics/fixnum/fixnum.factor index 8ead484cf1..e4d1735eae 100644 --- a/basis/compiler/cfg/intrinsics/fixnum/fixnum.factor +++ b/basis/compiler/cfg/intrinsics/fixnum/fixnum.factor @@ -21,7 +21,7 @@ IN: compiler.cfg.intrinsics.fixnum ds-push ; : tag-literal ( n -- tagged ) - literal>> [ tag-fixnum ] [ \ f tag-number ] if* ; + literal>> [ tag-fixnum ] [ \ f type-number ] if* ; : emit-fixnum-op ( insn -- ) [ 2inputs ] dip call ds-push ; inline diff --git a/basis/compiler/cfg/intrinsics/slots/slots.factor b/basis/compiler/cfg/intrinsics/slots/slots.factor index 39151083e5..ad7891b78d 100644 --- a/basis/compiler/cfg/intrinsics/slots/slots.factor +++ b/basis/compiler/cfg/intrinsics/slots/slots.factor @@ -8,7 +8,7 @@ compiler.cfg.instructions compiler.cfg.utilities compiler.cfg.builder.blocks compiler.constants ; IN: compiler.cfg.intrinsics.slots -: value-tag ( info -- n ) class>> class-tag ; inline +: value-tag ( info -- n ) class>> type-number ; inline : ^^tag-offset>slot ( slot tag -- vreg' ) [ ^^offset>slot ] dip ^^sub-imm ; diff --git a/basis/compiler/cfg/representations/representations.factor b/basis/compiler/cfg/representations/representations.factor index 9546721594..005fe8c90b 100644 --- a/basis/compiler/cfg/representations/representations.factor +++ b/basis/compiler/cfg/representations/representations.factor @@ -47,7 +47,7 @@ M:: vector-rep emit-box ( dst src rep -- ) int-rep next-vreg-rep :> temp dst 16 2 cells + byte-array int-rep next-vreg-rep ##allot temp 16 tag-fixnum ##load-immediate - temp dst 1 byte-array tag-number ##set-slot-imm + temp dst 1 byte-array type-number ##set-slot-imm dst byte-array-offset src rep ##set-alien-vector ; M: vector-rep emit-unbox diff --git a/basis/compiler/cfg/value-numbering/rewrite/rewrite.factor b/basis/compiler/cfg/value-numbering/rewrite/rewrite.factor index 5d4ff5efb9..4fd86c8e96 100755 --- a/basis/compiler/cfg/value-numbering/rewrite/rewrite.factor +++ b/basis/compiler/cfg/value-numbering/rewrite/rewrite.factor @@ -37,7 +37,7 @@ M: insn rewrite drop f ; dup ##compare-imm-branch? [ { [ cc>> cc/= eq? ] - [ src2>> \ f tag-number eq? ] + [ src2>> \ f type-number eq? ] } 1&& ] [ drop f ] if ; inline @@ -110,7 +110,7 @@ M: ##compare-imm rewrite-tagged-comparison : rewrite-redundant-comparison? ( insn -- ? ) { [ src1>> vreg>expr general-compare-expr? ] - [ src2>> \ f tag-number = ] + [ src2>> \ f type-number = ] [ cc>> { cc= cc/= } member-eq? ] } 1&& ; inline @@ -204,7 +204,7 @@ M: ##compare-branch rewrite [ dst>> ] dip { { t [ t \ ##load-constant new-insn ] } - { f [ \ f tag-number \ ##load-immediate new-insn ] } + { f [ \ f type-number \ ##load-immediate new-insn ] } } case ; : rewrite-self-compare ( insn -- insn' ) diff --git a/basis/compiler/constants/constants.factor b/basis/compiler/constants/constants.factor index ab607d2178..19cdb6eebd 100644 --- a/basis/compiler/constants/constants.factor +++ b/basis/compiler/constants/constants.factor @@ -12,19 +12,18 @@ CONSTANT: deck-bits 18 ! These constants must match vm/layouts.h : slot-offset ( slot tag -- n ) [ bootstrap-cells ] dip - ; inline -: header-offset ( -- n ) 0 object tag-number slot-offset ; inline -: float-offset ( -- n ) 8 float tag-number - ; inline -: string-offset ( -- n ) 4 string tag-number slot-offset ; inline -: string-aux-offset ( -- n ) 2 string tag-number slot-offset ; inline -: profile-count-offset ( -- n ) 8 \ word tag-number slot-offset ; inline -: byte-array-offset ( -- n ) 16 byte-array tag-number - ; inline -: alien-offset ( -- n ) 3 alien tag-number slot-offset ; inline -: underlying-alien-offset ( -- n ) 1 alien tag-number slot-offset ; inline -: tuple-class-offset ( -- n ) 1 tuple tag-number slot-offset ; inline -: word-xt-offset ( -- n ) 10 \ word tag-number slot-offset ; inline -: quot-xt-offset ( -- n ) 4 quotation tag-number slot-offset ; inline -: word-code-offset ( -- n ) 11 \ word tag-number slot-offset ; inline -: array-start-offset ( -- n ) 2 array tag-number slot-offset ; inline +: float-offset ( -- n ) 8 float type-number - ; inline +: string-offset ( -- n ) 4 string type-number slot-offset ; inline +: string-aux-offset ( -- n ) 2 string type-number slot-offset ; inline +: profile-count-offset ( -- n ) 8 \ word type-number slot-offset ; inline +: byte-array-offset ( -- n ) 16 byte-array type-number - ; inline +: alien-offset ( -- n ) 4 alien type-number slot-offset ; inline +: underlying-alien-offset ( -- n ) 1 alien type-number slot-offset ; inline +: tuple-class-offset ( -- n ) 1 tuple type-number slot-offset ; inline +: word-xt-offset ( -- n ) 10 \ word type-number slot-offset ; inline +: quot-xt-offset ( -- n ) 4 quotation type-number slot-offset ; inline +: word-code-offset ( -- n ) 11 \ word type-number slot-offset ; inline +: array-start-offset ( -- n ) 2 array type-number slot-offset ; inline : compiled-header-size ( -- n ) 4 bootstrap-cells ; inline ! Relocation classes diff --git a/basis/compiler/tests/codegen.factor b/basis/compiler/tests/codegen.factor index 18f3a618f6..eba6580574 100644 --- a/basis/compiler/tests/codegen.factor +++ b/basis/compiler/tests/codegen.factor @@ -175,20 +175,6 @@ TUPLE: my-tuple ; ] compile-call ] unit-test -[ 1 t ] [ - B{ 1 2 3 4 } [ - { c-ptr } declare - [ 0 alien-unsigned-1 ] keep hi-tag - ] compile-call byte-array type-number = -] unit-test - -[ t ] [ - B{ 1 2 3 4 } [ - { c-ptr } declare - 0 alien-cell hi-tag - ] compile-call alien type-number = -] unit-test - [ 2 1 ] [ 2 1 [ 2dup fixnum< [ [ die ] dip ] when ] compile-call diff --git a/basis/compiler/tests/intrinsics.factor b/basis/compiler/tests/intrinsics.factor index 75cfc1d67f..dfc1af9a11 100644 --- a/basis/compiler/tests/intrinsics.factor +++ b/basis/compiler/tests/intrinsics.factor @@ -419,7 +419,7 @@ cell 8 = [ "b" get [ [ 3 ] [ "b" get 2 [ alien-unsigned-1 ] compile-call ] unit-test [ 3 ] [ "b" get [ { alien } declare 2 alien-unsigned-1 ] compile-call ] unit-test - [ 3 ] [ "b" get 2 [ { simple-alien fixnum } declare alien-unsigned-1 ] compile-call ] unit-test + [ 3 ] [ "b" get 2 [ { alien fixnum } declare alien-unsigned-1 ] compile-call ] unit-test [ 3 ] [ "b" get 2 [ { c-ptr fixnum } declare alien-unsigned-1 ] compile-call ] unit-test [ ] [ "b" get free ] unit-test diff --git a/basis/compiler/tests/low-level-ir.factor b/basis/compiler/tests/low-level-ir.factor index 14c470d63f..583b228eb2 100644 --- a/basis/compiler/tests/low-level-ir.factor +++ b/basis/compiler/tests/low-level-ir.factor @@ -50,7 +50,7 @@ IN: compiler.tests.low-level-ir ! one of the sources [ t ] [ V{ - T{ ##load-immediate f 1 $[ 2 cell log2 shift array tag-number - ] } + T{ ##load-immediate f 1 $[ 2 cell log2 shift array type-number - ] } T{ ##load-reference f 0 { t f t } } T{ ##slot f 0 0 1 } } compile-test-bb @@ -59,13 +59,13 @@ IN: compiler.tests.low-level-ir [ t ] [ V{ T{ ##load-reference f 0 { t f t } } - T{ ##slot-imm f 0 0 2 $[ array tag-number ] } + T{ ##slot-imm f 0 0 2 $[ array type-number ] } } compile-test-bb ] unit-test [ t ] [ V{ - T{ ##load-immediate f 1 $[ 2 cell log2 shift array tag-number - ] } + T{ ##load-immediate f 1 $[ 2 cell log2 shift array type-number - ] } T{ ##load-reference f 0 { t f t } } T{ ##set-slot f 0 0 1 } } compile-test-bb @@ -75,7 +75,7 @@ IN: compiler.tests.low-level-ir [ t ] [ V{ T{ ##load-reference f 0 { t f t } } - T{ ##set-slot-imm f 0 0 2 $[ array tag-number ] } + T{ ##set-slot-imm f 0 0 2 $[ array type-number ] } } compile-test-bb dup first eq? ] unit-test diff --git a/basis/compiler/tree/propagation/known-words/known-words.factor b/basis/compiler/tree/propagation/known-words/known-words.factor index 5646dca3fb..8afbaf0099 100644 --- a/basis/compiler/tree/propagation/known-words/known-words.factor +++ b/basis/compiler/tree/propagation/known-words/known-words.factor @@ -279,7 +279,7 @@ generic-comparison-ops [ ] each \ alien-cell [ - 2drop simple-alien \ f class-or + 2drop alien \ f class-or ] "outputs" set-word-prop { } [ diff --git a/basis/compiler/tree/propagation/propagation-tests.factor b/basis/compiler/tree/propagation/propagation-tests.factor index 0f04a5e3d5..3627757acd 100644 --- a/basis/compiler/tree/propagation/propagation-tests.factor +++ b/basis/compiler/tree/propagation/propagation-tests.factor @@ -890,10 +890,10 @@ M: tuple-with-read-only-slot clone [ { 1 2 3 } dup tuple-with-read-only-slot boa clone x>> eq? ] final-classes ] unit-test -! alien-cell outputs a simple-alien or f +! alien-cell outputs a alien or f [ t ] [ [ { byte-array fixnum } declare alien-cell dup [ "OOPS" throw ] unless ] final-classes - first simple-alien class= + first alien class= ] unit-test ! Don't crash if bad literal inputs are passed to unsafe words diff --git a/basis/cpu/ppc/bootstrap.factor b/basis/cpu/ppc/bootstrap.factor index f7a7e58d7d..c16d564e13 100644 --- a/basis/cpu/ppc/bootstrap.factor +++ b/basis/cpu/ppc/bootstrap.factor @@ -69,7 +69,7 @@ CONSTANT: rs-reg 14 [ 3 ds-reg 0 LWZ ds-reg dup 4 SUBI - 0 3 \ f tag-number CMPI + 0 3 \ f type-number CMPI 2 BEQ 0 B rc-relative-ppc-3 rt-xt jit-rel 0 B rc-relative-ppc-3 rt-xt jit-rel @@ -174,40 +174,15 @@ CONSTANT: rs-reg 14 [ load-tag ] pic-tag jit-define -! Hi-tag -[ - 3 4 MR - load-tag - 0 4 object tag-number tag-fixnum CMPI - 2 BNE - 4 3 object tag-number neg LWZ -] pic-hi-tag jit-define - ! Tuple [ 3 4 MR load-tag - 0 4 tuple tag-number tag-fixnum CMPI + 0 4 tuple type-number tag-fixnum CMPI 2 BNE - 4 3 tuple tag-number neg bootstrap-cell + LWZ + 4 3 tuple type-number neg bootstrap-cell + LWZ ] pic-tuple jit-define -! Hi-tag and tuple -[ - 3 4 MR - load-tag - ! If bits 2 and 3 are set, the tag is either 6 (object) or 7 (tuple) - 0 4 BIN: 110 tag-fixnum CMPI - 5 BLT - ! Untag r3 - 3 3 0 0 31 tag-bits get - RLWINM - ! Set r4 to 0 for objects, and bootstrap-cell for tuples - 4 4 1 tag-fixnum ANDI - 4 4 1 SRAWI - ! Load header cell or tuple layout cell - 4 4 3 LWZX -] pic-hi-tag-tuple jit-define - [ 0 4 0 CMPI rc-absolute-ppc-2 rt-immediate jit-rel ] pic-check-tag jit-define @@ -215,7 +190,7 @@ CONSTANT: rs-reg 14 [ 0 5 LOAD32 rc-absolute-ppc-2/2 rt-immediate jit-rel 4 0 5 CMP -] pic-check jit-define +] pic-check-tuple jit-define [ 2 BNE 0 B rc-relative-ppc-3 rt-xt jit-rel ] pic-hit jit-define @@ -283,7 +258,7 @@ CONSTANT: rs-reg 14 [ 3 ds-reg 0 LWZ 4 ds-reg -4 LWZU - 3 3 1 SRAWI + 3 3 2 SRAWI 4 4 0 0 31 tag-bits get - RLWINM 4 3 3 LWZX 3 ds-reg 0 STW @@ -404,7 +379,7 @@ CONSTANT: rs-reg 14 5 ds-reg -4 LWZU 5 0 4 CMP 2 swap execute( offset -- ) ! magic number - \ f tag-number 3 LI + \ f type-number 3 LI 3 ds-reg 0 STW ; : define-jit-compare ( insn word -- ) @@ -423,7 +398,7 @@ CONSTANT: rs-reg 14 4 ds-reg 0 LWZ 3 3 4 OR 3 3 tag-mask get ANDI - \ f tag-number 4 LI + \ f type-number 4 LI 0 3 0 CMPI 2 BNE 1 tag-fixnum 4 LI diff --git a/basis/cpu/ppc/ppc.factor b/basis/cpu/ppc/ppc.factor index 823e2c8188..92cea0d82f 100644 --- a/basis/cpu/ppc/ppc.factor +++ b/basis/cpu/ppc/ppc.factor @@ -266,7 +266,7 @@ M:: ppc %unbox-any-c-ptr ( dst src temp -- ) ! We come back here with displaced aliens "start" resolve-label ! Is the object f? - 0 scratch-reg \ f tag-number CMPI + 0 scratch-reg \ f type-number CMPI ! If so, done "end" get BEQ ! Is the object an alien? @@ -288,25 +288,20 @@ M:: ppc %unbox-any-c-ptr ( dst src temp -- ) "end" resolve-label ] with-scope ; -: alien@ ( n -- n' ) cells object tag-number - ; - -:: %allot-alien ( dst displacement base temp -- ) - dst 4 cells alien temp %allot - temp \ f tag-number %load-immediate - ! Store underlying-alien slot - base dst 1 alien@ STW - ! Store expired slot - temp dst 2 alien@ STW - ! Store offset - displacement dst 3 alien@ STW ; +: alien@ ( n -- n' ) cells alien type-number - ; M:: ppc %box-alien ( dst src temp -- ) [ "f" define-label - dst \ f tag-number %load-immediate + dst %load-immediate 0 src 0 CMPI "f" get BEQ - dst src temp temp %allot-alien + dst 5 cells alien temp %allot + temp \ f type-number %load-immediate + temp dst 1 alien@ STW + temp dst 2 alien@ STW + displacement dst 3 alien@ STW + displacement dst 4 alien@ STW "f" resolve-label ] with-scope ; @@ -323,7 +318,7 @@ M:: ppc %box-displaced-alien ( dst displacement base displacement' base' base-cl displacement' :> temp dst 4 cells alien temp %allot ! If base is already a displaced alien, unpack it - 0 base \ f tag-number CMPI + 0 base \ f type-number CMPI "simple-case" get BEQ temp base header-offset LWZ 0 temp alien type-number tag-fixnum CMPI @@ -343,7 +338,7 @@ M:: ppc %box-displaced-alien ( dst displacement base displacement' base' base-cl ! Store offset displacement' dst 3 alien@ STW ! Store expired slot (its ok to clobber displacement') - temp \ f tag-number %load-immediate + temp \ f type-number %load-immediate temp dst 2 alien@ STW "end" resolve-label ] with-scope ; @@ -382,7 +377,7 @@ M: ppc %set-alien-double -rot STFD ; scratch-reg dst 0 STW ; : store-tagged ( dst tag -- ) - dupd tag-number ORI ; + dupd type-number ORI ; M:: ppc %allot ( dst size class nursery-ptr -- ) nursery-ptr dst load-allot-ptr @@ -460,7 +455,7 @@ M: ppc %epilogue ( n -- ) :: (%boolean) ( dst temp branch1 branch2 -- ) "end" define-label - dst \ f tag-number %load-immediate + dst \ f type-number %load-immediate "end" get branch1 execute( label -- ) branch2 [ "end" get branch2 execute( label -- ) ] when dst \ t %load-reference diff --git a/basis/cpu/x86/32/bootstrap.factor b/basis/cpu/x86/32/bootstrap.factor index e532d42dfe..f777040e86 100644 --- a/basis/cpu/x86/32/bootstrap.factor +++ b/basis/cpu/x86/32/bootstrap.factor @@ -21,7 +21,7 @@ IN: bootstrap.x86 : stack-reg ( -- reg ) ESP ; : ds-reg ( -- reg ) ESI ; : rs-reg ( -- reg ) EDI ; -: fixnum>slot@ ( -- ) temp0 1 SAR ; +: fixnum>slot@ ( -- ) temp0 2 SAR ; : rex-length ( -- n ) 0 ; [ diff --git a/basis/cpu/x86/64/bootstrap.factor b/basis/cpu/x86/64/bootstrap.factor index 662eaed3e0..0fc029fdfe 100644 --- a/basis/cpu/x86/64/bootstrap.factor +++ b/basis/cpu/x86/64/bootstrap.factor @@ -18,7 +18,7 @@ IN: bootstrap.x86 : stack-reg ( -- reg ) RSP ; : ds-reg ( -- reg ) R14 ; : rs-reg ( -- reg ) R15 ; -: fixnum>slot@ ( -- ) ; +: fixnum>slot@ ( -- ) temp0 1 SAR ; : rex-length ( -- n ) 1 ; [ diff --git a/basis/cpu/x86/bootstrap.factor b/basis/cpu/x86/bootstrap.factor index 25a826cde4..98a5188962 100644 --- a/basis/cpu/x86/bootstrap.factor +++ b/basis/cpu/x86/bootstrap.factor @@ -60,7 +60,7 @@ big-endian off ! pop boolean ds-reg bootstrap-cell SUB ! compare boolean with f - temp0 \ f tag-number CMP + temp0 \ f type-number CMP ! jump to true branch if not equal 0 JNE rc-relative rt-xt jit-rel ! jump to false branch if equal @@ -154,7 +154,7 @@ big-endian off ! ! ! Polymorphic inline caches -! The PIC and megamorphic code stubs are not permitted to touch temp3. +! The PIC stubs are not permitted to touch temp3. ! Load a value from a stack position [ @@ -171,41 +171,15 @@ big-endian off ! The 'make' trick lets us compute the jump distance for the ! conditional branches there -! Hi-tag -[ - temp0 temp1 MOV - load-tag - temp1 object tag-number tag-fixnum CMP - [ temp1 temp0 object tag-number neg [+] MOV ] { } make - [ length JNE ] [ % ] bi -] pic-hi-tag jit-define - ! Tuple [ temp0 temp1 MOV load-tag - temp1 tuple tag-number tag-fixnum CMP - [ temp1 temp0 tuple tag-number neg bootstrap-cell + [+] MOV ] { } make + temp1 tuple type-number tag-fixnum CMP + [ temp1 temp0 tuple type-number neg bootstrap-cell + [+] MOV ] { } make [ length JNE ] [ % ] bi ] pic-tuple jit-define -! Hi-tag and tuple -[ - temp0 temp1 MOV - load-tag - ! If bits 2 and 3 are set, the tag is either 6 (object) or 7 (tuple) - temp1 BIN: 110 tag-fixnum CMP - [ - ! Untag temp0 - temp0 tag-mask get bitnot AND - ! Set temp1 to 0 for objects, and bootstrap-cell for tuples - temp1 1 tag-fixnum AND - bootstrap-cell 4 = [ temp1 1 SHR ] when - ! Load header cell or tuple layout cell - temp1 temp0 temp1 [+] MOV - ] [ ] make [ length JL ] [ % ] bi -] pic-hi-tag-tuple jit-define - [ temp1 HEX: ffffffff CMP rc-absolute rt-immediate jit-rel ] pic-check-tag jit-define @@ -213,7 +187,7 @@ big-endian off [ temp2 HEX: ffffffff MOV rc-absolute-cell rt-immediate jit-rel temp1 temp2 CMP -] pic-check jit-define +] pic-check-tuple jit-define [ 0 JE rc-relative rt-xt jit-rel ] pic-hit jit-define @@ -224,14 +198,7 @@ big-endian off temp0 0 MOV rc-absolute-cell rt-immediate jit-rel ! key = hashcode(class) temp2 temp1 MOV - temp2 3 SHR - temp3 temp1 MOV - temp3 8 SHR - temp2 temp3 ADD - temp3 temp1 MOV - temp3 13 SHR - temp2 temp3 ADD - temp2 bootstrap-cell 4 = 3 4 ? SHL + bootstrap-cell 4 = [ temp2 1 SHR ] when ! key &= cache.length - 1 temp2 mega-cache-size get 1 - bootstrap-cell * AND ! cache += array-start-offset @@ -417,7 +384,7 @@ big-endian off t jit-literal temp3 0 MOV rc-absolute-cell rt-immediate jit-rel ! load f - temp1 \ f tag-number MOV + temp1 \ f type-number MOV ! load first value temp0 ds-reg [] MOV ! adjust stack pointer @@ -547,7 +514,7 @@ big-endian off ds-reg bootstrap-cell SUB temp0 ds-reg [] OR temp0 tag-mask get AND - temp0 \ f tag-number MOV + temp0 \ f type-number MOV temp1 1 tag-fixnum MOV temp0 temp1 CMOVE ds-reg [] temp0 MOV diff --git a/basis/cpu/x86/x86.factor b/basis/cpu/x86/x86.factor index d5fd039a59..7d576c0b1c 100644 --- a/basis/cpu/x86/x86.factor +++ b/basis/cpu/x86/x86.factor @@ -179,46 +179,37 @@ M: x86 %unbox-alien ( dst src -- ) M:: x86 %unbox-any-c-ptr ( dst src temp -- ) [ - { "is-byte-array" "end" "start" } [ define-label ] each - dst 0 MOV + "end" define-label + ! Compute tag in temp register temp src MOV - ! We come back here with displaced aliens - "start" resolve-label + temp tag-mask get AND + dst 0 MOV ! Is the object f? - temp \ f tag-number CMP + src \ f type-number CMP "end" get JE - ! Is the object an alien? - temp header-offset [+] alien type-number tag-fixnum CMP - "is-byte-array" get JNE - ! If so, load the offset and add it to the address - dst temp alien-offset [+] ADD - ! Now recurse on the underlying alien - temp temp underlying-alien-offset [+] MOV - "start" get JMP - "is-byte-array" resolve-label - ! Add byte array address to address being computed - dst temp ADD ! Add an offset to start of byte array's data - dst byte-array-offset ADD + dst src byte-array-offset [+] LEA + ! Is the object an alien? + temp alien type-number CMP + "end" get JNE + ! If so, load the offset and add it to the address + dst src alien-offset [+] MOV "end" resolve-label ] with-scope ; -: alien@ ( reg n -- op ) cells alien tag-number - [+] ; - -:: %allot-alien ( dst displacement base temp -- ) - dst 4 cells alien temp %allot - dst 1 alien@ base MOV ! alien - dst 2 alien@ \ f tag-number MOV ! expired - dst 3 alien@ displacement MOV ! displacement - ; +: alien@ ( reg n -- op ) cells alien type-number - [+] ; M:: x86 %box-alien ( dst src temp -- ) [ "end" define-label - dst \ f tag-number MOV + dst \ f type-number MOV src 0 CMP "end" get JE - dst src \ f tag-number temp %allot-alien + dst 5 cells alien temp %allot + dst 1 alien@ \ f type-number MOV ! base + dst 2 alien@ \ f type-number MOV ! expired + dst 3 alien@ displacement MOV ! displacement + dst 4 alien@ displacement MOV ! address "end" resolve-label ] with-scope ; @@ -235,9 +226,10 @@ M:: x86 %box-displaced-alien ( dst displacement base displacement' base' base-cl ! If base is already a displaced alien, unpack it base' base MOV displacement' displacement MOV - base \ f tag-number CMP + base \ f type-number CMP "ok" get JE - base header-offset [+] alien type-number tag-fixnum CMP + ! XXX + base 0 [+] alien type-number tag-fixnum CMP "ok" get JNE ! displacement += base.displacement displacement' base 3 alien@ ADD @@ -245,7 +237,7 @@ M:: x86 %box-displaced-alien ( dst displacement base displacement' base' base-cl base' base 1 alien@ MOV "ok" resolve-label dst 1 alien@ base' MOV ! alien - dst 2 alien@ \ f tag-number MOV ! expired + dst 2 alien@ \ f type-number MOV ! expired dst 3 alien@ displacement' MOV ! displacement "end" resolve-label ] with-scope ; @@ -402,7 +394,7 @@ M: x86 %vm-field-ptr ( dst field -- ) [ [] ] [ type-number tag-fixnum ] bi* MOV ; : store-tagged ( dst tag -- ) - tag-number OR ; + type-number OR ; M:: x86 %allot ( dst size class nursery-ptr -- ) nursery-ptr dst load-allot-ptr @@ -444,7 +436,7 @@ M: x86 %alien-global ( dst symbol library -- ) M: x86 %epilogue ( n -- ) cell - incr-stack-reg ; :: %boolean ( dst temp word -- ) - dst \ f tag-number MOV + dst \ f type-number MOV temp 0 MOV \ t rc-absolute-cell rel-immediate dst temp word execute ; inline diff --git a/basis/io/buffers/buffers.factor b/basis/io/buffers/buffers.factor index 91524dd6e1..f45d3bb062 100644 --- a/basis/io/buffers/buffers.factor +++ b/basis/io/buffers/buffers.factor @@ -8,7 +8,7 @@ IN: io.buffers TUPLE: buffer { size fixnum } -{ ptr simple-alien } +{ ptr alien } { fill fixnum } { pos fixnum } disposed ; diff --git a/basis/stack-checker/known-words/known-words.factor b/basis/stack-checker/known-words/known-words.factor index d3a9d2d4ce..aea9c4b1ce 100644 --- a/basis/stack-checker/known-words/known-words.factor +++ b/basis/stack-checker/known-words/known-words.factor @@ -592,7 +592,7 @@ M: bad-executable summary \ set-alien-double { float c-ptr integer } { } define-primitive -\ alien-cell { c-ptr integer } { simple-c-ptr } define-primitive +\ alien-cell { c-ptr integer } { pinned-c-ptr } define-primitive \ alien-cell make-flushable \ set-alien-cell { c-ptr c-ptr integer } { } define-primitive diff --git a/basis/tools/time/time.factor b/basis/tools/time/time.factor index 3d0509e87d..af1b528051 100644 --- a/basis/tools/time/time.factor +++ b/basis/tools/time/time.factor @@ -13,18 +13,18 @@ IN: tools.time : dispatch-stats. ( stats -- ) "== Megamorphic caches ==" print nl - { "Hits" "Misses" } swap zip simple-table. ; + [ { "Hits" "Misses" } ] dip zip simple-table. ; : inline-cache-stats. ( stats -- ) "== Polymorphic inline caches ==" print nl 3 cut [ "- Transitions:" print - { "Cold to monomorphic" "Mono to polymorphic" "Poly to megamorphic" } swap zip + [ { "Cold to monomorphic" "Mono to polymorphic" "Poly to megamorphic" } ] dip zip simple-table. nl ] [ "- Type check stubs:" print - { "Tag only" "Hi-tag" "Tuple" "Hi-tag and tuple" } swap zip + [ { "Tag" "Tuple" } ] dip zip simple-table. ] bi* ; diff --git a/core/alien/alien.factor b/core/alien/alien.factor index 3f2b5f95bf..1c4a6cc168 100644 --- a/core/alien/alien.factor +++ b/core/alien/alien.factor @@ -4,19 +4,9 @@ USING: accessors assocs kernel math namespaces sequences system kernel.private byte-arrays arrays init ; IN: alien -! Some predicate classes used by the compiler for optimization -! purposes -PREDICATE: simple-alien < alien underlying>> not ; +PREDICATE: pinned-alien < alien underlying>> not ; -UNION: simple-c-ptr -simple-alien POSTPONE: f byte-array ; - -DEFER: pinned-c-ptr? - -PREDICATE: pinned-alien < alien underlying>> pinned-c-ptr? ; - -UNION: pinned-c-ptr - pinned-alien POSTPONE: f ; +UNION: pinned-c-ptr pinned-alien POSTPONE: f ; GENERIC: >c-ptr ( obj -- c-ptr ) @@ -33,7 +23,7 @@ M: alien expired? expired>> ; M: f expired? drop t ; : ( address -- alien ) - f { simple-c-ptr } declare ; inline + f { pinned-c-ptr } declare ; inline : ( -- alien ) -1 t >>expired ; inline diff --git a/core/bootstrap/layouts/layouts.factor b/core/bootstrap/layouts/layouts.factor index fef7ba2a83..e2d686a8db 100644 --- a/core/bootstrap/layouts/layouts.factor +++ b/core/bootstrap/layouts/layouts.factor @@ -7,32 +7,26 @@ kernel.private ; 16 data-alignment set -BIN: 111 tag-mask set -8 num-tags set -3 tag-bits set +BIN: 1111 tag-mask set +4 tag-bits set -15 num-types set +14 num-types set 32 mega-cache-size set H{ - { fixnum BIN: 000 } - { bignum BIN: 001 } - { array BIN: 010 } - { float BIN: 011 } - { quotation BIN: 100 } - { POSTPONE: f BIN: 101 } - { object BIN: 110 } - { hi-tag BIN: 110 } - { tuple BIN: 111 } -} tag-numbers set - -tag-numbers get H{ + { fixnum 0 } + { bignum 1 } + { array 2 } + { float 3 } + { quotation 4 } + { POSTPONE: f 5 } + { alien 6 } + { tuple 7 } { wrapper 8 } { byte-array 9 } { callstack 10 } { string 11 } { word 12 } { dll 13 } - { alien 14 } -} assoc-union type-numbers set +} type-numbers set diff --git a/core/bootstrap/primitives.factor b/core/bootstrap/primitives.factor index 81c09f19fa..92f6c6f551 100644 --- a/core/bootstrap/primitives.factor +++ b/core/bootstrap/primitives.factor @@ -177,10 +177,6 @@ bi "object?" "kernel" vocab-words delete-at -! Class of objects with object tag -"hi-tag" "kernel.private" create -builtins get num-tags get tail define-union-class - ! Empty class with no instances "null" "kernel" create [ f { } f union-class define-class ] diff --git a/core/classes/algebra/algebra-docs.factor b/core/classes/algebra/algebra-docs.factor index 1b2ea7dfd4..65e6f85678 100644 --- a/core/classes/algebra/algebra-docs.factor +++ b/core/classes/algebra/algebra-docs.factor @@ -17,7 +17,6 @@ ARTICLE: "class-operations" "Class operations" flatten-class flatten-builtin-class class-types - class-tags } ; ARTICLE: "class-linearization" "Class linearization" diff --git a/core/classes/algebra/algebra-tests.factor b/core/classes/algebra/algebra-tests.factor index 855a15b66f..72c2dd575c 100644 --- a/core/classes/algebra/algebra-tests.factor +++ b/core/classes/algebra/algebra-tests.factor @@ -95,8 +95,6 @@ UNION: z1 b1 c1 ; [ f ] [ a1 c1 class-or b1 c1 class-or class-and a1 b1 class-or classes-intersect? ] unit-test -[ f ] [ growable \ hi-tag classes-intersect? ] unit-test - [ t ] [ growable tuple sequence class-and class<= ] unit-test diff --git a/core/classes/algebra/algebra.factor b/core/classes/algebra/algebra.factor index afaae444bc..06857d3c71 100755 --- a/core/classes/algebra/algebra.factor +++ b/core/classes/algebra/algebra.factor @@ -237,11 +237,5 @@ M: anonymous-union (flatten-class) flatten-builtin-class keys [ "type" word-prop ] map natural-sort ; -: class-tags ( class -- seq ) - class-types [ - dup num-tags get >= - [ drop \ hi-tag tag-number ] when - ] map prune ; - -: class-tag ( class -- tag/f ) - class-tags dup length 1 = [ first ] [ drop f ] if ; +: class-type ( class -- tag/f ) + class-types dup length 1 = [ first ] [ drop f ] if ; diff --git a/core/classes/builtin/builtin.factor b/core/classes/builtin/builtin.factor index 8eeb4ce357..6185e4f24d 100644 --- a/core/classes/builtin/builtin.factor +++ b/core/classes/builtin/builtin.factor @@ -12,34 +12,20 @@ PREDICATE: builtin-class < class : class>type ( class -- n ) "type" word-prop ; foldable -PREDICATE: lo-tag-class < builtin-class class>type 7 <= ; - -PREDICATE: hi-tag-class < builtin-class class>type 7 > ; - : type>class ( n -- class ) builtins get-global nth ; : bootstrap-type>class ( n -- class ) builtins get nth ; -M: hi-tag class hi-tag type>class ; inline - M: object class tag type>class ; inline M: builtin-class rank-class drop 0 ; GENERIC: define-builtin-predicate ( class -- ) -M: lo-tag-class define-builtin-predicate +M: builtin-class define-builtin-predicate dup class>type [ eq? ] curry [ tag ] prepend define-predicate ; -M: hi-tag-class define-builtin-predicate - dup class>type [ eq? ] curry [ hi-tag ] prepend 1quotation - [ dup tag 6 eq? ] [ [ drop f ] if ] surround - define-predicate ; - -M: lo-tag-class instance? [ tag ] [ class>type ] bi* eq? ; - -M: hi-tag-class instance? - over tag 6 eq? [ [ hi-tag ] [ class>type ] bi* eq? ] [ 2drop f ] if ; +M: builtin-class instance? [ tag ] [ class>type ] bi* eq? ; M: builtin-class (flatten-class) dup set ; diff --git a/core/classes/classes-tests.factor b/core/classes/classes-tests.factor index 5607bc3a22..10a5f674bd 100644 --- a/core/classes/classes-tests.factor +++ b/core/classes/classes-tests.factor @@ -11,7 +11,6 @@ IN: classes.tests [ f ] [ 3 float instance? ] unit-test [ t ] [ 3 number instance? ] unit-test [ f ] [ 3 null instance? ] unit-test -[ t ] [ "hi" \ hi-tag instance? ] unit-test ! Regression GENERIC: method-forget-test ( obj -- obj ) diff --git a/core/generic/single/single.factor b/core/generic/single/single.factor index 9e773fe700..1434acf521 100644 --- a/core/generic/single/single.factor +++ b/core/generic/single/single.factor @@ -112,15 +112,6 @@ TUPLE: tuple-dispatch-engine echelons ; tuple bootstrap-word \ convert-methods ; -! 2.2 Convert hi-tag methods -TUPLE: hi-tag-dispatch-engine methods ; - -C: hi-tag-dispatch-engine - -: convert-hi-tag-methods ( assoc -- assoc' ) - \ hi-tag bootstrap-word - \ convert-methods ; - ! 3 Tag methods TUPLE: tag-dispatch-engine methods ; @@ -129,7 +120,6 @@ C: tag-dispatch-engine : ( assoc -- engine ) flatten-methods convert-tuple-methods - convert-hi-tag-methods ; ! ! ! Compile engine ! ! ! @@ -144,23 +134,12 @@ GENERIC: compile-engine ( engine -- obj ) : direct-dispatch-table ( assoc n -- table ) default get [ swap update ] keep ; -: lo-tag-number ( class -- n ) - "type" word-prop dup num-tags get iota member? - [ drop object tag-number ] unless ; +: tag-number ( class -- n ) "type" word-prop ; M: tag-dispatch-engine compile-engine methods>> compile-engines* - [ [ lo-tag-number ] dip ] assoc-map - num-tags get direct-dispatch-table ; - -: num-hi-tags ( -- n ) num-types get num-tags get - ; - -: hi-tag-number ( class -- n ) "type" word-prop ; - -M: hi-tag-dispatch-engine compile-engine - methods>> compile-engines* - [ [ hi-tag-number num-tags get - ] dip ] assoc-map - num-hi-tags direct-dispatch-table ; + [ [ tag-number ] dip ] assoc-map + num-types get direct-dispatch-table ; : build-fast-hash ( methods -- buckets ) >alist V{ } clone [ hashcode 1array ] distribute-buckets diff --git a/core/kernel/kernel-docs.factor b/core/kernel/kernel-docs.factor index f7ae292630..f70d9d4214 100644 --- a/core/kernel/kernel-docs.factor +++ b/core/kernel/kernel-docs.factor @@ -651,7 +651,7 @@ HELP: declare HELP: tag ( object -- n ) { $values { "object" object } { "n" "a tag number" } } -{ $description "Outputs an object's tag number, between zero and one less than " { $link num-tags } ". This is implementation detail and user code should call " { $link class } " instead." } ; +{ $description "Outputs an object's tag number, between zero and one less than " { $link num-types } ". This is implementation detail and user code should call " { $link class } " instead." } ; HELP: getenv ( n -- obj ) { $values { "n" "a non-negative integer" } { "obj" object } } diff --git a/core/kernel/kernel.factor b/core/kernel/kernel.factor index 22c96c4318..a0934c2b17 100644 --- a/core/kernel/kernel.factor +++ b/core/kernel/kernel.factor @@ -230,8 +230,6 @@ ERROR: assert got expect ; : declare ( spec -- ) drop ; -: hi-tag ( obj -- n ) { hi-tag } declare 0 slot ; inline - : do-primitive ( number -- ) "Improper primitive call" throw ; PRIVATE> diff --git a/core/layouts/layouts-docs.factor b/core/layouts/layouts-docs.factor index 8dd1e6901f..efea1ffb4e 100644 --- a/core/layouts/layouts-docs.factor +++ b/core/layouts/layouts-docs.factor @@ -7,18 +7,11 @@ HELP: tag-bits { $var-description "Number of least significant bits reserved for a type tag in a tagged pointer." } { $see-also tag } ; -HELP: num-tags -{ $var-description "Number of distinct pointer tags. This is one more than the maximum value from the " { $link tag } " primitive." } ; - HELP: tag-mask { $var-description "Taking the bitwise and of a tagged pointer with this mask leaves the tag." } ; HELP: num-types -{ $var-description "Number of distinct built-in types. This is one more than the maximum value from the " { $link hi-tag } " primitive." } ; - -HELP: tag-number -{ $values { "class" class } { "n" "an integer or " { $link f } } } -{ $description "Outputs the pointer tag for pointers to instances of " { $link class } ". Will output " { $link f } " if instances of this class are not identified by a distinct pointer tag." } ; +{ $var-description "Number of distinct built-in types. This is one more than the maximum value from the " { $link tag } " primitive." } ; HELP: type-number { $values { "class" class } { "n" "an integer or " { $link f } } } @@ -76,7 +69,7 @@ HELP: bootstrap-cell-bits ARTICLE: "layouts-types" "Type numbers" "Corresponding to every built-in class is a built-in type number. An object can be asked for its built-in type number:" -{ $subsections hi-tag } +{ $subsections tag } "Built-in type numbers can be converted to classes, and vice versa:" { $subsections type>class @@ -88,14 +81,10 @@ ARTICLE: "layouts-types" "Type numbers" ARTICLE: "layouts-tags" "Tagged pointers" "Every pointer stored on the stack or in the heap has a " { $emphasis "tag" } ", which is a small integer identifying the type of the pointer. If the tag is not equal to one of the two special tags, the remaining bits contain the memory address of a heap-allocated object. The two special tags are the " { $link fixnum } " tag and the " { $link f } " tag." $nl -"Getting the tag of an object:" -{ $link tag } "Words for working with tagged pointers:" { $subsections tag-bits - num-tags tag-mask - tag-number } "The Factor VM does not actually expose any words for working with tagged pointers directly. The above words operate on integers; they are used in the bootstrap image generator and the optimizing compiler." ; diff --git a/core/layouts/layouts.factor b/core/layouts/layouts.factor index 2f0fa12d44..426bd560bf 100644 --- a/core/layouts/layouts.factor +++ b/core/layouts/layouts.factor @@ -8,14 +8,10 @@ SYMBOL: data-alignment SYMBOL: tag-mask -SYMBOL: num-tags - SYMBOL: tag-bits SYMBOL: num-types -SYMBOL: tag-numbers - SYMBOL: type-numbers SYMBOL: mega-cache-size @@ -23,9 +19,6 @@ SYMBOL: mega-cache-size : type-number ( class -- n ) type-numbers get at ; -: tag-number ( class -- n ) - type-number dup num-tags get >= [ drop object tag-number ] when ; - : tag-fixnum ( n -- tagged ) tag-bits get shift ; diff --git a/vm/alien.cpp b/vm/alien.cpp index ed3adf5c9b..4171c99d62 100755 --- a/vm/alien.cpp +++ b/vm/alien.cpp @@ -14,7 +14,10 @@ char *factor_vm::pinned_alien_offset(cell obj) alien *ptr = untag(obj); if(to_boolean(ptr->expired)) general_error(ERROR_EXPIRED,obj,false_object,NULL); - return pinned_alien_offset(ptr->base) + ptr->displacement; + if(to_boolean(ptr->base)) + type_error(ALIEN_TYPE,obj); + else + return (char *)ptr->address; } case F_TYPE: return NULL; @@ -41,6 +44,7 @@ cell factor_vm::allot_alien(cell delegate_, cell displacement) new_alien->displacement = displacement; new_alien->expired = false_object; + new_alien->update_address(); return new_alien.value(); } @@ -168,12 +172,7 @@ char *factor_vm::alien_offset(cell obj) case BYTE_ARRAY_TYPE: return untag(obj)->data(); case ALIEN_TYPE: - { - alien *ptr = untag(obj); - if(to_boolean(ptr->expired)) - general_error(ERROR_EXPIRED,obj,false_object,NULL); - return alien_offset(ptr->base) + ptr->displacement; - } + return (char *)untag(obj)->address; case F_TYPE: return NULL; default: diff --git a/vm/collector.hpp b/vm/collector.hpp index a52d5f97b1..29711aeb9c 100644 --- a/vm/collector.hpp +++ b/vm/collector.hpp @@ -111,9 +111,11 @@ template struct collector { workhorse.visit_handle(handle); } - void trace_slots(object *ptr) + void trace_object(object *ptr) { workhorse.visit_slots(ptr); + if(ptr->h.hi_tag() == ALIEN_TYPE) + ((alien *)ptr)->update_address(); } void trace_roots() diff --git a/vm/copying_collector.hpp b/vm/copying_collector.hpp index a21147ff0c..89501a3a4a 100644 --- a/vm/copying_collector.hpp +++ b/vm/copying_collector.hpp @@ -12,7 +12,7 @@ struct copying_collector : collector { { while(scan && scan < this->target->here) { - this->trace_slots((object *)scan); + this->trace_object((object *)scan); scan = this->target->next_object_after(scan); } } diff --git a/vm/cpu-ppc.S b/vm/cpu-ppc.S index 61b05a1735..1071d8b8a9 100644 --- a/vm/cpu-ppc.S +++ b/vm/cpu-ppc.S @@ -37,7 +37,7 @@ DEF(void,primitive_fixnum_multiply,(void *vm)): lwz r3,0(DS_REG) lwz r4,-4(DS_REG) subi DS_REG,DS_REG,4 - srawi r3,r3,3 + srawi r3,r3,4 mullwo. r6,r3,r4 bso multiply_overflow stw r6,0(DS_REG) diff --git a/vm/cpu-x86.S b/vm/cpu-x86.S index c497a0aad2..706369876f 100644 --- a/vm/cpu-x86.S +++ b/vm/cpu-x86.S @@ -25,7 +25,7 @@ DEF(void,primitive_fixnum_multiply,(void *myvm)): mov (DS_REG),ARITH_TEMP_1 mov ARITH_TEMP_1,DIV_RESULT mov -CELL_SIZE(DS_REG),ARITH_TEMP_2 - sar $3,ARITH_TEMP_2 + sar $4,ARITH_TEMP_2 sub $CELL_SIZE,DS_REG imul ARITH_TEMP_2 jo multiply_overflow diff --git a/vm/dispatch.cpp b/vm/dispatch.cpp index 333a49bfbe..30c4617cf0 100755 --- a/vm/dispatch.cpp +++ b/vm/dispatch.cpp @@ -70,16 +70,6 @@ cell factor_vm::lookup_tuple_method(cell obj, cell methods) return false_object; } -cell factor_vm::lookup_hi_tag_method(cell obj, cell methods) -{ - array *hi_tag_methods = untag(methods); - cell tag = untag(obj)->h.hi_tag() - HEADER_TYPE; -#ifdef FACTOR_DEBUG - assert(tag < TYPE_COUNT - HEADER_TYPE); -#endif - return array_nth(hi_tag_methods,tag); -} - cell factor_vm::lookup_method(cell obj, cell methods) { cell tag = TAG(obj); @@ -92,13 +82,6 @@ cell factor_vm::lookup_method(cell obj, cell methods) else return method; } - else if(tag == OBJECT_TYPE) - { - if(TAG(method) == ARRAY_TYPE) - return lookup_hi_tag_method(obj,method); - else - return method; - } else return method; } @@ -112,21 +95,17 @@ void factor_vm::primitive_lookup_method() cell factor_vm::object_class(cell obj) { - switch(TAG(obj)) - { - case TUPLE_TYPE: + cell tag = TAG(obj); + if(tag == TUPLE_TYPE) return untag(obj)->layout; - case OBJECT_TYPE: - return untag(obj)->h.value; - default: - return tag_fixnum(TAG(obj)); - } + else + return tag_fixnum(tag); } cell factor_vm::method_cache_hashcode(cell klass, array *array) { cell capacity = (array_capacity(array) >> 1) - 1; - return (((klass >> 3) + (klass >> 8) + (klass >> 13)) & capacity) << 1; + return ((klass >> TAG_BITS) & capacity) << 1; } void factor_vm::update_method_cache(cell cache, cell klass, cell method) @@ -174,7 +153,7 @@ void quotation_jit::emit_mega_cache_lookup(cell methods_, fixnum index, cell cac gc_root cache(cache_,parent); /* Generate machine code to determine the object's class. */ - emit_class_lookup(index,PIC_HI_TAG_TUPLE); + emit_class_lookup(index,PIC_TUPLE); /* Do a cache lookup. */ emit_with(parent->special_objects[MEGA_LOOKUP],cache.value()); diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index bbce01be76..369fc38f09 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -52,7 +52,7 @@ void factor_vm::collect_mark_impl(bool trace_contexts_p) { object *obj = mark_stack->back(); mark_stack->pop_back(); - collector.trace_slots(obj); + collector.trace_object(obj); code_marker.visit_object_code_block(obj); } diff --git a/vm/image.cpp b/vm/image.cpp index fce730df5a..0524a145a8 100755 --- a/vm/image.cpp +++ b/vm/image.cpp @@ -90,9 +90,12 @@ void factor_vm::fixup_quotation(quotation *quot, cell code_relocation_base) quot->xt = (void *)lazy_jit_compile; } -void factor_vm::fixup_alien(alien *d) +void factor_vm::fixup_alien(alien *ptr) { - if(!to_boolean(d->base)) d->expired = true_object; + if(!to_boolean(ptr->base)) + ptr->expired = true_object; + else + ptr->update_address(); } struct stack_frame_fixupper { diff --git a/vm/inline_cache.cpp b/vm/inline_cache.cpp index ee221c3797..3542a92b78 100755 --- a/vm/inline_cache.cpp +++ b/vm/inline_cache.cpp @@ -9,7 +9,8 @@ void factor_vm::init_inline_caching(int max_size) cold_call_to_ic_transitions = 0; ic_to_pic_transitions = 0; pic_to_mega_transitions = 0; - for(int i = 0; i < 4; i++) pic_counts[i] = 0; + pic_counts[0] = 0; + pic_counts[1] = 0; } void factor_vm::deallocate_inline_cache(cell return_address) @@ -29,39 +30,20 @@ void factor_vm::deallocate_inline_cache(cell return_address) it contains */ cell factor_vm::determine_inline_cache_type(array *cache_entries) { - bool seen_hi_tag = false, seen_tuple = false; + bool seen_tuple = false; cell i; for(i = 0; i < array_capacity(cache_entries); i += 2) { - cell klass = array_nth(cache_entries,i); - /* Is it a tuple layout? */ - switch(TAG(klass)) + if(TAG(array_nth(cache_entries,i)) == ARRAY_TYPE) { - case FIXNUM_TYPE: - { - fixnum type = untag_fixnum(klass); - if(type >= HEADER_TYPE) - seen_hi_tag = true; - } - break; - case ARRAY_TYPE: seen_tuple = true; break; - default: - critical_error("Expected a fixnum or array",klass); - break; } } - if(seen_hi_tag && seen_tuple) return PIC_HI_TAG_TUPLE; - if(seen_hi_tag && !seen_tuple) return PIC_HI_TAG; - if(!seen_hi_tag && seen_tuple) return PIC_TUPLE; - if(!seen_hi_tag && !seen_tuple) return PIC_TAG; - - critical_error("Oops",0); - return 0; + return seen_tuple ? PIC_TUPLE : PIC_TAG; } void factor_vm::update_pic_count(cell type) @@ -85,10 +67,10 @@ struct inline_cache_jit : public jit { void inline_cache_jit::emit_check(cell klass) { cell code_template; - if(TAG(klass) == FIXNUM_TYPE && untag_fixnum(klass) < HEADER_TYPE) + if(TAG(klass) == FIXNUM_TYPE) code_template = parent->special_objects[PIC_CHECK_TAG]; else - code_template = parent->special_objects[PIC_CHECK]; + code_template = parent->special_objects[PIC_CHECK_TUPLE]; emit_with(code_template,klass); } @@ -250,8 +232,8 @@ VM_C_API void *inline_cache_miss(cell return_address, factor_vm *parent) void factor_vm::primitive_reset_inline_cache_stats() { cold_call_to_ic_transitions = ic_to_pic_transitions = pic_to_mega_transitions = 0; - cell i; - for(i = 0; i < 4; i++) pic_counts[i] = 0; + pic_counts[0] = 0; + pic_counts[1] = 0; } void factor_vm::primitive_inline_cache_stats() @@ -260,9 +242,8 @@ void factor_vm::primitive_inline_cache_stats() stats.add(allot_cell(cold_call_to_ic_transitions)); stats.add(allot_cell(ic_to_pic_transitions)); stats.add(allot_cell(pic_to_mega_transitions)); - cell i; - for(i = 0; i < 4; i++) - stats.add(allot_cell(pic_counts[i])); + stats.add(allot_cell(pic_counts[0])); + stats.add(allot_cell(pic_counts[1])); stats.trim(); dpush(stats.elements.value()); } diff --git a/vm/layouts.hpp b/vm/layouts.hpp index f6c88064d4..2e4a90cc0e 100644 --- a/vm/layouts.hpp +++ b/vm/layouts.hpp @@ -27,8 +27,8 @@ static const cell data_alignment = 16; #define WORD_SIZE (signed)(sizeof(cell)*8) -#define TAG_MASK 7 -#define TAG_BITS 3 +#define TAG_MASK 15 +#define TAG_BITS 4 #define TAG(x) ((cell)(x) & TAG_MASK) #define UNTAG(x) ((cell)(x) & ~TAG_MASK) #define RETAG(x,tag) (UNTAG(x) | (tag)) @@ -40,23 +40,18 @@ static const cell data_alignment = 16; #define FLOAT_TYPE 3 #define QUOTATION_TYPE 4 #define F_TYPE 5 -#define OBJECT_TYPE 6 +#define ALIEN_TYPE 6 #define TUPLE_TYPE 7 - -#define HEADER_TYPE 8 /* anything less than this is a tag */ - -#define GC_COLLECTED 5 /* can be anything other than FIXNUM_TYPE */ - -/*** Header types ***/ #define WRAPPER_TYPE 8 #define BYTE_ARRAY_TYPE 9 #define CALLSTACK_TYPE 10 #define STRING_TYPE 11 #define WORD_TYPE 12 #define DLL_TYPE 13 -#define ALIEN_TYPE 14 -#define TYPE_COUNT 15 +#define TYPE_COUNT 14 + +#define GC_COLLECTED 5 /* can be anything other than FIXNUM_TYPE */ enum code_block_type { @@ -97,11 +92,6 @@ inline static cell tag_fixnum(fixnum untagged) return RETAG(untagged << TAG_BITS,FIXNUM_TYPE); } -inline static cell tag_for(cell type) -{ - return type < HEADER_TYPE ? type : OBJECT_TYPE; -} - struct object; struct header { @@ -334,6 +324,16 @@ struct alien : public object { cell expired; /* untagged */ cell displacement; + /* untagged */ + cell address; + + void update_address() + { + if(base == false_object) + address = displacement; + else + address = UNTAG(base) + sizeof(byte_array) + displacement; + } }; struct dll : public object { diff --git a/vm/run.hpp b/vm/run.hpp index 714ac1f64a..6ca2e50464 100755 --- a/vm/run.hpp +++ b/vm/run.hpp @@ -65,11 +65,9 @@ enum special_object { /* Polymorphic inline cache generation in inline_cache.c */ PIC_LOAD = 47, PIC_TAG, - PIC_HI_TAG, PIC_TUPLE, - PIC_HI_TAG_TUPLE, PIC_CHECK_TAG, - PIC_CHECK, + PIC_CHECK_TUPLE, PIC_HIT, PIC_MISS_WORD, PIC_MISS_TAIL_WORD, @@ -77,7 +75,7 @@ enum special_object { /* Megamorphic cache generation in dispatch.c */ MEGA_LOOKUP = 57, MEGA_LOOKUP_WORD, - MEGA_MISS_WORD, + MEGA_MISS_WORD, OBJ_UNDEFINED = 60, /* default quotation for undefined words */ diff --git a/vm/tagged.hpp b/vm/tagged.hpp index ea696c6358..77cb6e5287 100755 --- a/vm/tagged.hpp +++ b/vm/tagged.hpp @@ -3,12 +3,12 @@ namespace factor template cell tag(Type *value) { - return RETAG(value,tag_for(Type::type_number)); + return RETAG(value,Type::type_number); } inline static cell tag_dynamic(object *value) { - return RETAG(value,tag_for(value->h.hi_tag())); + return RETAG(value,value->h.hi_tag()); } template @@ -17,11 +17,7 @@ struct tagged cell value_; cell type() const { - cell tag = TAG(value_); - if(tag == OBJECT_TYPE) - return ((object *)UNTAG(value_))->h.hi_tag(); - else - return tag; + return TAG(value_); } bool type_p(cell type_) const diff --git a/vm/to_tenured_collector.cpp b/vm/to_tenured_collector.cpp index 6067bf1bf4..0cee748205 100644 --- a/vm/to_tenured_collector.cpp +++ b/vm/to_tenured_collector.cpp @@ -16,7 +16,7 @@ void to_tenured_collector::tenure_reachable_objects() { object *obj = mark_stack->back(); mark_stack->pop_back(); - this->trace_slots(obj); + this->trace_object(obj); } } diff --git a/vm/vm.hpp b/vm/vm.hpp index 5cb11c12f7..d58ce37742 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -83,8 +83,8 @@ struct factor_vm cell cold_call_to_ic_transitions; cell ic_to_pic_transitions; cell pic_to_mega_transitions; - /* Indexed by PIC_TAG, PIC_HI_TAG, PIC_TUPLE, PIC_HI_TAG_TUPLE */ - cell pic_counts[4]; + /* Indexed by PIC_TAG, PIC_TUPLE */ + cell pic_counts[2]; /* Number of entries in a polymorphic inline cache */ cell max_pic_size; @@ -619,7 +619,6 @@ struct factor_vm cell nth_superclass(tuple_layout *layout, fixnum echelon); cell nth_hashcode(tuple_layout *layout, fixnum echelon); cell lookup_tuple_method(cell obj, cell methods); - cell lookup_hi_tag_method(cell obj, cell methods); cell lookup_method(cell obj, cell methods); void primitive_lookup_method(); cell object_class(cell obj); From a3b5c07e8049bc62acc25c2811f0ef550fc64d20 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Mon, 2 Nov 2009 17:41:36 -0600 Subject: [PATCH 53/74] Minor bug fixes for 4-bit tags --- basis/compiler/cfg/gc-checks/gc-checks.factor | 4 ++-- basis/compiler/cfg/intrinsics/slots/slots.factor | 2 +- basis/cpu/x86/x86.factor | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/basis/compiler/cfg/gc-checks/gc-checks.factor b/basis/compiler/cfg/gc-checks/gc-checks.factor index 29616aaf8f..6d192ec54a 100644 --- a/basis/compiler/cfg/gc-checks/gc-checks.factor +++ b/basis/compiler/cfg/gc-checks/gc-checks.factor @@ -1,7 +1,7 @@ ! Copyright (C) 2009 Slava Pestov. ! See http://factorcode.org/license.txt for BSD license. USING: accessors kernel sequences assocs fry math -cpu.architecture layouts +cpu.architecture layouts namespaces compiler.cfg.rpo compiler.cfg.registers compiler.cfg.instructions @@ -28,7 +28,7 @@ M: ##box-displaced-alien allocation-size* drop 5 cells ; : allocation-size ( bb -- n ) instructions>> [ ##allocation? ] filter - [ allocation-size* data-alignment align ] map-sum ; + [ allocation-size* data-alignment get align ] map-sum ; : insert-gc-check ( bb -- ) dup dup '[ diff --git a/basis/compiler/cfg/intrinsics/slots/slots.factor b/basis/compiler/cfg/intrinsics/slots/slots.factor index ad7891b78d..1424aba354 100644 --- a/basis/compiler/cfg/intrinsics/slots/slots.factor +++ b/basis/compiler/cfg/intrinsics/slots/slots.factor @@ -8,7 +8,7 @@ compiler.cfg.instructions compiler.cfg.utilities compiler.cfg.builder.blocks compiler.constants ; IN: compiler.cfg.intrinsics.slots -: value-tag ( info -- n ) class>> type-number ; inline +: value-tag ( info -- n ) class>> class-type ; inline : ^^tag-offset>slot ( slot tag -- vreg' ) [ ^^offset>slot ] dip ^^sub-imm ; diff --git a/basis/cpu/x86/x86.factor b/basis/cpu/x86/x86.factor index 7d576c0b1c..fc56113e90 100644 --- a/basis/cpu/x86/x86.factor +++ b/basis/cpu/x86/x86.factor @@ -208,8 +208,8 @@ M:: x86 %box-alien ( dst src temp -- ) dst 5 cells alien temp %allot dst 1 alien@ \ f type-number MOV ! base dst 2 alien@ \ f type-number MOV ! expired - dst 3 alien@ displacement MOV ! displacement - dst 4 alien@ displacement MOV ! address + dst 3 alien@ src MOV ! displacement + dst 4 alien@ src MOV ! address "end" resolve-label ] with-scope ; From 6c2c87758a6ec5e4ada02a3f68fa5bea6087134c Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Mon, 2 Nov 2009 18:10:34 -0600 Subject: [PATCH 54/74] vm: rename gc_root to data_root, add code_root to fix a problem where code blocks would move underneath the PIC compiler if PIC compiler allocated enough --- vm/alien.cpp | 12 ++++---- vm/arrays.cpp | 28 +++++++++--------- vm/arrays.hpp | 2 +- vm/byte_arrays.cpp | 2 +- vm/byte_arrays.hpp | 2 +- vm/callstack.cpp | 10 +++---- vm/callstack.hpp | 2 +- vm/code_block.cpp | 10 +++---- vm/code_heap.cpp | 12 ++++---- vm/compaction.cpp | 4 +++ vm/dispatch.cpp | 4 +-- vm/errors.cpp | 4 +-- vm/full_collector.cpp | 51 +++++++++++++++++++++++++++++++++ vm/gc.cpp | 14 ++++----- vm/gc.hpp | 2 +- vm/generic_arrays.hpp | 2 +- vm/image.cpp | 4 +-- vm/inline_cache.cpp | 66 +++++++++++++++++++++++++------------------ vm/io.cpp | 6 ++-- vm/jit.cpp | 10 +++---- vm/jit.hpp | 8 +++--- vm/layouts.hpp | 9 +++--- vm/local_roots.hpp | 46 ------------------------------ vm/master.hpp | 3 +- vm/profiler.cpp | 4 +-- vm/quotations.cpp | 16 +++++------ vm/quotations.hpp | 2 +- vm/run.cpp | 2 +- vm/slot_visitor.hpp | 16 +++++------ vm/strings.cpp | 10 +++---- vm/tagged.hpp | 13 ++++++--- vm/tuples.cpp | 4 +-- vm/vm.hpp | 15 ++++++---- vm/words.cpp | 10 +++---- 34 files changed, 218 insertions(+), 187 deletions(-) delete mode 100644 vm/local_roots.hpp diff --git a/vm/alien.cpp b/vm/alien.cpp index 4171c99d62..d07b6e353b 100755 --- a/vm/alien.cpp +++ b/vm/alien.cpp @@ -30,8 +30,8 @@ char *factor_vm::pinned_alien_offset(cell obj) /* make an alien */ cell factor_vm::allot_alien(cell delegate_, cell displacement) { - gc_root delegate(delegate_,this); - gc_root new_alien(allot(sizeof(alien)),this); + data_root delegate(delegate_,this); + data_root new_alien(allot(sizeof(alien)),this); if(delegate.type_p(ALIEN_TYPE)) { @@ -117,9 +117,9 @@ DEFINE_ALIEN_ACCESSOR(cell,void *,box_alien,pinned_alien_offset) /* open a native library and push a handle */ void factor_vm::primitive_dlopen() { - gc_root path(dpop(),this); + data_root path(dpop(),this); path.untag_check(this); - gc_root library(allot(sizeof(dll)),this); + data_root library(allot(sizeof(dll)),this); library->path = path.value(); ffi_dlopen(library.untagged()); dpush(library.value()); @@ -128,8 +128,8 @@ void factor_vm::primitive_dlopen() /* look up a symbol in a native library */ void factor_vm::primitive_dlsym() { - gc_root library(dpop(),this); - gc_root name(dpop(),this); + data_root library(dpop(),this); + data_root name(dpop(),this); name.untag_check(this); symbol_char *sym = name->data(); diff --git a/vm/arrays.cpp b/vm/arrays.cpp index 3c69368f29..3060ac70a3 100644 --- a/vm/arrays.cpp +++ b/vm/arrays.cpp @@ -6,8 +6,8 @@ namespace factor /* make a new array with an initial element */ array *factor_vm::allot_array(cell capacity, cell fill_) { - gc_root fill(fill_,this); - gc_root new_array(allot_uninitialized_array(capacity),this); + data_root fill(fill_,this); + data_root new_array(allot_uninitialized_array(capacity),this); memset_cell(new_array->data(),fill.value(),capacity * sizeof(cell)); return new_array.untagged(); } @@ -22,17 +22,17 @@ void factor_vm::primitive_array() cell factor_vm::allot_array_1(cell obj_) { - gc_root obj(obj_,this); - gc_root a(allot_uninitialized_array(1),this); + data_root obj(obj_,this); + data_root a(allot_uninitialized_array(1),this); set_array_nth(a.untagged(),0,obj.value()); return a.value(); } cell factor_vm::allot_array_2(cell v1_, cell v2_) { - gc_root v1(v1_,this); - gc_root v2(v2_,this); - gc_root a(allot_uninitialized_array(2),this); + data_root v1(v1_,this); + data_root v2(v2_,this); + data_root a(allot_uninitialized_array(2),this); set_array_nth(a.untagged(),0,v1.value()); set_array_nth(a.untagged(),1,v2.value()); return a.value(); @@ -40,11 +40,11 @@ cell factor_vm::allot_array_2(cell v1_, cell v2_) cell factor_vm::allot_array_4(cell v1_, cell v2_, cell v3_, cell v4_) { - gc_root v1(v1_,this); - gc_root v2(v2_,this); - gc_root v3(v3_,this); - gc_root v4(v4_,this); - gc_root a(allot_uninitialized_array(4),this); + data_root v1(v1_,this); + data_root v2(v2_,this); + data_root v3(v3_,this); + data_root v4(v4_,this); + data_root a(allot_uninitialized_array(4),this); set_array_nth(a.untagged(),0,v1.value()); set_array_nth(a.untagged(),1,v2.value()); set_array_nth(a.untagged(),2,v3.value()); @@ -62,7 +62,7 @@ void factor_vm::primitive_resize_array() void growable_array::add(cell elt_) { factor_vm *parent = elements.parent; - gc_root elt(elt_,parent); + data_root elt(elt_,parent); if(count == array_capacity(elements.untagged())) elements = parent->reallot_array(elements.untagged(),count * 2); @@ -72,7 +72,7 @@ void growable_array::add(cell elt_) void growable_array::append(array *elts_) { factor_vm *parent = elements.parent; - gc_root elts(elts_,parent); + data_root elts(elts_,parent); cell capacity = array_capacity(elts.untagged()); if(count + capacity > array_capacity(elements.untagged())) { diff --git a/vm/arrays.hpp b/vm/arrays.hpp index 6063269e7f..8eb2b530b0 100755 --- a/vm/arrays.hpp +++ b/vm/arrays.hpp @@ -23,7 +23,7 @@ inline void factor_vm::set_array_nth(array *array, cell slot, cell value) struct growable_array { cell count; - gc_root elements; + data_root elements; explicit growable_array(factor_vm *parent, cell capacity = 10) : count(0), elements(parent->allot_array(capacity,false_object),parent) {} diff --git a/vm/byte_arrays.cpp b/vm/byte_arrays.cpp index fa02ede6c3..b317c39f62 100644 --- a/vm/byte_arrays.cpp +++ b/vm/byte_arrays.cpp @@ -43,7 +43,7 @@ void growable_byte_array::append_bytes(void *elts, cell len) void growable_byte_array::append_byte_array(cell byte_array_) { - gc_root byte_array(byte_array_,elements.parent); + data_root byte_array(byte_array_,elements.parent); cell len = array_capacity(byte_array.untagged()); cell new_size = count + len; diff --git a/vm/byte_arrays.hpp b/vm/byte_arrays.hpp index 0817f7dcdc..a3d6fcf941 100755 --- a/vm/byte_arrays.hpp +++ b/vm/byte_arrays.hpp @@ -3,7 +3,7 @@ namespace factor struct growable_byte_array { cell count; - gc_root elements; + data_root elements; explicit growable_byte_array(factor_vm *parent,cell capacity = 40) : count(0), elements(parent->allot_byte_array(capacity),parent) { } diff --git a/vm/callstack.cpp b/vm/callstack.cpp index 623db8a3fe..85f392af0e 100755 --- a/vm/callstack.cpp +++ b/vm/callstack.cpp @@ -130,8 +130,8 @@ struct stack_frame_accumulator { void operator()(stack_frame *frame) { - gc_root executing(parent->frame_executing(frame),parent); - gc_root scan(parent->frame_scan(frame),parent); + data_root executing(parent->frame_executing(frame),parent); + data_root scan(parent->frame_scan(frame),parent); frames.add(executing.value()); frames.add(scan.value()); @@ -142,7 +142,7 @@ struct stack_frame_accumulator { void factor_vm::primitive_callstack_to_array() { - gc_root callstack(dpop(),this); + data_root callstack(dpop(),this); stack_frame_accumulator accum(this); iterate_callstack_object(callstack.untagged(),accum); @@ -184,8 +184,8 @@ void factor_vm::primitive_innermost_stack_frame_scan() void factor_vm::primitive_set_innermost_stack_frame_quot() { - gc_root callstack(dpop(),this); - gc_root quot(dpop(),this); + data_root callstack(dpop(),this); + data_root quot(dpop(),this); callstack.untag_check(this); quot.untag_check(this); diff --git a/vm/callstack.hpp b/vm/callstack.hpp index 4d44904281..76bf3ecea4 100755 --- a/vm/callstack.hpp +++ b/vm/callstack.hpp @@ -12,7 +12,7 @@ VM_ASM_API void save_callstack_bottom(stack_frame *callstack_bottom, factor_vm * keep the callstack in a GC root and use relative offsets */ template void factor_vm::iterate_callstack_object(callstack *stack_, Iterator &iterator) { - gc_root stack(stack_,this); + data_root stack(stack_,this); fixnum frame_offset = untag_fixnum(stack->length) - sizeof(stack_frame); while(frame_offset >= 0) diff --git a/vm/code_block.cpp b/vm/code_block.cpp index d3670af3c0..9869630732 100755 --- a/vm/code_block.cpp +++ b/vm/code_block.cpp @@ -467,11 +467,11 @@ code_block *factor_vm::allot_code_block(cell size, code_block_type type) /* Might GC */ code_block *factor_vm::add_code_block(code_block_type type, cell code_, cell labels_, cell owner_, cell relocation_, cell literals_) { - gc_root code(code_,this); - gc_root labels(labels_,this); - gc_root owner(owner_,this); - gc_root relocation(relocation_,this); - gc_root literals(literals_,this); + data_root code(code_,this); + data_root labels(labels_,this); + data_root owner(owner_,this); + data_root relocation(relocation_,this); + data_root literals(literals_,this); cell code_length = array_capacity(code.untagged()); code_block *compiled = allot_code_block(code_length,type); diff --git a/vm/code_heap.cpp b/vm/code_heap.cpp index f3d36527b5..b4e071d644 100755 --- a/vm/code_heap.cpp +++ b/vm/code_heap.cpp @@ -73,8 +73,8 @@ bool factor_vm::in_code_heap_p(cell ptr) /* Compile a word definition with the non-optimizing compiler. Allocates memory */ void factor_vm::jit_compile_word(cell word_, cell def_, bool relocate) { - gc_root word(word_,this); - gc_root def(def_,this); + data_root word(word_,this); + data_root def(def_,this); jit_compile(def.value(),relocate); @@ -145,7 +145,7 @@ void factor_vm::relocate_code_heap() void factor_vm::primitive_modify_code_heap() { - gc_root alist(dpop(),this); + data_root alist(dpop(),this); cell count = array_capacity(alist.untagged()); @@ -155,10 +155,10 @@ void factor_vm::primitive_modify_code_heap() cell i; for(i = 0; i < count; i++) { - gc_root pair(array_nth(alist.untagged(),i),this); + data_root pair(array_nth(alist.untagged(),i),this); - gc_root word(array_nth(pair.untagged(),0),this); - gc_root data(array_nth(pair.untagged(),1),this); + data_root word(array_nth(pair.untagged(),0),this); + data_root data(array_nth(pair.untagged(),1),this); switch(data.type()) { diff --git a/vm/compaction.cpp b/vm/compaction.cpp index 86d3efd3d6..10e37db263 100644 --- a/vm/compaction.cpp +++ b/vm/compaction.cpp @@ -139,6 +139,8 @@ void factor_vm::collect_compact_impl(bool trace_contexts_p) code_forwarder.visit_callback_code_blocks(); } + update_code_roots_for_compaction(); + current_gc->event->ended_compaction(); } @@ -182,6 +184,8 @@ void factor_vm::collect_compact_code_impl(bool trace_contexts_p) code_block_compaction_updater code_block_updater(this,slot_forwarder); standard_sizer code_block_sizer; code->allocator->compact(code_block_updater,code_block_sizer); + + update_code_roots_for_compaction(); } } diff --git a/vm/dispatch.cpp b/vm/dispatch.cpp index 30c4617cf0..6bef970944 100755 --- a/vm/dispatch.cpp +++ b/vm/dispatch.cpp @@ -149,8 +149,8 @@ void factor_vm::primitive_dispatch_stats() void quotation_jit::emit_mega_cache_lookup(cell methods_, fixnum index, cell cache_) { - gc_root methods(methods_,parent); - gc_root cache(cache_,parent); + data_root methods(methods_,parent); + data_root cache(cache_,parent); /* Generate machine code to determine the object's class. */ emit_class_lookup(index,PIC_TUPLE); diff --git a/vm/errors.cpp b/vm/errors.cpp index 3161f625cd..2bbe3e5328 100755 --- a/vm/errors.cpp +++ b/vm/errors.cpp @@ -37,8 +37,8 @@ void factor_vm::throw_error(cell error, stack_frame *callstack_top) gc_off = false; /* Reset local roots */ - gc_locals.clear(); - gc_bignums.clear(); + data_roots.clear(); + bignum_roots.clear(); /* If we had an underflow or overflow, stack pointers might be out of bounds */ diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index 369fc38f09..acce777301 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -9,6 +9,56 @@ full_collector::full_collector(factor_vm *parent_) : parent_->data->tenured, full_policy(parent_)) {} +/* After a sweep, invalidate any code heap roots which are not marked, +so that if a block makes a tail call to a generic word, and the PIC +compiler triggers a GC, and the caller block gets gets GCd as a result, +the PIC code won't try to overwrite the call site */ +void factor_vm::update_code_roots_for_sweep() +{ + std::vector::const_iterator iter = code_roots.begin(); + std::vector::const_iterator end = code_roots.end(); + + mark_bits *state = &code->allocator->state; + + for(; iter < end; iter++) + { + printf("We have a code root!\n"); + code_root *root = *iter; + code_block *block = (code_block *)(root->value & -block_granularity); + if(root->valid && !state->marked_p(block)) + root->valid = false; + } +} + +/* After a compaction, invalidate any code heap roots which are not +marked as above, and also slide the valid roots up so that call sites +can be updated correctly. */ +void factor_vm::update_code_roots_for_compaction() +{ + std::vector::const_iterator iter = code_roots.begin(); + std::vector::const_iterator end = code_roots.end(); + + mark_bits *state = &code->allocator->state; + + for(; iter < end; iter++) + { + printf("We have a code root - compaction!\n"); + code_root *root = *iter; + code_block *block = (code_block *)(root->value & -block_granularity); + + /* Offset of return address within 16-byte allocation line */ + cell offset = root->value - (cell)block; + + if(root->valid && state->marked_p((code_block *)root->value)) + { + block = state->forward_block(block); + root->value = (cell)block + offset; + } + else + root->valid = false; + } +} + struct code_block_marker { code_heap *code; full_collector *collector; @@ -66,6 +116,7 @@ void factor_vm::collect_sweep_impl() { current_gc->event->started_data_sweep(); data->tenured->sweep(); + update_code_roots_for_sweep(); current_gc->event->ended_data_sweep(); } diff --git a/vm/gc.cpp b/vm/gc.cpp index afc1798911..06e9d78ce2 100755 --- a/vm/gc.cpp +++ b/vm/gc.cpp @@ -249,20 +249,20 @@ void factor_vm::primitive_become() compile_all_words(); } -void factor_vm::inline_gc(cell *gc_roots_base, cell gc_roots_size) +void factor_vm::inline_gc(cell *data_roots_base, cell data_roots_size) { - for(cell i = 0; i < gc_roots_size; i++) - gc_locals.push_back((cell)&gc_roots_base[i]); + for(cell i = 0; i < data_roots_size; i++) + data_roots.push_back((cell)&data_roots_base[i]); primitive_minor_gc(); - for(cell i = 0; i < gc_roots_size; i++) - gc_locals.pop_back(); + for(cell i = 0; i < data_roots_size; i++) + data_roots.pop_back(); } -VM_C_API void inline_gc(cell *gc_roots_base, cell gc_roots_size, factor_vm *parent) +VM_C_API void inline_gc(cell *data_roots_base, cell data_roots_size, factor_vm *parent) { - parent->inline_gc(gc_roots_base,gc_roots_size); + parent->inline_gc(data_roots_base,data_roots_size); } /* diff --git a/vm/gc.hpp b/vm/gc.hpp index 7ad7e71524..a9250eddb2 100755 --- a/vm/gc.hpp +++ b/vm/gc.hpp @@ -53,6 +53,6 @@ struct gc_state { void start_again(gc_op op_, factor_vm *parent); }; -VM_C_API void inline_gc(cell *gc_roots_base, cell gc_roots_size, factor_vm *parent); +VM_C_API void inline_gc(cell *data_roots_base, cell data_roots_size, factor_vm *parent); } diff --git a/vm/generic_arrays.hpp b/vm/generic_arrays.hpp index 89eb56a70d..f45785f9ef 100755 --- a/vm/generic_arrays.hpp +++ b/vm/generic_arrays.hpp @@ -33,7 +33,7 @@ template bool factor_vm::reallot_array_in_place_p(Array *array, template Array *factor_vm::reallot_array(Array *array_, cell capacity) { - gc_root array(array_,this); + data_root array(array_,this); if(reallot_array_in_place_p(array.untagged(),capacity)) { diff --git a/vm/image.cpp b/vm/image.cpp index 0524a145a8..b3a9eae7a5 100755 --- a/vm/image.cpp +++ b/vm/image.cpp @@ -311,7 +311,7 @@ void factor_vm::primitive_save_image() /* do a full GC to push everything into tenured space */ primitive_compact_gc(); - gc_root path(dpop(),this); + data_root path(dpop(),this); path.untag_check(this); save_image((vm_char *)(path.untagged() + 1)); } @@ -321,7 +321,7 @@ void factor_vm::primitive_save_image_and_exit() /* We unbox this before doing anything else. This is the only point where we might throw an error, so we have to throw an error here since later steps destroy the current image. */ - gc_root path(dpop(),this); + data_root path(dpop(),this); path.untag_check(this); /* strip out special_objects data which is set on startup anyway */ diff --git a/vm/inline_cache.cpp b/vm/inline_cache.cpp index 3542a92b78..fd5e93560b 100755 --- a/vm/inline_cache.cpp +++ b/vm/inline_cache.cpp @@ -83,9 +83,9 @@ void inline_cache_jit::compile_inline_cache(fixnum index, cell cache_entries_, bool tail_call_p) { - gc_root generic_word(generic_word_,parent); - gc_root methods(methods_,parent); - gc_root cache_entries(cache_entries_,parent); + data_root generic_word(generic_word_,parent); + data_root methods(methods_,parent); + data_root cache_entries(cache_entries_,parent); cell inline_cache_type = parent->determine_inline_cache_type(cache_entries.untagged()); parent->update_pic_count(inline_cache_type); @@ -118,11 +118,15 @@ void inline_cache_jit::compile_inline_cache(fixnum index, word_special(parent->special_objects[tail_call_p ? PIC_MISS_TAIL_WORD : PIC_MISS_WORD]); } -code_block *factor_vm::compile_inline_cache(fixnum index,cell generic_word_,cell methods_,cell cache_entries_,bool tail_call_p) +code_block *factor_vm::compile_inline_cache(fixnum index, + cell generic_word_, + cell methods_, + cell cache_entries_, + bool tail_call_p) { - gc_root generic_word(generic_word_,this); - gc_root methods(methods_,this); - gc_root cache_entries(cache_entries_,this); + data_root generic_word(generic_word_,this); + data_root methods(methods_,this); + data_root cache_entries(cache_entries_,this); inline_cache_jit jit(generic_word.value(),this); jit.compile_inline_cache(index, @@ -149,12 +153,12 @@ cell factor_vm::inline_cache_size(cell cache_entries) /* Allocates memory */ cell factor_vm::add_inline_cache_entry(cell cache_entries_, cell klass_, cell method_) { - gc_root cache_entries(cache_entries_,this); - gc_root klass(klass_,this); - gc_root method(method_,this); + data_root cache_entries(cache_entries_,this); + data_root klass(klass_,this); + data_root method(method_,this); cell pic_size = array_capacity(cache_entries.untagged()); - gc_root new_cache_entries(reallot_array(cache_entries.untagged(),pic_size + 2),this); + data_root new_cache_entries(reallot_array(cache_entries.untagged(),pic_size + 2),this); set_array_nth(new_cache_entries.untagged(),pic_size,klass.value()); set_array_nth(new_cache_entries.untagged(),pic_size + 1,method.value()); return new_cache_entries.value(); @@ -170,22 +174,27 @@ void factor_vm::update_pic_transitions(cell pic_size) ic_to_pic_transitions++; } -/* The cache_entries parameter is either f (on cold call site) or an array (on cache miss). -Called from assembly with the actual return address */ -void *factor_vm::inline_cache_miss(cell return_address) +/* The cache_entries parameter is either f (on cold call site) or an array +(on cache miss). Called from assembly with the actual return address. +Compilation of the inline cache may trigger a GC, which may trigger a compaction; +also, the block containing the return address may now be dead. Use a code_root +to take care of the details. */ +void *factor_vm::inline_cache_miss(cell return_address_) { - check_code_pointer(return_address); + code_root return_address(return_address_,this); + + check_code_pointer(return_address.value); /* Since each PIC is only referenced from a single call site, if the old call target was a PIC, we can deallocate it immediately, instead of leaving dead PICs around until the next GC. */ - deallocate_inline_cache(return_address); + deallocate_inline_cache(return_address.value); - gc_root cache_entries(dpop(),this); + data_root cache_entries(dpop(),this); fixnum index = untag_fixnum(dpop()); - gc_root methods(dpop(),this); - gc_root generic_word(dpop(),this); - gc_root object(((cell *)ds)[-index],this); + data_root methods(dpop(),this); + data_root generic_word(dpop(),this); + data_root object(((cell *)ds)[-index],this); void *xt; @@ -200,7 +209,7 @@ void *factor_vm::inline_cache_miss(cell return_address) cell klass = object_class(object.value()); cell method = lookup_method(object.value(),methods.value()); - gc_root new_cache_entries(add_inline_cache_entry( + data_root new_cache_entries(add_inline_cache_entry( cache_entries.value(), klass, method),this); @@ -208,18 +217,21 @@ void *factor_vm::inline_cache_miss(cell return_address) generic_word.value(), methods.value(), new_cache_entries.value(), - tail_call_site_p(return_address))->xt(); + tail_call_site_p(return_address.value))->xt(); } /* Install the new stub. */ - set_call_target(return_address,xt); + if(return_address.valid) + { + set_call_target(return_address.value,xt); #ifdef PIC_DEBUG - std::cout << "Updated " - << (tail_call_site_p(return_address) ? "tail" : "non-tail") - << " call site 0x" << std::hex << return_address << std::dec - << " with " << std::hex << (cell)xt << std::dec; + std::cout << "Updated " + << (tail_call_site_p(return_address) ? "tail" : "non-tail") + << " call site 0x" << std::hex << return_address << std::dec + << " with " << std::hex << (cell)xt << std::dec; #endif + } return xt; } diff --git a/vm/io.cpp b/vm/io.cpp index bbcac0b849..a8f9cb6897 100755 --- a/vm/io.cpp +++ b/vm/io.cpp @@ -33,8 +33,8 @@ void factor_vm::io_error() void factor_vm::primitive_fopen() { - gc_root mode(dpop(),this); - gc_root path(dpop(),this); + data_root mode(dpop(),this); + data_root path(dpop(),this); mode.untag_check(this); path.untag_check(this); @@ -88,7 +88,7 @@ void factor_vm::primitive_fread() return; } - gc_root buf(allot_uninitialized_array(size),this); + data_root buf(allot_uninitialized_array(size),this); for(;;) { diff --git a/vm/jit.cpp b/vm/jit.cpp index 2fa948e4d6..e72e88bfdf 100644 --- a/vm/jit.cpp +++ b/vm/jit.cpp @@ -24,7 +24,7 @@ jit::jit(code_block_type type_, cell owner_, factor_vm *vm) void jit::emit_relocation(cell code_template_) { - gc_root code_template(code_template_,parent); + data_root code_template(code_template_,parent); cell capacity = array_capacity(code_template.untagged()); for(cell i = 1; i < capacity; i += 3) { @@ -43,11 +43,11 @@ void jit::emit_relocation(cell code_template_) /* Allocates memory */ void jit::emit(cell code_template_) { - gc_root code_template(code_template_,parent); + data_root code_template(code_template_,parent); emit_relocation(code_template.value()); - gc_root insns(array_nth(code_template.untagged(),0),parent); + data_root insns(array_nth(code_template.untagged(),0),parent); if(computing_offset_p) { @@ -71,8 +71,8 @@ void jit::emit(cell code_template_) } void jit::emit_with(cell code_template_, cell argument_) { - gc_root code_template(code_template_,parent); - gc_root argument(argument_,parent); + data_root code_template(code_template_,parent); + data_root argument(argument_,parent); literal(argument.value()); emit(code_template.value()); } diff --git a/vm/jit.hpp b/vm/jit.hpp index 9feade4cc1..b5a2457d57 100644 --- a/vm/jit.hpp +++ b/vm/jit.hpp @@ -3,7 +3,7 @@ namespace factor struct jit { code_block_type type; - gc_root owner; + data_root owner; growable_byte_array code; growable_byte_array relocation; growable_array literals; @@ -28,7 +28,7 @@ struct jit { void word_jump(cell word_) { - gc_root word(word_,parent); + data_root word(word_,parent); literal(tag_fixnum(xt_tail_pic_offset)); literal(word.value()); emit(parent->special_objects[JIT_WORD_JUMP]); @@ -46,8 +46,8 @@ struct jit { void emit_subprimitive(cell word_) { - gc_root word(word_,parent); - gc_root code_pair(word->subprimitive,parent); + data_root word(word_,parent); + data_root code_pair(word->subprimitive,parent); literals.append(untag(array_nth(code_pair.untagged(),0))); emit(array_nth(code_pair.untagged(),1)); } diff --git a/vm/layouts.hpp b/vm/layouts.hpp index 2e4a90cc0e..6b96ade8f5 100644 --- a/vm/layouts.hpp +++ b/vm/layouts.hpp @@ -51,7 +51,7 @@ static const cell data_alignment = 16; #define TYPE_COUNT 14 -#define GC_COLLECTED 5 /* can be anything other than FIXNUM_TYPE */ +#define FORWARDING_POINTER 5 /* can be anything other than FIXNUM_TYPE */ enum code_block_type { @@ -117,7 +117,7 @@ struct header { bool forwarding_pointer_p() const { - return TAG(value) == GC_COLLECTED; + return TAG(value) == FORWARDING_POINTER; } object *forwarding_pointer() const @@ -127,7 +127,7 @@ struct header { void forward_to(object *pointer) { - value = RETAG(pointer,GC_COLLECTED); + value = RETAG(pointer,FORWARDING_POINTER); } }; @@ -344,8 +344,7 @@ struct dll : public object { void *dll; }; -struct stack_frame -{ +struct stack_frame { void *xt; /* Frame size in bytes */ cell size; diff --git a/vm/local_roots.hpp b/vm/local_roots.hpp deleted file mode 100644 index 442a91f350..0000000000 --- a/vm/local_roots.hpp +++ /dev/null @@ -1,46 +0,0 @@ -namespace factor -{ - -template -struct gc_root : public tagged -{ - factor_vm *parent; - - void push() { parent->gc_locals.push_back((cell)this); } - - explicit gc_root(cell value_,factor_vm *vm) : tagged(value_),parent(vm) { push(); } - explicit gc_root(Type *value_, factor_vm *vm) : tagged(value_),parent(vm) { push(); } - - const gc_root& operator=(const Type *x) { tagged::operator=(x); return *this; } - const gc_root& operator=(const cell &x) { tagged::operator=(x); return *this; } - - ~gc_root() { -#ifdef FACTOR_DEBUG - assert(parent->gc_locals.back() == (cell)this); -#endif - parent->gc_locals.pop_back(); - } -}; - -/* A similar hack for the bignum implementation */ -struct gc_bignum -{ - bignum **addr; - factor_vm *parent; - gc_bignum(bignum **addr_, factor_vm *vm) : addr(addr_), parent(vm) { - if(*addr_) - parent->check_data_pointer(*addr_); - parent->gc_bignums.push_back((cell)addr); - } - - ~gc_bignum() { -#ifdef FACTOR_DEBUG - assert(parent->gc_bignums.back() == (cell)addr); -#endif - parent->gc_bignums.pop_back(); - } -}; - -#define GC_BIGNUM(x) gc_bignum x##__gc_root(&x,this) - -} diff --git a/vm/master.hpp b/vm/master.hpp index 0f8276cf66..0654e63bcb 100755 --- a/vm/master.hpp +++ b/vm/master.hpp @@ -74,7 +74,8 @@ namespace factor #include "vm.hpp" #include "allot.hpp" #include "tagged.hpp" -#include "local_roots.hpp" +#include "data_roots.hpp" +#include "code_roots.hpp" #include "slot_visitor.hpp" #include "collector.hpp" #include "copying_collector.hpp" diff --git a/vm/profiler.cpp b/vm/profiler.cpp index 2f5fc6fcf4..50e88cc57a 100755 --- a/vm/profiler.cpp +++ b/vm/profiler.cpp @@ -11,7 +11,7 @@ void factor_vm::init_profiler() /* Allocates memory */ code_block *factor_vm::compile_profiling_stub(cell word_) { - gc_root word(word_,this); + data_root word(word_,this); jit jit(code_block_profiling,word.value(),this); jit.emit_with(special_objects[JIT_PROFILING],word.value()); @@ -31,7 +31,7 @@ void factor_vm::set_profiling(bool profiling) and allocate profiling blocks if necessary */ primitive_full_gc(); - gc_root words(find_all_words(),this); + data_root words(find_all_words(),this); cell i; cell length = array_capacity(words.untagged()); diff --git a/vm/quotations.cpp b/vm/quotations.cpp index 17b7c4328b..2c5e401ad9 100755 --- a/vm/quotations.cpp +++ b/vm/quotations.cpp @@ -110,7 +110,7 @@ bool quotation_jit::trivial_quotation_p(array *elements) void quotation_jit::emit_quot(cell quot_) { - gc_root quot(quot_,parent); + data_root quot(quot_,parent); array *elements = untag(quot->array); @@ -143,7 +143,7 @@ void quotation_jit::iterate_quotation() { set_position(i); - gc_root obj(array_nth(elements.untagged(),i),parent); + data_root obj(array_nth(elements.untagged(),i),parent); switch(obj.type()) { @@ -290,7 +290,7 @@ void factor_vm::set_quot_xt(quotation *quot, code_block *code) /* Allocates memory */ void factor_vm::jit_compile(cell quot_, bool relocating) { - gc_root quot(quot_,this); + data_root quot(quot_,this); if(quot->code) return; quotation_jit compiler(quot.value(),true,relocating,this); @@ -327,13 +327,13 @@ void factor_vm::primitive_quotation_xt() void factor_vm::compile_all_words() { - gc_root words(find_all_words(),this); + data_root words(find_all_words(),this); cell i; cell length = array_capacity(words.untagged()); for(i = 0; i < length; i++) { - gc_root word(array_nth(words.untagged(),i),this); + data_root word(array_nth(words.untagged(),i),this); if(!word->code || !word->code->optimized_p()) jit_compile_word(word.value(),word->def,false); @@ -348,8 +348,8 @@ void factor_vm::compile_all_words() /* Allocates memory */ fixnum factor_vm::quot_code_offset_to_scan(cell quot_, cell offset) { - gc_root quot(quot_,this); - gc_root array(quot->array,this); + data_root quot(quot_,this); + data_root array(quot->array,this); quotation_jit compiler(quot.value(),false,false,this); compiler.compute_position(offset); @@ -360,7 +360,7 @@ fixnum factor_vm::quot_code_offset_to_scan(cell quot_, cell offset) cell factor_vm::lazy_jit_compile_impl(cell quot_, stack_frame *stack) { - gc_root quot(quot_,this); + data_root quot(quot_,this); ctx->callstack_top = stack; jit_compile(quot.value(),true); return quot.value(); diff --git a/vm/quotations.hpp b/vm/quotations.hpp index e6e6afcd0b..6d04d80de3 100755 --- a/vm/quotations.hpp +++ b/vm/quotations.hpp @@ -2,7 +2,7 @@ namespace factor { struct quotation_jit : public jit { - gc_root elements; + data_root elements; bool compiling, relocate; explicit quotation_jit(cell quot, bool compiling_, bool relocate_, factor_vm *vm) diff --git a/vm/run.cpp b/vm/run.cpp index b6e3324502..6d3e9f7374 100755 --- a/vm/run.cpp +++ b/vm/run.cpp @@ -52,7 +52,7 @@ void factor_vm::primitive_load_locals() cell factor_vm::clone_object(cell obj_) { - gc_root obj(obj_,this); + data_root obj(obj_,this); if(immediate_p(obj.value())) return obj.value(); diff --git a/vm/slot_visitor.hpp b/vm/slot_visitor.hpp index 38d0081c0a..e777347622 100644 --- a/vm/slot_visitor.hpp +++ b/vm/slot_visitor.hpp @@ -42,19 +42,19 @@ template struct slot_visitor { visit_handle(ptr); } - void visit_registered_locals() + void visit_data_roots() { - std::vector::const_iterator iter = parent->gc_locals.begin(); - std::vector::const_iterator end = parent->gc_locals.end(); + std::vector::const_iterator iter = parent->data_roots.begin(); + std::vector::const_iterator end = parent->data_roots.end(); for(; iter < end; iter++) visit_handle((cell *)(*iter)); } - void visit_registered_bignums() + void visit_bignum_roots() { - std::vector::const_iterator iter = parent->gc_bignums.begin(); - std::vector::const_iterator end = parent->gc_bignums.end(); + std::vector::const_iterator iter = parent->bignum_roots.begin(); + std::vector::const_iterator end = parent->bignum_roots.end(); for(; iter < end; iter++) { @@ -72,8 +72,8 @@ template struct slot_visitor { visit_handle(&parent->bignum_pos_one); visit_handle(&parent->bignum_neg_one); - visit_registered_locals(); - visit_registered_bignums(); + visit_data_roots(); + visit_bignum_roots(); for(cell i = 0; i < special_object_count; i++) visit_handle(&parent->special_objects[i]); diff --git a/vm/strings.cpp b/vm/strings.cpp index 3022611319..9e135e6779 100644 --- a/vm/strings.cpp +++ b/vm/strings.cpp @@ -29,7 +29,7 @@ void factor_vm::set_string_nth_fast(string *str, cell index, cell ch) void factor_vm::set_string_nth_slow(string *str_, cell index, cell ch) { - gc_root str(str_,this); + data_root str(str_,this); byte_array *aux; @@ -78,7 +78,7 @@ string *factor_vm::allot_string_internal(cell capacity) /* Allocates memory */ void factor_vm::fill_string(string *str_, cell start, cell capacity, cell fill) { - gc_root str(str_,this); + data_root str(str_,this); if(fill <= 0x7f) memset(&str->data()[start],fill,capacity - start); @@ -94,7 +94,7 @@ void factor_vm::fill_string(string *str_, cell start, cell capacity, cell fill) /* Allocates memory */ string *factor_vm::allot_string(cell capacity, cell fill) { - gc_root str(allot_string_internal(capacity),this); + data_root str(allot_string_internal(capacity),this); fill_string(str.untagged(),0,capacity,fill); return str.untagged(); } @@ -115,7 +115,7 @@ bool factor_vm::reallot_string_in_place_p(string *str, cell capacity) string* factor_vm::reallot_string(string *str_, cell capacity) { - gc_root str(str_,this); + data_root str(str_,this); if(reallot_string_in_place_p(str.untagged(),capacity)) { @@ -135,7 +135,7 @@ string* factor_vm::reallot_string(string *str_, cell capacity) if(capacity < to_copy) to_copy = capacity; - gc_root new_str(allot_string_internal(capacity),this); + data_root new_str(allot_string_internal(capacity),this); memcpy(new_str->data(),str->data(),to_copy); diff --git a/vm/tagged.hpp b/vm/tagged.hpp index 77cb6e5287..e520e326fa 100755 --- a/vm/tagged.hpp +++ b/vm/tagged.hpp @@ -16,7 +16,8 @@ struct tagged { cell value_; - cell type() const { + cell type() const + { return TAG(value_); } @@ -33,20 +34,24 @@ struct tagged return type_p(Type::type_number); } - cell value() const { + cell value() const + { #ifdef FACTOR_DEBUG assert(type_p()); #endif return value_; } - Type *untagged() const { + + Type *untagged() const + { #ifdef FACTOR_DEBUG assert(type_p()); #endif return (Type *)(UNTAG(value_)); } - Type *untag_check(factor_vm *parent) const { + Type *untag_check(factor_vm *parent) const + { if(!type_p()) parent->type_error(Type::type_number,value_); return untagged(); diff --git a/vm/tuples.cpp b/vm/tuples.cpp index 8c759b4884..eaac437753 100755 --- a/vm/tuples.cpp +++ b/vm/tuples.cpp @@ -6,7 +6,7 @@ namespace factor /* push a new tuple on the stack, filling its slots with f */ void factor_vm::primitive_tuple() { - gc_root layout(dpop(),this); + data_root layout(dpop(),this); tagged t(allot(tuple_size(layout.untagged()))); t->layout = layout.value(); @@ -18,7 +18,7 @@ void factor_vm::primitive_tuple() /* push a new tuple on the stack, filling its slots from the stack */ void factor_vm::primitive_tuple_boa() { - gc_root layout(dpop(),this); + data_root layout(dpop(),this); tagged t(allot(tuple_size(layout.untagged()))); t->layout = layout.value(); diff --git a/vm/vm.hpp b/vm/vm.hpp index d58ce37742..0789590c49 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -2,6 +2,7 @@ namespace factor { struct growable_array; +struct code_root; struct factor_vm { @@ -62,10 +63,12 @@ struct factor_vm std::vector *gc_events; /* If a runtime function needs to call another function which potentially - allocates memory, it must wrap any local variable references to Factor - objects in gc_root instances */ - std::vector gc_locals; - std::vector gc_bignums; + allocates memory, it must wrap any references to the data and code + heaps with data_root and code_root smart pointers, which register + themselves here. See data_roots.hpp and code_roots.hpp */ + std::vector data_roots; + std::vector bignum_roots; + std::vector code_roots; /* Debugger */ bool fep_disabled; @@ -255,6 +258,8 @@ struct factor_vm void collect_nursery(); void collect_aging(); void collect_to_tenured(); + void update_code_roots_for_sweep(); + void update_code_roots_for_compaction(); void collect_mark_impl(bool trace_contexts_p); void collect_sweep_impl(); void collect_compact_impl(bool trace_contexts_p); @@ -265,7 +270,7 @@ struct factor_vm void primitive_full_gc(); void primitive_compact_gc(); void primitive_become(); - void inline_gc(cell *gc_roots_base, cell gc_roots_size); + void inline_gc(cell *data_roots_base, cell data_roots_size); void primitive_enable_gc_events(); void primitive_disable_gc_events(); object *allot_object(header header, cell size); diff --git a/vm/words.cpp b/vm/words.cpp index 4248c14b7d..c375ec2174 100644 --- a/vm/words.cpp +++ b/vm/words.cpp @@ -5,10 +5,10 @@ namespace factor word *factor_vm::allot_word(cell name_, cell vocab_, cell hashcode_) { - gc_root vocab(vocab_,this); - gc_root name(name_,this); + data_root vocab(vocab_,this); + data_root name(name_,this); - gc_root new_word(allot(sizeof(word)),this); + data_root new_word(allot(sizeof(word)),this); new_word->hashcode = hashcode_; new_word->vocabulary = vocab.value(); @@ -43,7 +43,7 @@ void factor_vm::primitive_word() /* word-xt ( word -- start end ) */ void factor_vm::primitive_word_xt() { - gc_root w(dpop(),this); + data_root w(dpop(),this); w.untag_check(this); if(profiling_p) @@ -61,7 +61,7 @@ void factor_vm::primitive_word_xt() /* Allocates memory */ void factor_vm::update_word_xt(word *w_) { - gc_root w(w_,this); + data_root w(w_,this); if(profiling_p) { From 68217016d06634dc58c0ee7fbd289c731f0e637a Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Mon, 2 Nov 2009 18:11:12 -0600 Subject: [PATCH 55/74] vm: add two missing files --- vm/code_roots.hpp | 29 +++++++++++++++++++++++ vm/data_roots.hpp | 59 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 vm/code_roots.hpp create mode 100644 vm/data_roots.hpp diff --git a/vm/code_roots.hpp b/vm/code_roots.hpp new file mode 100644 index 0000000000..64f2b0c0ed --- /dev/null +++ b/vm/code_roots.hpp @@ -0,0 +1,29 @@ +namespace factor +{ + +struct code_root { + cell value; + bool valid; + factor_vm *parent; + + void push() + { + parent->code_roots.push_back(this); + } + + explicit code_root(cell value_, factor_vm *parent_) : + value(value_), valid(true), parent(parent_) + { + push(); + } + + ~code_root() + { +#ifdef FACTOR_DEBUG + assert(parent->code_roots.back() == this); +#endif + parent->code_roots.pop_back(); + } +}; + +} diff --git a/vm/data_roots.hpp b/vm/data_roots.hpp new file mode 100644 index 0000000000..52654c49f0 --- /dev/null +++ b/vm/data_roots.hpp @@ -0,0 +1,59 @@ +namespace factor +{ + +template +struct data_root : public tagged { + factor_vm *parent; + + void push() + { + parent->data_roots.push_back((cell)this); + } + + explicit data_root(cell value_, factor_vm *parent_) + : tagged(value_), parent(parent_) + { + push(); + } + + explicit data_root(Type *value_, factor_vm *parent_) : + tagged(value_), parent(parent_) + { + push(); + } + + const data_root& operator=(const Type *x) { tagged::operator=(x); return *this; } + const data_root& operator=(const cell &x) { tagged::operator=(x); return *this; } + + ~data_root() + { +#ifdef FACTOR_DEBUG + assert(parent->data_roots.back() == (cell)this); +#endif + parent->data_roots.pop_back(); + } +}; + +/* A similar hack for the bignum implementation */ +struct gc_bignum { + bignum **addr; + factor_vm *parent; + + gc_bignum(bignum **addr_, factor_vm *parent_) : addr(addr_), parent(parent_) + { + if(*addr_) parent->check_data_pointer(*addr_); + parent->bignum_roots.push_back((cell)addr); + } + + ~gc_bignum() + { +#ifdef FACTOR_DEBUG + assert(parent->bignum_roots.back() == (cell)addr); +#endif + parent->bignum_roots.pop_back(); + } +}; + +#define GC_BIGNUM(x) gc_bignum x##__data_root(&x,this) + +} From 3b767c4d0808d87094beb9e2537ee7e77aac3931 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Mon, 2 Nov 2009 18:24:04 -0600 Subject: [PATCH 56/74] vm: remove debug messages --- vm/full_collector.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index acce777301..dfc1b563bd 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -22,7 +22,6 @@ void factor_vm::update_code_roots_for_sweep() for(; iter < end; iter++) { - printf("We have a code root!\n"); code_root *root = *iter; code_block *block = (code_block *)(root->value & -block_granularity); if(root->valid && !state->marked_p(block)) @@ -42,7 +41,6 @@ void factor_vm::update_code_roots_for_compaction() for(; iter < end; iter++) { - printf("We have a code root - compaction!\n"); code_root *root = *iter; code_block *block = (code_block *)(root->value & -block_granularity); From 857d0ba1fd769436f54257e49054935d8b23f8bc Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Mon, 2 Nov 2009 20:21:10 -0600 Subject: [PATCH 57/74] core: bootstrap fixes --- core/alien/alien.factor | 3 ++- core/slots/slots.factor | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/core/alien/alien.factor b/core/alien/alien.factor index 1c4a6cc168..f008a4bd59 100644 --- a/core/alien/alien.factor +++ b/core/alien/alien.factor @@ -39,7 +39,8 @@ M: alien equal? 2drop f ] if ; -M: simple-alien hashcode* nip dup expired>> [ drop 1234 ] [ alien-address ] if ; +M: pinned-alien hashcode* + nip dup expired>> [ drop 1234 ] [ alien-address ] if ; ERROR: alien-callback-error ; diff --git a/core/slots/slots.factor b/core/slots/slots.factor index 95a854f493..0422478884 100755 --- a/core/slots/slots.factor +++ b/core/slots/slots.factor @@ -173,7 +173,7 @@ M: class initial-value* no-initial-value ; { [ string bootstrap-word over class<= ] [ "" ] } { [ array bootstrap-word over class<= ] [ { } ] } { [ byte-array bootstrap-word over class<= ] [ B{ } ] } - { [ simple-alien bootstrap-word over class<= ] [ ] } + { [ pinned-alien bootstrap-word over class<= ] [ ] } { [ quotation bootstrap-word over class<= ] [ [ ] ] } [ dup initial-value* ] } cond nip ; From 2c0531b01529ec6af62ab0950ef2223842729e6c Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Mon, 2 Nov 2009 20:21:21 -0600 Subject: [PATCH 58/74] vm: faster immediate_p() --- core/bootstrap/layouts/layouts.factor | 4 ++-- vm/layouts.hpp | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/core/bootstrap/layouts/layouts.factor b/core/bootstrap/layouts/layouts.factor index e2d686a8db..8b6547ce5c 100644 --- a/core/bootstrap/layouts/layouts.factor +++ b/core/bootstrap/layouts/layouts.factor @@ -16,11 +16,11 @@ BIN: 1111 tag-mask set H{ { fixnum 0 } - { bignum 1 } + { POSTPONE: f 1 } { array 2 } { float 3 } { quotation 4 } - { POSTPONE: f 5 } + { bignum 5 } { alien 6 } { tuple 7 } { wrapper 8 } diff --git a/vm/layouts.hpp b/vm/layouts.hpp index 6b96ade8f5..5f3f3e9310 100644 --- a/vm/layouts.hpp +++ b/vm/layouts.hpp @@ -35,11 +35,11 @@ static const cell data_alignment = 16; /*** Tags ***/ #define FIXNUM_TYPE 0 -#define BIGNUM_TYPE 1 +#define F_TYPE 1 #define ARRAY_TYPE 2 #define FLOAT_TYPE 3 #define QUOTATION_TYPE 4 -#define F_TYPE 5 +#define BIGNUM_TYPE 5 #define ALIEN_TYPE 6 #define TUPLE_TYPE 7 #define WRAPPER_TYPE 8 @@ -76,7 +76,8 @@ static const cell false_object = F_TYPE; inline static bool immediate_p(cell obj) { - return (obj == false_object || TAG(obj) == FIXNUM_TYPE); + /* We assume that fixnums have tag 0 and false_object has tag 1 */ + return TAG(obj) <= F_TYPE; } inline static fixnum untag_fixnum(cell tagged) From d7dceed0967e139baa5745f67c1c7818be612648 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Mon, 2 Nov 2009 20:28:43 -0600 Subject: [PATCH 59/74] slots: another bootstrap fix --- core/slots/slots-docs.factor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/slots/slots-docs.factor b/core/slots/slots-docs.factor index ce29c14b01..92b34db6ec 100644 --- a/core/slots/slots-docs.factor +++ b/core/slots/slots-docs.factor @@ -71,7 +71,7 @@ $nl { { { $link float } } { $snippet "0.0" } } { { { $link string } } { $snippet "\"\"" } } { { { $link byte-array } } { $snippet "B{ }" } } - { { { $link simple-alien } } { $snippet "BAD-ALIEN" } } + { { { $link pinned-alien } } { $snippet "BAD-ALIEN" } } } "All other classes are handled with one of two cases:" { $list From 51e9a891a8d4b585079749309fee34fb8dc0b972 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Tue, 3 Nov 2009 02:42:27 -0600 Subject: [PATCH 60/74] cpu.x86: update %box-displaced-alien for introduction of address field --- .../cfg/instructions/instructions.factor | 15 ++- .../cfg/intrinsics/alien/alien.factor | 2 +- .../value-numbering/rewrite/rewrite.factor | 2 +- basis/cpu/architecture/architecture.factor | 4 +- basis/cpu/x86/x86.factor | 97 +++++++++++++------ 5 files changed, 80 insertions(+), 40 deletions(-) diff --git a/basis/compiler/cfg/instructions/instructions.factor b/basis/compiler/cfg/instructions/instructions.factor index fecc087dae..91ac923273 100644 --- a/basis/compiler/cfg/instructions/instructions.factor +++ b/basis/compiler/cfg/instructions/instructions.factor @@ -512,13 +512,12 @@ temp: temp/int-rep ; PURE-INSN: ##box-displaced-alien def: dst/int-rep use: displacement/int-rep base/int-rep -temp: temp1/int-rep temp2/int-rep +temp: temp/int-rep literal: base-class ; PURE-INSN: ##unbox-any-c-ptr def: dst/int-rep -use: src/int-rep -temp: temp/int-rep ; +use: src/int-rep ; : ##unbox-f ( dst src -- ) drop 0 ##load-immediate ; : ##unbox-byte-array ( dst src -- ) byte-array-offset ##add-imm ; @@ -527,12 +526,12 @@ PURE-INSN: ##unbox-alien def: dst/int-rep use: src/int-rep ; -: ##unbox-c-ptr ( dst src class temp -- ) +: ##unbox-c-ptr ( dst src class -- ) { - { [ over \ f class<= ] [ 2drop ##unbox-f ] } - { [ over alien class<= ] [ 2drop ##unbox-alien ] } - { [ over byte-array class<= ] [ 2drop ##unbox-byte-array ] } - [ nip ##unbox-any-c-ptr ] + { [ dup \ f class<= ] [ drop ##unbox-f ] } + { [ dup alien class<= ] [ drop ##unbox-alien ] } + { [ dup byte-array class<= ] [ drop ##unbox-byte-array ] } + [ drop ##unbox-any-c-ptr ] } cond ; ! Alien accessors diff --git a/basis/compiler/cfg/intrinsics/alien/alien.factor b/basis/compiler/cfg/intrinsics/alien/alien.factor index fb993681e8..320a0a08f7 100644 --- a/basis/compiler/cfg/intrinsics/alien/alien.factor +++ b/basis/compiler/cfg/intrinsics/alien/alien.factor @@ -33,7 +33,7 @@ IN: compiler.cfg.intrinsics.alien bi and ; : ^^unbox-c-ptr ( src class -- dst ) - [ next-vreg dup ] 2dip next-vreg ##unbox-c-ptr ; + [ next-vreg dup ] 2dip ##unbox-c-ptr ; : prepare-alien-accessor ( info -- ptr-vreg offset ) class>> [ 2inputs ^^untag-fixnum swap ] dip ^^unbox-c-ptr ^^add 0 ; diff --git a/basis/compiler/cfg/value-numbering/rewrite/rewrite.factor b/basis/compiler/cfg/value-numbering/rewrite/rewrite.factor index 4fd86c8e96..4864a8bfb7 100755 --- a/basis/compiler/cfg/value-numbering/rewrite/rewrite.factor +++ b/basis/compiler/cfg/value-numbering/rewrite/rewrite.factor @@ -440,7 +440,7 @@ M: ##sar rewrite \ ##sar-imm rewrite-arithmetic ; :: rewrite-unbox-displaced-alien ( insn expr -- insns ) [ next-vreg :> temp - temp expr base>> vn>vreg expr base-class>> insn temp>> ##unbox-c-ptr + temp expr base>> vn>vreg expr base-class>> ##unbox-c-ptr insn dst>> temp expr displacement>> vn>vreg ##add ] { } make ; diff --git a/basis/cpu/architecture/architecture.factor b/basis/cpu/architecture/architecture.factor index 75fbb85542..6723956780 100644 --- a/basis/cpu/architecture/architecture.factor +++ b/basis/cpu/architecture/architecture.factor @@ -386,9 +386,9 @@ M: object %horizontal-shl-vector-imm-reps { } ; M: object %horizontal-shr-vector-imm-reps { } ; HOOK: %unbox-alien cpu ( dst src -- ) -HOOK: %unbox-any-c-ptr cpu ( dst src temp -- ) +HOOK: %unbox-any-c-ptr cpu ( dst src -- ) HOOK: %box-alien cpu ( dst src temp -- ) -HOOK: %box-displaced-alien cpu ( dst displacement base temp1 temp2 base-class -- ) +HOOK: %box-displaced-alien cpu ( dst displacement base temp base-class -- ) HOOK: %alien-unsigned-1 cpu ( dst src offset -- ) HOOK: %alien-unsigned-2 cpu ( dst src offset -- ) diff --git a/basis/cpu/x86/x86.factor b/basis/cpu/x86/x86.factor index fc56113e90..5cd9ab2199 100644 --- a/basis/cpu/x86/x86.factor +++ b/basis/cpu/x86/x86.factor @@ -177,20 +177,20 @@ M: x86 %fixnum-mul ( label dst src1 src2 -- ) M: x86 %unbox-alien ( dst src -- ) alien-offset [+] MOV ; -M:: x86 %unbox-any-c-ptr ( dst src temp -- ) +M:: x86 %unbox-any-c-ptr ( dst src -- ) [ "end" define-label - ! Compute tag in temp register - temp src MOV - temp tag-mask get AND - dst 0 MOV + dst dst XOR ! Is the object f? src \ f type-number CMP "end" get JE + ! Compute tag in dst register + dst src MOV + dst tag-mask get AND + ! Is the object an alien? + dst alien type-number CMP ! Add an offset to start of byte array's data dst src byte-array-offset [+] LEA - ! Is the object an alien? - temp alien type-number CMP "end" get JNE ! If so, load the offset and add it to the address dst src alien-offset [+] MOV @@ -203,7 +203,7 @@ M:: x86 %box-alien ( dst src temp -- ) [ "end" define-label dst \ f type-number MOV - src 0 CMP + src src TEST "end" get JE dst 5 cells alien temp %allot dst 1 alien@ \ f type-number MOV ! base @@ -213,32 +213,73 @@ M:: x86 %box-alien ( dst src temp -- ) "end" resolve-label ] with-scope ; -M:: x86 %box-displaced-alien ( dst displacement base displacement' base' base-class -- ) +M:: x86 %box-displaced-alien ( dst displacement base temp base-class -- ) + ! This is ridiculous [ "end" define-label - "ok" define-label + "not-f" define-label + "not-alien" define-label + ! If displacement is zero, return the base dst base MOV - displacement 0 CMP + displacement displacement TEST "end" get JE - ! Quickly use displacement' before its needed for real, as allot temporary - dst 4 cells alien displacement' %allot - ! If base is already a displaced alien, unpack it - base' base MOV - displacement' displacement MOV + + ! Displacement is non-zero, we're going to be allocating a new + ! object + dst 5 cells alien temp %allot + + ! Set expired to f + dst 2 alien@ \ f type-number MOV + + ! Is base f? base \ f type-number CMP - "ok" get JE - ! XXX - base 0 [+] alien type-number tag-fixnum CMP - "ok" get JNE - ! displacement += base.displacement - displacement' base 3 alien@ ADD - ! base = base.base - base' base 1 alien@ MOV - "ok" resolve-label - dst 1 alien@ base' MOV ! alien - dst 2 alien@ \ f type-number MOV ! expired - dst 3 alien@ displacement' MOV ! displacement + "not-f" get JNE + + ! Yes, it is f. Fill in new object + dst 1 alien@ base MOV + dst 3 alien@ displacement MOV + dst 4 alien@ displacement MOV + + "end" get JMP + + "not-f" resolve-label + + ! Check base type + temp base MOV + temp tag-mask get AND + + ! Is base an alien? + temp alien type-number CMP + "not-alien" get JNE + + ! Yes, it is an alien. Set new alien's base to base.base + temp base 1 alien@ MOV + dst 1 alien@ temp MOV + + ! Compute displacement + temp base 3 alien@ MOV + temp displacement ADD + dst 3 alien@ temp MOV + + ! Compute address + temp base 4 alien@ MOV + temp displacement ADD + dst 4 alien@ temp MOV + + ! We are done + "end" get JMP + + ! Is base a byte array? It has to be, by now... + "not-alien" resolve-label + + dst 1 alien@ base MOV + dst 3 alien@ displacement MOV + temp base MOV + temp byte-array-offset ADD + temp displacement ADD + dst 4 alien@ temp MOV + "end" resolve-label ] with-scope ; From ed3ab1335e9f05a2892b6b912aad5b9dc410d609 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Tue, 3 Nov 2009 04:56:58 -0600 Subject: [PATCH 61/74] vm: minor fixes after code review --- vm/errors.cpp | 1 + vm/inline_cache.cpp | 4 ++-- vm/math.cpp | 32 ++++++++++++++++---------------- vm/quotations.cpp | 4 ++-- vm/words.cpp | 6 +++--- 5 files changed, 24 insertions(+), 23 deletions(-) diff --git a/vm/errors.cpp b/vm/errors.cpp index 2bbe3e5328..7d7b1f0080 100755 --- a/vm/errors.cpp +++ b/vm/errors.cpp @@ -39,6 +39,7 @@ void factor_vm::throw_error(cell error, stack_frame *callstack_top) /* Reset local roots */ data_roots.clear(); bignum_roots.clear(); + code_roots.clear(); /* If we had an underflow or overflow, stack pointers might be out of bounds */ diff --git a/vm/inline_cache.cpp b/vm/inline_cache.cpp index fd5e93560b..21912966d0 100755 --- a/vm/inline_cache.cpp +++ b/vm/inline_cache.cpp @@ -139,7 +139,7 @@ code_block *factor_vm::compile_inline_cache(fixnum index, return code; } -/* A generic word's definition performs general method lookup. Allocates memory */ +/* A generic word's definition performs general method lookup. */ void *factor_vm::megamorphic_call_stub(cell generic_word) { return untag(generic_word)->xt; @@ -174,7 +174,7 @@ void factor_vm::update_pic_transitions(cell pic_size) ic_to_pic_transitions++; } -/* The cache_entries parameter is either f (on cold call site) or an array +/* The cache_entries parameter is empty (on cold call site) or has entries (on cache miss). Called from assembly with the actual return address. Compilation of the inline cache may trigger a GC, which may trigger a compaction; also, the block containing the return address may now be dead. Use a code_root diff --git a/vm/math.cpp b/vm/math.cpp index d5d4e89837..4266edc09c 100755 --- a/vm/math.cpp +++ b/vm/math.cpp @@ -379,7 +379,7 @@ fixnum factor_vm::to_fixnum(cell tagged) } } -VM_C_API fixnum to_fixnum(cell tagged,factor_vm *parent) +VM_C_API fixnum to_fixnum(cell tagged, factor_vm *parent) { return parent->to_fixnum(tagged); } @@ -399,7 +399,7 @@ void factor_vm::box_signed_1(s8 n) dpush(tag_fixnum(n)); } -VM_C_API void box_signed_1(s8 n,factor_vm *parent) +VM_C_API void box_signed_1(s8 n, factor_vm *parent) { return parent->box_signed_1(n); } @@ -409,7 +409,7 @@ void factor_vm::box_unsigned_1(u8 n) dpush(tag_fixnum(n)); } -VM_C_API void box_unsigned_1(u8 n,factor_vm *parent) +VM_C_API void box_unsigned_1(u8 n, factor_vm *parent) { return parent->box_unsigned_1(n); } @@ -419,7 +419,7 @@ void factor_vm::box_signed_2(s16 n) dpush(tag_fixnum(n)); } -VM_C_API void box_signed_2(s16 n,factor_vm *parent) +VM_C_API void box_signed_2(s16 n, factor_vm *parent) { return parent->box_signed_2(n); } @@ -429,7 +429,7 @@ void factor_vm::box_unsigned_2(u16 n) dpush(tag_fixnum(n)); } -VM_C_API void box_unsigned_2(u16 n,factor_vm *parent) +VM_C_API void box_unsigned_2(u16 n, factor_vm *parent) { return parent->box_unsigned_2(n); } @@ -439,7 +439,7 @@ void factor_vm::box_signed_4(s32 n) dpush(allot_integer(n)); } -VM_C_API void box_signed_4(s32 n,factor_vm *parent) +VM_C_API void box_signed_4(s32 n, factor_vm *parent) { return parent->box_signed_4(n); } @@ -449,7 +449,7 @@ void factor_vm::box_unsigned_4(u32 n) dpush(allot_cell(n)); } -VM_C_API void box_unsigned_4(u32 n,factor_vm *parent) +VM_C_API void box_unsigned_4(u32 n, factor_vm *parent) { return parent->box_unsigned_4(n); } @@ -459,7 +459,7 @@ void factor_vm::box_signed_cell(fixnum integer) dpush(allot_integer(integer)); } -VM_C_API void box_signed_cell(fixnum integer,factor_vm *parent) +VM_C_API void box_signed_cell(fixnum integer, factor_vm *parent) { return parent->box_signed_cell(integer); } @@ -469,7 +469,7 @@ void factor_vm::box_unsigned_cell(cell cell) dpush(allot_cell(cell)); } -VM_C_API void box_unsigned_cell(cell cell,factor_vm *parent) +VM_C_API void box_unsigned_cell(cell cell, factor_vm *parent) { return parent->box_unsigned_cell(cell); } @@ -482,7 +482,7 @@ void factor_vm::box_signed_8(s64 n) dpush(tag_fixnum(n)); } -VM_C_API void box_signed_8(s64 n,factor_vm *parent) +VM_C_API void box_signed_8(s64 n, factor_vm *parent) { return parent->box_signed_8(n); } @@ -501,7 +501,7 @@ s64 factor_vm::to_signed_8(cell obj) } } -VM_C_API s64 to_signed_8(cell obj,factor_vm *parent) +VM_C_API s64 to_signed_8(cell obj, factor_vm *parent) { return parent->to_signed_8(obj); } @@ -514,7 +514,7 @@ void factor_vm::box_unsigned_8(u64 n) dpush(tag_fixnum(n)); } -VM_C_API void box_unsigned_8(u64 n,factor_vm *parent) +VM_C_API void box_unsigned_8(u64 n, factor_vm *parent) { return parent->box_unsigned_8(n); } @@ -533,7 +533,7 @@ u64 factor_vm::to_unsigned_8(cell obj) } } -VM_C_API u64 to_unsigned_8(cell obj,factor_vm *parent) +VM_C_API u64 to_unsigned_8(cell obj, factor_vm *parent) { return parent->to_unsigned_8(obj); } @@ -553,7 +553,7 @@ float factor_vm::to_float(cell value) return untag_float_check(value); } -VM_C_API float to_float(cell value,factor_vm *parent) +VM_C_API float to_float(cell value, factor_vm *parent) { return parent->to_float(value); } @@ -563,7 +563,7 @@ void factor_vm::box_double(double flo) dpush(allot_float(flo)); } -VM_C_API void box_double(double flo,factor_vm *parent) +VM_C_API void box_double(double flo, factor_vm *parent) { return parent->box_double(flo); } @@ -573,7 +573,7 @@ double factor_vm::to_double(cell value) return untag_float_check(value); } -VM_C_API double to_double(cell value,factor_vm *parent) +VM_C_API double to_double(cell value, factor_vm *parent) { return parent->to_double(value); } diff --git a/vm/quotations.cpp b/vm/quotations.cpp index 2c5e401ad9..fc19266cee 100755 --- a/vm/quotations.cpp +++ b/vm/quotations.cpp @@ -6,11 +6,11 @@ namespace factor /* Simple non-optimizing compiler. This is one of the two compilers implementing Factor; the second one is written -in Factor and performs advanced optimizations. See core/compiler/compiler.factor. +in Factor and performs advanced optimizations. See basis/compiler/compiler.factor. The non-optimizing compiler compiles a quotation at a time by concatenating machine code chunks; prolog, epilog, call word, jump to word, etc. These machine -code chunks are generated from Factor code in core/cpu/.../bootstrap.factor. +code chunks are generated from Factor code in basis/cpu/.../bootstrap.factor. Calls to words and constant quotations (referenced by conditionals and dips) are direct jumps to machine code blocks. Literals are also referenced directly diff --git a/vm/words.cpp b/vm/words.cpp index c375ec2174..dfaeed2496 100644 --- a/vm/words.cpp +++ b/vm/words.cpp @@ -67,9 +67,9 @@ void factor_vm::update_word_xt(word *w_) { if(!w->profiling) { - /* Note: can't do w->profiling = ... since if LHS - evaluates before RHS, since in that case if RHS does a - GC, we will have an invalid pointer on the LHS */ + /* Note: can't do w->profiling = ... since LHS evaluates + before RHS, and if RHS does a GC, we will have an + invalid pointer on the LHS */ code_block *profiling = compile_profiling_stub(w.value()); w->profiling = profiling; } From 109aa88b0605c74f02284144a268a06bc61b0a96 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Tue, 3 Nov 2009 21:40:29 -0600 Subject: [PATCH 62/74] compiler: update unit tests for 4-bit tag change --- .../value-numbering-tests.factor | 40 +++++++++---------- basis/compiler/tests/intrinsics.factor | 18 +++++---- basis/compiler/tests/low-level-ir.factor | 16 ++++---- basis/compiler/tests/optimizer.factor | 2 +- core/layouts/layouts.factor | 2 +- 5 files changed, 40 insertions(+), 38 deletions(-) diff --git a/basis/compiler/cfg/value-numbering/value-numbering-tests.factor b/basis/compiler/cfg/value-numbering/value-numbering-tests.factor index 55ff39e9d2..b404c4d4a4 100644 --- a/basis/compiler/cfg/value-numbering/value-numbering-tests.factor +++ b/basis/compiler/cfg/value-numbering/value-numbering-tests.factor @@ -82,7 +82,7 @@ IN: compiler.cfg.value-numbering.tests T{ ##load-reference f 1 + } T{ ##peek f 2 D 0 } T{ ##compare f 4 2 1 cc> } - T{ ##compare-imm f 6 4 5 cc/= } + T{ ##compare-imm f 6 4 $[ \ f type-number ] cc/= } T{ ##replace f 6 D 0 } } value-numbering-step trim-temps ] unit-test @@ -100,7 +100,7 @@ IN: compiler.cfg.value-numbering.tests T{ ##load-reference f 1 + } T{ ##peek f 2 D 0 } T{ ##compare f 4 2 1 cc<= } - T{ ##compare-imm f 6 4 5 cc= } + T{ ##compare-imm f 6 4 $[ \ f type-number ] cc= } T{ ##replace f 6 D 0 } } value-numbering-step trim-temps ] unit-test @@ -118,7 +118,7 @@ IN: compiler.cfg.value-numbering.tests T{ ##peek f 8 D 0 } T{ ##peek f 9 D -1 } T{ ##compare-float-unordered f 12 8 9 cc< } - T{ ##compare-imm f 14 12 5 cc= } + T{ ##compare-imm f 14 12 $[ \ f type-number ] cc= } T{ ##replace f 14 D 0 } } value-numbering-step trim-temps ] unit-test @@ -135,7 +135,7 @@ IN: compiler.cfg.value-numbering.tests T{ ##peek f 29 D -1 } T{ ##peek f 30 D -2 } T{ ##compare f 33 29 30 cc<= } - T{ ##compare-imm-branch f 33 5 cc/= } + T{ ##compare-imm-branch f 33 $[ \ f type-number ] cc/= } } value-numbering-step trim-temps ] unit-test @@ -149,7 +149,7 @@ IN: compiler.cfg.value-numbering.tests { T{ ##peek f 1 D -1 } T{ ##test-vector f 2 1 f float-4-rep vcc-any } - T{ ##compare-imm-branch f 2 5 cc/= } + T{ ##compare-imm-branch f 2 $[ \ f type-number ] cc/= } } value-numbering-step trim-temps ] unit-test @@ -1071,14 +1071,14 @@ cell 8 = [ ! Branch folding [ { - T{ ##load-immediate f 1 1 } - T{ ##load-immediate f 2 2 } - T{ ##load-immediate f 3 5 } + T{ ##load-immediate f 1 10 } + T{ ##load-immediate f 2 20 } + T{ ##load-immediate f 3 $[ \ f type-number ] } } ] [ { - T{ ##load-immediate f 1 1 } - T{ ##load-immediate f 2 2 } + T{ ##load-immediate f 1 10 } + T{ ##load-immediate f 2 20 } T{ ##compare f 3 1 2 cc= } } value-numbering-step ] unit-test @@ -1113,14 +1113,14 @@ cell 8 = [ [ { - T{ ##load-immediate f 1 1 } - T{ ##load-immediate f 2 2 } - T{ ##load-immediate f 3 5 } + T{ ##load-immediate f 1 10 } + T{ ##load-immediate f 2 20 } + T{ ##load-immediate f 3 $[ \ f type-number ] } } ] [ { - T{ ##load-immediate f 1 1 } - T{ ##load-immediate f 2 2 } + T{ ##load-immediate f 1 10 } + T{ ##load-immediate f 2 20 } T{ ##compare f 3 2 1 cc< } } value-numbering-step ] unit-test @@ -1128,7 +1128,7 @@ cell 8 = [ [ { T{ ##peek f 0 D 0 } - T{ ##load-immediate f 1 5 } + T{ ##load-immediate f 1 $[ \ f type-number ] } } ] [ { @@ -1152,7 +1152,7 @@ cell 8 = [ [ { T{ ##peek f 0 D 0 } - T{ ##load-immediate f 1 5 } + T{ ##load-immediate f 1 $[ \ f type-number ] } } ] [ { @@ -1176,7 +1176,7 @@ cell 8 = [ [ { T{ ##peek f 0 D 0 } - T{ ##load-immediate f 1 5 } + T{ ##load-immediate f 1 $[ \ f type-number ] } } ] [ { @@ -1557,7 +1557,7 @@ cell 8 = [ { T{ ##peek f 0 D 0 } T{ ##compare f 1 0 0 cc<= } - T{ ##compare-imm-branch f 1 5 cc/= } + T{ ##compare-imm-branch f 1 $[ \ f type-number ] cc/= } } test-branch-folding ] unit-test @@ -1659,7 +1659,7 @@ V{ T{ ##copy { dst 21 } { src 20 } { rep any-rep } } T{ ##compare-imm-branch { src1 21 } - { src2 5 } + { src2 $[ \ f type-number ] } { cc cc/= } } } 1 test-bb diff --git a/basis/compiler/tests/intrinsics.factor b/basis/compiler/tests/intrinsics.factor index dfc1af9a11..202ee57a33 100644 --- a/basis/compiler/tests/intrinsics.factor +++ b/basis/compiler/tests/intrinsics.factor @@ -285,8 +285,8 @@ cell 8 = [ ! 64-bit overflow cell 8 = [ - [ t ] [ 1 59 fixnum-shift dup [ fixnum+ ] compile-call 1 60 fixnum-shift = ] unit-test - [ -1152921504606846977 ] [ 1 60 shift neg >fixnum [ -1 fixnum+ ] compile-call ] unit-test + [ t ] [ 1 58 fixnum-shift dup [ fixnum+ ] compile-call 1 59 fixnum-shift = ] unit-test + [ -576460752303423489 ] [ 1 59 shift neg >fixnum [ -1 fixnum+ ] compile-call ] unit-test [ t ] [ 1 40 shift 1 40 shift [ fixnum* ] compile-call 1 80 shift = ] unit-test [ t ] [ 1 40 shift neg 1 40 shift [ fixnum* ] compile-call 1 80 shift neg = ] unit-test @@ -301,9 +301,9 @@ cell 8 = [ [ -18446744073709551616 ] [ -1 [ 64 fixnum-shift ] compile-call ] unit-test [ -18446744073709551616 ] [ -1 [ 32 fixnum-shift 32 fixnum-shift ] compile-call ] unit-test - [ 1152921504606846976 ] [ -1152921504606846976 >fixnum -1 [ fixnum/i ] compile-call ] unit-test + [ 576460752303423488 ] [ -576460752303423488 >fixnum -1 [ fixnum/i ] compile-call ] unit-test - [ 1152921504606846976 0 ] [ -1152921504606846976 >fixnum -1 [ fixnum/mod ] compile-call ] unit-test + [ 576460752303423488 0 ] [ -576460752303423488 >fixnum -1 [ fixnum/mod ] compile-call ] unit-test [ -268435457 ] [ 28 2^ [ fixnum-bitnot ] compile-call ] unit-test ] when @@ -311,12 +311,14 @@ cell 8 = [ ! Some randomized tests : compiled-fixnum* ( a b -- c ) fixnum* ; +ERROR: bug-in-fixnum* x y a b ; + [ ] [ 10000 [ - 32 random-bits >fixnum 32 random-bits >fixnum - 2dup - [ fixnum* ] 2keep compiled-fixnum* = - [ 2drop ] [ "Oops" throw ] if + 32 random-bits >fixnum + 32 random-bits >fixnum + 2dup [ fixnum* ] [ compiled-fixnum* ] 2bi 2dup = + [ 2drop 2drop ] [ bug-in-fixnum* ] if ] times ] unit-test diff --git a/basis/compiler/tests/low-level-ir.factor b/basis/compiler/tests/low-level-ir.factor index 583b228eb2..b6b8e1c031 100644 --- a/basis/compiler/tests/low-level-ir.factor +++ b/basis/compiler/tests/low-level-ir.factor @@ -36,7 +36,7 @@ IN: compiler.tests.low-level-ir ! loading immediates [ f ] [ V{ - T{ ##load-immediate f 0 5 } + T{ ##load-immediate f 0 $[ \ f type-number ] } } compile-test-bb ] unit-test @@ -80,7 +80,7 @@ IN: compiler.tests.low-level-ir dup first eq? ] unit-test -[ 8 ] [ +[ 4 ] [ V{ T{ ##load-immediate f 0 4 } T{ ##shl f 0 0 0 } @@ -90,16 +90,16 @@ IN: compiler.tests.low-level-ir [ 4 ] [ V{ T{ ##load-immediate f 0 4 } - T{ ##shl-imm f 0 0 3 } + T{ ##shl-imm f 0 0 4 } } compile-test-bb ] unit-test [ 31 ] [ V{ T{ ##load-reference f 1 B{ 31 67 52 } } - T{ ##unbox-any-c-ptr f 0 1 2 } + T{ ##unbox-any-c-ptr f 0 1 } T{ ##alien-unsigned-1 f 0 0 0 } - T{ ##shl-imm f 0 0 3 } + T{ ##shl-imm f 0 0 4 } } compile-test-bb ] unit-test @@ -108,13 +108,13 @@ IN: compiler.tests.low-level-ir T{ ##load-reference f 0 "hello world" } T{ ##load-immediate f 1 3 } T{ ##string-nth f 0 0 1 2 } - T{ ##shl-imm f 0 0 3 } + T{ ##shl-imm f 0 0 4 } } compile-test-bb ] unit-test [ 1 ] [ V{ - T{ ##load-immediate f 0 16 } - T{ ##add-imm f 0 0 -8 } + T{ ##load-immediate f 0 32 } + T{ ##add-imm f 0 0 -16 } } compile-test-bb ] unit-test diff --git a/basis/compiler/tests/optimizer.factor b/basis/compiler/tests/optimizer.factor index 32f5750cd3..0831d6e8dd 100644 --- a/basis/compiler/tests/optimizer.factor +++ b/basis/compiler/tests/optimizer.factor @@ -202,7 +202,7 @@ USE: binary-search.private dup length 1 <= [ from>> ] [ - [ midpoint swap call ] 3keep roll dup zero? + [ midpoint swap call ] 3keep [ rot ] dip swap dup zero? [ drop dup from>> swap midpoint@ + ] [ drop dup midpoint@ head-slice old-binsearch ] if ] if ; inline recursive diff --git a/core/layouts/layouts.factor b/core/layouts/layouts.factor index 426bd560bf..7518dbf0cb 100644 --- a/core/layouts/layouts.factor +++ b/core/layouts/layouts.factor @@ -53,7 +53,7 @@ SYMBOL: mega-cache-size first-bignum neg >fixnum ; inline : (max-array-capacity) ( b -- n ) - 5 - 2^ 1 - ; inline + 6 - 2^ 1 - ; inline : max-array-capacity ( -- n ) cell-bits (max-array-capacity) ; inline From 9ffb01a9bde865ed18bbd0ef4cac4e6227171504 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Tue, 3 Nov 2009 21:40:47 -0600 Subject: [PATCH 63/74] vm: fix fixnum* overflow case for 4-bit tags --- vm/cpu-ppc.S | 2 +- vm/cpu-x86.S | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vm/cpu-ppc.S b/vm/cpu-ppc.S index 1071d8b8a9..40f2521e50 100644 --- a/vm/cpu-ppc.S +++ b/vm/cpu-ppc.S @@ -43,7 +43,7 @@ DEF(void,primitive_fixnum_multiply,(void *vm)): stw r6,0(DS_REG) blr multiply_overflow: - srawi r4,r4,3 + srawi r4,r4,4 b MANGLE(overflow_fixnum_multiply) /* Note that the XT is passed to the quotation in r11 */ diff --git a/vm/cpu-x86.S b/vm/cpu-x86.S index 706369876f..846300120d 100644 --- a/vm/cpu-x86.S +++ b/vm/cpu-x86.S @@ -33,7 +33,7 @@ DEF(void,primitive_fixnum_multiply,(void *myvm)): pop ARG2 ret multiply_overflow: - sar $3,ARITH_TEMP_1 + sar $4,ARITH_TEMP_1 mov ARITH_TEMP_1,ARG0 mov ARITH_TEMP_2,ARG1 pop ARG2 From e1c365e69a9408fc8961027d55e7d03383ab351d Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Tue, 3 Nov 2009 22:25:22 -0600 Subject: [PATCH 64/74] vm: fix issue with unnecessary heap growth --- vm/code_block.cpp | 3 +-- vm/data_heap.cpp | 5 +++++ vm/data_heap.hpp | 1 + vm/full_collector.cpp | 16 ++++++++++++++++ vm/gc.cpp | 28 ++++++++++++++-------------- vm/vm.hpp | 2 ++ 6 files changed, 39 insertions(+), 16 deletions(-) diff --git a/vm/code_block.cpp b/vm/code_block.cpp index 9869630732..56dd27c6b7 100755 --- a/vm/code_block.cpp +++ b/vm/code_block.cpp @@ -127,9 +127,8 @@ void *factor_vm::get_rel_symbol(array *literals, cell index) } case ARRAY_TYPE: { - cell i; array *names = untag(symbol); - for(i = 0; i < array_capacity(names); i++) + for(cell i = 0; i < array_capacity(names); i++) { symbol_char *name = alien_offset(array_nth(names,i)); void *sym = ffi_dlsym(d,name); diff --git a/vm/data_heap.cpp b/vm/data_heap.cpp index 21008af5cb..bb705e276c 100755 --- a/vm/data_heap.cpp +++ b/vm/data_heap.cpp @@ -98,6 +98,11 @@ void data_heap::reset_generation(tenured_space *gen) clear_decks(gen); } +bool data_heap::low_memory_p() +{ + return (tenured->free_space() <= nursery->size + aging->size); +} + void factor_vm::set_data_heap(data_heap *data_) { data = data_; diff --git a/vm/data_heap.hpp b/vm/data_heap.hpp index a434024873..760a10942e 100755 --- a/vm/data_heap.hpp +++ b/vm/data_heap.hpp @@ -29,6 +29,7 @@ struct data_heap { void reset_generation(nursery_space *gen); void reset_generation(aging_space *gen); void reset_generation(tenured_space *gen); + bool low_memory_p(); }; struct data_heap_room { diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index dfc1b563bd..3b92e2574e 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -118,6 +118,22 @@ void factor_vm::collect_sweep_impl() current_gc->event->ended_data_sweep(); } +void factor_vm::collect_full(bool trace_contexts_p) +{ + collect_mark_impl(trace_contexts_p); + collect_sweep_impl(); + if(data->tenured->largest_free_block() <= data->nursery->size + data->aging->size) + collect_compact_impl(trace_contexts_p); + else + update_code_heap_words_and_literals(); +} + +void factor_vm::collect_compact(bool trace_contexts_p) +{ + collect_mark_impl(trace_contexts_p); + collect_compact_impl(trace_contexts_p); +} + void factor_vm::collect_growing_heap(cell requested_bytes, bool trace_contexts_p) { /* Grow the data heap and copy all live objects to the new heap. */ diff --git a/vm/gc.cpp b/vm/gc.cpp index 06e9d78ce2..de8a2886f7 100755 --- a/vm/gc.cpp +++ b/vm/gc.cpp @@ -152,12 +152,6 @@ void factor_vm::gc(gc_op op, cell requested_bytes, bool trace_contexts_p) start_gc_again(); } - if(current_gc->op == collect_aging_op || current_gc->op == collect_to_tenured_op) - { - if(data->tenured->free_space() <= data->nursery->size + data->aging->size) - current_gc->op = collect_full_op; - } - current_gc->event->op = current_gc->op; switch(current_gc->op) @@ -167,21 +161,27 @@ void factor_vm::gc(gc_op op, cell requested_bytes, bool trace_contexts_p) break; case collect_aging_op: collect_aging(); + if(data->low_memory_p()) + { + current_gc->op = collect_full_op; + current_gc->event->op = collect_full_op; + collect_full(trace_contexts_p); + } break; case collect_to_tenured_op: collect_to_tenured(); + if(data->low_memory_p()) + { + current_gc->op = collect_full_op; + current_gc->event->op = collect_full_op; + collect_full(trace_contexts_p); + } break; case collect_full_op: - collect_mark_impl(trace_contexts_p); - collect_sweep_impl(); - if(data->tenured->largest_free_block() <= data->nursery->size + data->aging->size) - collect_compact_impl(trace_contexts_p); - else - update_code_heap_words_and_literals(); + collect_full(trace_contexts_p); break; case collect_compact_op: - collect_mark_impl(trace_contexts_p); - collect_compact_impl(trace_contexts_p); + collect_compact(trace_contexts_p); break; case collect_growing_heap_op: collect_growing_heap(requested_bytes,trace_contexts_p); diff --git a/vm/vm.hpp b/vm/vm.hpp index 0789590c49..aa04c8dffc 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -262,8 +262,10 @@ struct factor_vm void update_code_roots_for_compaction(); void collect_mark_impl(bool trace_contexts_p); void collect_sweep_impl(); + void collect_full(bool trace_contexts_p); void collect_compact_impl(bool trace_contexts_p); void collect_compact_code_impl(bool trace_contexts_p); + void collect_compact(bool trace_contexts_p); void collect_growing_heap(cell requested_bytes, bool trace_contexts_p); void gc(gc_op op, cell requested_bytes, bool trace_contexts_p); void primitive_minor_gc(); From 3c4c05e9158a0db145ecf531f0e88596bca6bdf7 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Tue, 3 Nov 2009 23:44:20 -0600 Subject: [PATCH 65/74] compiler.cfg.intrinsics.allot: fix inline byte array allocation on 32-bit platforms --- basis/compiler/cfg/intrinsics/allot/allot.factor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/basis/compiler/cfg/intrinsics/allot/allot.factor b/basis/compiler/cfg/intrinsics/allot/allot.factor index 43747f88c9..9804244ecb 100644 --- a/basis/compiler/cfg/intrinsics/allot/allot.factor +++ b/basis/compiler/cfg/intrinsics/allot/allot.factor @@ -62,7 +62,7 @@ IN: compiler.cfg.intrinsics.allot : bytes>cells ( m -- n ) cell align cell /i ; : ^^allot-byte-array ( n -- dst ) - 2 cells + byte-array ^^allot ; + 16 + byte-array ^^allot ; : emit-allot-byte-array ( len -- dst ) ds-drop From 2b1a26228be743e7a9e4e33fd2d14acac82ecc87 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Tue, 3 Nov 2009 23:51:44 -0600 Subject: [PATCH 66/74] Align stack pointer on non-Mac OS X x86-32 platforms, and use aligned loads/stores for SIMD values --- basis/cpu/x86/32/32.factor | 3 --- basis/cpu/x86/x86.factor | 18 ++++++++---------- basis/math/vectors/simd/functor/functor.factor | 4 ++-- basis/math/vectors/simd/simd-tests.factor | 17 +++++++++++++++++ vm/cpu-x86.32.S | 2 -- vm/cpu-x86.64.S | 4 ---- vm/cpu-x86.S | 15 ++++++++++++--- 7 files changed, 39 insertions(+), 24 deletions(-) diff --git a/basis/cpu/x86/32/32.factor b/basis/cpu/x86/32/32.factor index cff5c561c8..8a29c82dad 100755 --- a/basis/cpu/x86/32/32.factor +++ b/basis/cpu/x86/32/32.factor @@ -11,9 +11,6 @@ cpu.x86.assembler cpu.x86.assembler.operands cpu.x86 cpu.architecture ; IN: cpu.x86.32 -! We implement the FFI for Linux, OS X and Windows all at once. -! OS X requires that the stack be 16-byte aligned. - M: x86.32 machine-registers { { int-regs { EAX ECX EDX EBP EBX } } diff --git a/basis/cpu/x86/x86.factor b/basis/cpu/x86/x86.factor index 5cd9ab2199..a63b92e050 100644 --- a/basis/cpu/x86/x86.factor +++ b/basis/cpu/x86/x86.factor @@ -45,8 +45,7 @@ HOOK: extra-stack-space cpu ( stack-frame -- n ) : incr-stack-reg ( n -- ) dup 0 = [ drop ] [ stack-reg swap ADD ] if ; -: align-stack ( n -- n' ) - os macosx? cpu x86.64? or [ 16 align ] when ; +: align-stack ( n -- n' ) 16 align ; M: x86 stack-frame-size ( stack-frame -- i ) [ (stack-frame-size) ] @@ -141,8 +140,10 @@ M: x86 %not int-rep one-operand NOT ; M: x86 %neg int-rep one-operand NEG ; M: x86 %log2 BSR ; +! A bit of logic to avoid using MOVSS/MOVSD for reg-reg moves +! since this induces partial register stalls GENERIC: copy-register* ( dst src rep -- ) -GENERIC: copy-unaligned* ( dst src rep -- ) +GENERIC: copy-memory* ( dst src rep -- ) M: int-rep copy-register* drop MOV ; M: tagged-rep copy-register* drop MOV ; @@ -152,17 +153,14 @@ M: float-4-rep copy-register* drop MOVAPS ; M: double-2-rep copy-register* drop MOVAPS ; M: vector-rep copy-register* drop MOVDQA ; -M: object copy-unaligned* copy-register* ; -M: float-rep copy-unaligned* drop MOVSS ; -M: double-rep copy-unaligned* drop MOVSD ; -M: float-4-rep copy-unaligned* drop MOVUPS ; -M: double-2-rep copy-unaligned* drop MOVUPS ; -M: vector-rep copy-unaligned* drop MOVDQU ; +M: object copy-memory* copy-register* ; +M: float-rep copy-memory* drop MOVSS ; +M: double-rep copy-memory* drop MOVSD ; M: x86 %copy ( dst src rep -- ) 2over eq? [ 3drop ] [ [ [ dup spill-slot? [ n>> spill@ ] when ] bi@ ] dip - 2over [ register? ] both? [ copy-register* ] [ copy-unaligned* ] if + 2over [ register? ] both? [ copy-register* ] [ copy-memory* ] if ] if ; M: x86 %fixnum-add ( label dst src1 src2 -- ) diff --git a/basis/math/vectors/simd/functor/functor.factor b/basis/math/vectors/simd/functor/functor.factor index 480981d165..44907df68e 100644 --- a/basis/math/vectors/simd/functor/functor.factor +++ b/basis/math/vectors/simd/functor/functor.factor @@ -146,7 +146,7 @@ TUPLE: simd class elt-class ops special-wrappers schema-wrappers ctor rep ; [ rep alien-vector class boa ] >>getter [ [ underlying>> ] 2dip rep set-alien-vector ] >>setter 16 >>size - 8 >>align + 16 >>align rep >>rep class c:typedef ; @@ -315,7 +315,7 @@ SLOT: underlying2 3bi ] >>setter 32 >>size - 8 >>align + 16 >>align rep >>rep class c:typedef ; diff --git a/basis/math/vectors/simd/simd-tests.factor b/basis/math/vectors/simd/simd-tests.factor index 7ba9f243ce..396b8da22a 100644 --- a/basis/math/vectors/simd/simd-tests.factor +++ b/basis/math/vectors/simd/simd-tests.factor @@ -582,3 +582,20 @@ STRUCT: simd-struct float-4{ 1.0 0.0 1.0 0.0 } pi [ broken 3array ] [ compile-call ] [ call ] 3bi = ] unit-test + +! Spilling SIMD values -- this basically just tests that the +! stack was aligned properly by the runtime + +: simd-spill-test-1 ( a b c -- v ) + { float-4 float-4 float } declare + [ v+ ] dip sin v*n ; + +[ float-4{ 0 0 0 0 } ] +[ float-4{ 1 2 3 4 } float-4{ 4 5 6 7 } 0.0 simd-spill-test-1 ] unit-test + +: simd-spill-test-2 ( a b d c -- v ) + { float float-4 float-4 float } declare + [ [ 3.0 + ] 2dip v+ ] dip sin v*n n*v ; + +[ float-4{ 0 0 0 0 } ] +[ 5.0 float-4{ 1 2 3 4 } float-4{ 4 5 6 7 } 0.0 simd-spill-test-2 ] unit-test diff --git a/vm/cpu-x86.32.S b/vm/cpu-x86.32.S index 2e85be0f81..c0532f0ece 100644 --- a/vm/cpu-x86.32.S +++ b/vm/cpu-x86.32.S @@ -19,11 +19,9 @@ #define PUSH_NONVOLATILE \ push %ebx ; \ - push %ebp ; \ push %ebp #define POP_NONVOLATILE \ - pop %ebp ; \ pop %ebp ; \ pop %ebx diff --git a/vm/cpu-x86.64.S b/vm/cpu-x86.64.S index 5e307f0500..8ccd703bfe 100644 --- a/vm/cpu-x86.64.S +++ b/vm/cpu-x86.64.S @@ -27,11 +27,9 @@ push %rdi ; \ push %rsi ; \ push %rbx ; \ - push %rbp ; \ push %rbp #define POP_NONVOLATILE \ - pop %rbp ; \ pop %rbp ; \ pop %rbx ; \ pop %rsi ; \ @@ -50,11 +48,9 @@ push %rbx ; \ push %rbp ; \ push %r12 ; \ - push %r13 ; \ push %r13 #define POP_NONVOLATILE \ - pop %r13 ; \ pop %r13 ; \ pop %r12 ; \ pop %rbp ; \ diff --git a/vm/cpu-x86.S b/vm/cpu-x86.S index 846300120d..411a0cdaa6 100644 --- a/vm/cpu-x86.S +++ b/vm/cpu-x86.S @@ -43,14 +43,20 @@ DEF(F_FASTCALL void,c_to_factor,(CELL quot, void *vm)): PUSH_NONVOLATILE mov ARG0,NV0 mov ARG1,NV1 - + + /* Save old stack pointer and align */ + mov STACK_REG,ARG0 + and $-16,STACK_REG + add $CELL_SIZE,STACK_REG + push ARG0 + /* Create register shadow area for Win64 */ sub $32,STACK_REG - + /* Save stack pointer */ lea -CELL_SIZE(STACK_REG),ARG0 call MANGLE(save_callstack_bottom) - + /* Call quot-xt */ mov NV0,ARG0 mov NV1,ARG1 @@ -59,6 +65,9 @@ DEF(F_FASTCALL void,c_to_factor,(CELL quot, void *vm)): /* Tear down register shadow area */ add $32,STACK_REG + /* Undo stack alignment */ + mov (STACK_REG),STACK_REG + POP_NONVOLATILE ret From 0913758805849683ed81ec6b9367e84031166b70 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Tue, 3 Nov 2009 23:52:02 -0600 Subject: [PATCH 67/74] tools.memory: tweak --- basis/tools/memory/memory.factor | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/basis/tools/memory/memory.factor b/basis/tools/memory/memory.factor index 5c6e910108..93151b403e 100644 --- a/basis/tools/memory/memory.factor +++ b/basis/tools/memory/memory.factor @@ -60,7 +60,7 @@ IN: tools.memory } fancy-table. ; : data-room. ( -- ) - "==== DATA HEAP" print nl + "== Data heap ==" print nl data-room data-heap-room memory>struct { [ nursery-room. nl ] [ aging-room. nl ] @@ -69,7 +69,7 @@ IN: tools.memory } cleave ; : code-room. ( -- ) - "==== CODE HEAP" print nl + "== Code heap ==" print nl code-room mark-sweep-sizes memory>struct mark-sweep-table. ; PRIVATE> From 8c4ad9bf184d3e517e932448b8c6e57c89753806 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Wed, 4 Nov 2009 18:28:46 -0600 Subject: [PATCH 68/74] compiler: fix intrinsics tests on 32-bit --- basis/compiler/tests/intrinsics.factor | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) mode change 100644 => 100755 basis/compiler/tests/intrinsics.factor diff --git a/basis/compiler/tests/intrinsics.factor b/basis/compiler/tests/intrinsics.factor old mode 100644 new mode 100755 index 202ee57a33..a26ba5a27a --- a/basis/compiler/tests/intrinsics.factor +++ b/basis/compiler/tests/intrinsics.factor @@ -244,20 +244,20 @@ IN: compiler.tests.intrinsics [ -4294967296 ] [ -1 [ 16 fixnum-shift 16 fixnum-shift ] compile-call ] unit-test [ HEX: 10000000 ] [ HEX: 1000000 HEX: 10 [ fixnum* ] compile-call ] unit-test -[ HEX: 10000000 ] [ HEX: -10000000 >fixnum [ 0 swap fixnum- ] compile-call ] unit-test -[ HEX: 10000000 ] [ HEX: -fffffff >fixnum [ 1 swap fixnum- ] compile-call ] unit-test +[ HEX: 8000000 ] [ HEX: -8000000 >fixnum [ 0 swap fixnum- ] compile-call ] unit-test +[ HEX: 8000000 ] [ HEX: -7ffffff >fixnum [ 1 swap fixnum- ] compile-call ] unit-test -[ t ] [ 1 27 fixnum-shift dup [ fixnum+ ] compile-call 1 28 fixnum-shift = ] unit-test -[ -268435457 ] [ 1 28 shift neg >fixnum [ -1 fixnum+ ] compile-call ] unit-test +[ t ] [ 1 26 fixnum-shift dup [ fixnum+ ] compile-call 1 27 fixnum-shift = ] unit-test +[ -134217729 ] [ 1 27 shift neg >fixnum [ -1 fixnum+ ] compile-call ] unit-test [ t ] [ 1 20 shift 1 20 shift [ fixnum* ] compile-call 1 40 shift = ] unit-test [ t ] [ 1 20 shift neg 1 20 shift [ fixnum* ] compile-call 1 40 shift neg = ] unit-test [ t ] [ 1 20 shift neg 1 20 shift neg [ fixnum* ] compile-call 1 40 shift = ] unit-test [ -351382792 ] [ -43922849 [ 3 fixnum-shift ] compile-call ] unit-test -[ 268435456 ] [ -268435456 >fixnum -1 [ fixnum/i ] compile-call ] unit-test +[ 134217728 ] [ -134217728 >fixnum -1 [ fixnum/i ] compile-call ] unit-test -[ 268435456 0 ] [ -268435456 >fixnum -1 [ fixnum/mod ] compile-call ] unit-test +[ 134217728 0 ] [ -134217728 >fixnum -1 [ fixnum/mod ] compile-call ] unit-test [ t ] [ f [ f eq? ] compile-call ] unit-test From 42fc3c0f056abda95b444e31877de73502cfa6a9 Mon Sep 17 00:00:00 2001 From: carlok Date: Thu, 5 Nov 2009 07:35:16 +0100 Subject: [PATCH 69/74] Add support for CP-1250 encoding --- basis/io/encodings/8-bit/8-bit.factor | 1 + basis/io/encodings/8-bit/CP1250.TXT | 274 ++++++++++++++++++++++++++ 2 files changed, 275 insertions(+) create mode 100644 basis/io/encodings/8-bit/CP1250.TXT diff --git a/basis/io/encodings/8-bit/8-bit.factor b/basis/io/encodings/8-bit/8-bit.factor index bba22268c6..c13bbccd43 100644 --- a/basis/io/encodings/8-bit/8-bit.factor +++ b/basis/io/encodings/8-bit/8-bit.factor @@ -27,6 +27,7 @@ CONSTANT: mappings { { "latin9" "ISO-8859-15" "8859-15" } { "latin10" "ISO-8859-16" "8859-16" } { "koi8-r" "KOI8-R" "KOI8-R" } + { "windows-1250" "windows-1250" "CP1250" } { "windows-1252" "windows-1252" "CP1252" } { "ebcdic" "IBM037" "CP037" } { "mac-roman" "macintosh" "ROMAN" } diff --git a/basis/io/encodings/8-bit/CP1250.TXT b/basis/io/encodings/8-bit/CP1250.TXT new file mode 100644 index 0000000000..6bfab9380d --- /dev/null +++ b/basis/io/encodings/8-bit/CP1250.TXT @@ -0,0 +1,274 @@ +# +# Name: cp1250 to Unicode table +# Unicode version: 2.0 +# Table version: 2.01 +# Table format: Format A +# Date: 04/15/98 +# +# Contact: Shawn.Steele@microsoft.com +# +# General notes: none +# +# Format: Three tab-separated columns +# Column #1 is the cp1250 code (in hex) +# Column #2 is the Unicode (in hex as 0xXXXX) +# Column #3 is the Unicode name (follows a comment sign, '#') +# +# The entries are in cp1250 order +# +0x00 0x0000 #NULL +0x01 0x0001 #START OF HEADING +0x02 0x0002 #START OF TEXT +0x03 0x0003 #END OF TEXT +0x04 0x0004 #END OF TRANSMISSION +0x05 0x0005 #ENQUIRY +0x06 0x0006 #ACKNOWLEDGE +0x07 0x0007 #BELL +0x08 0x0008 #BACKSPACE +0x09 0x0009 #HORIZONTAL TABULATION +0x0A 0x000A #LINE FEED +0x0B 0x000B #VERTICAL TABULATION +0x0C 0x000C #FORM FEED +0x0D 0x000D #CARRIAGE RETURN +0x0E 0x000E #SHIFT OUT +0x0F 0x000F #SHIFT IN +0x10 0x0010 #DATA LINK ESCAPE +0x11 0x0011 #DEVICE CONTROL ONE +0x12 0x0012 #DEVICE CONTROL TWO +0x13 0x0013 #DEVICE CONTROL THREE +0x14 0x0014 #DEVICE CONTROL FOUR +0x15 0x0015 #NEGATIVE ACKNOWLEDGE +0x16 0x0016 #SYNCHRONOUS IDLE +0x17 0x0017 #END OF TRANSMISSION BLOCK +0x18 0x0018 #CANCEL +0x19 0x0019 #END OF MEDIUM +0x1A 0x001A #SUBSTITUTE +0x1B 0x001B #ESCAPE +0x1C 0x001C #FILE SEPARATOR +0x1D 0x001D #GROUP SEPARATOR +0x1E 0x001E #RECORD SEPARATOR +0x1F 0x001F #UNIT SEPARATOR +0x20 0x0020 #SPACE +0x21 0x0021 #EXCLAMATION MARK +0x22 0x0022 #QUOTATION MARK +0x23 0x0023 #NUMBER SIGN +0x24 0x0024 #DOLLAR SIGN +0x25 0x0025 #PERCENT SIGN +0x26 0x0026 #AMPERSAND +0x27 0x0027 #APOSTROPHE +0x28 0x0028 #LEFT PARENTHESIS +0x29 0x0029 #RIGHT PARENTHESIS +0x2A 0x002A #ASTERISK +0x2B 0x002B #PLUS SIGN +0x2C 0x002C #COMMA +0x2D 0x002D #HYPHEN-MINUS +0x2E 0x002E #FULL STOP +0x2F 0x002F #SOLIDUS +0x30 0x0030 #DIGIT ZERO +0x31 0x0031 #DIGIT ONE +0x32 0x0032 #DIGIT TWO +0x33 0x0033 #DIGIT THREE +0x34 0x0034 #DIGIT FOUR +0x35 0x0035 #DIGIT FIVE +0x36 0x0036 #DIGIT SIX +0x37 0x0037 #DIGIT SEVEN +0x38 0x0038 #DIGIT EIGHT +0x39 0x0039 #DIGIT NINE +0x3A 0x003A #COLON +0x3B 0x003B #SEMICOLON +0x3C 0x003C #LESS-THAN SIGN +0x3D 0x003D #EQUALS SIGN +0x3E 0x003E #GREATER-THAN SIGN +0x3F 0x003F #QUESTION MARK +0x40 0x0040 #COMMERCIAL AT +0x41 0x0041 #LATIN CAPITAL LETTER A +0x42 0x0042 #LATIN CAPITAL LETTER B +0x43 0x0043 #LATIN CAPITAL LETTER C +0x44 0x0044 #LATIN CAPITAL LETTER D +0x45 0x0045 #LATIN CAPITAL LETTER E +0x46 0x0046 #LATIN CAPITAL LETTER F +0x47 0x0047 #LATIN CAPITAL LETTER G +0x48 0x0048 #LATIN CAPITAL LETTER H +0x49 0x0049 #LATIN CAPITAL LETTER I +0x4A 0x004A #LATIN CAPITAL LETTER J +0x4B 0x004B #LATIN CAPITAL LETTER K +0x4C 0x004C #LATIN CAPITAL LETTER L +0x4D 0x004D #LATIN CAPITAL LETTER M +0x4E 0x004E #LATIN CAPITAL LETTER N +0x4F 0x004F #LATIN CAPITAL LETTER O +0x50 0x0050 #LATIN CAPITAL LETTER P +0x51 0x0051 #LATIN CAPITAL LETTER Q +0x52 0x0052 #LATIN CAPITAL LETTER R +0x53 0x0053 #LATIN CAPITAL LETTER S +0x54 0x0054 #LATIN CAPITAL LETTER T +0x55 0x0055 #LATIN CAPITAL LETTER U +0x56 0x0056 #LATIN CAPITAL LETTER V +0x57 0x0057 #LATIN CAPITAL LETTER W +0x58 0x0058 #LATIN CAPITAL LETTER X +0x59 0x0059 #LATIN CAPITAL LETTER Y +0x5A 0x005A #LATIN CAPITAL LETTER Z +0x5B 0x005B #LEFT SQUARE BRACKET +0x5C 0x005C #REVERSE SOLIDUS +0x5D 0x005D #RIGHT SQUARE BRACKET +0x5E 0x005E #CIRCUMFLEX ACCENT +0x5F 0x005F #LOW LINE +0x60 0x0060 #GRAVE ACCENT +0x61 0x0061 #LATIN SMALL LETTER A +0x62 0x0062 #LATIN SMALL LETTER B +0x63 0x0063 #LATIN SMALL LETTER C +0x64 0x0064 #LATIN SMALL LETTER D +0x65 0x0065 #LATIN SMALL LETTER E +0x66 0x0066 #LATIN SMALL LETTER F +0x67 0x0067 #LATIN SMALL LETTER G +0x68 0x0068 #LATIN SMALL LETTER H +0x69 0x0069 #LATIN SMALL LETTER I +0x6A 0x006A #LATIN SMALL LETTER J +0x6B 0x006B #LATIN SMALL LETTER K +0x6C 0x006C #LATIN SMALL LETTER L +0x6D 0x006D #LATIN SMALL LETTER M +0x6E 0x006E #LATIN SMALL LETTER N +0x6F 0x006F #LATIN SMALL LETTER O +0x70 0x0070 #LATIN SMALL LETTER P +0x71 0x0071 #LATIN SMALL LETTER Q +0x72 0x0072 #LATIN SMALL LETTER R +0x73 0x0073 #LATIN SMALL LETTER S +0x74 0x0074 #LATIN SMALL LETTER T +0x75 0x0075 #LATIN SMALL LETTER U +0x76 0x0076 #LATIN SMALL LETTER V +0x77 0x0077 #LATIN SMALL LETTER W +0x78 0x0078 #LATIN SMALL LETTER X +0x79 0x0079 #LATIN SMALL LETTER Y +0x7A 0x007A #LATIN SMALL LETTER Z +0x7B 0x007B #LEFT CURLY BRACKET +0x7C 0x007C #VERTICAL LINE +0x7D 0x007D #RIGHT CURLY BRACKET +0x7E 0x007E #TILDE +0x7F 0x007F #DELETE +0x80 0x20AC #EURO SIGN +0x81 #UNDEFINED +0x82 0x201A #SINGLE LOW-9 QUOTATION MARK +0x83 #UNDEFINED +0x84 0x201E #DOUBLE LOW-9 QUOTATION MARK +0x85 0x2026 #HORIZONTAL ELLIPSIS +0x86 0x2020 #DAGGER +0x87 0x2021 #DOUBLE DAGGER +0x88 #UNDEFINED +0x89 0x2030 #PER MILLE SIGN +0x8A 0x0160 #LATIN CAPITAL LETTER S WITH CARON +0x8B 0x2039 #SINGLE LEFT-POINTING ANGLE QUOTATION MARK +0x8C 0x015A #LATIN CAPITAL LETTER S WITH ACUTE +0x8D 0x0164 #LATIN CAPITAL LETTER T WITH CARON +0x8E 0x017D #LATIN CAPITAL LETTER Z WITH CARON +0x8F 0x0179 #LATIN CAPITAL LETTER Z WITH ACUTE +0x90 #UNDEFINED +0x91 0x2018 #LEFT SINGLE QUOTATION MARK +0x92 0x2019 #RIGHT SINGLE QUOTATION MARK +0x93 0x201C #LEFT DOUBLE QUOTATION MARK +0x94 0x201D #RIGHT DOUBLE QUOTATION MARK +0x95 0x2022 #BULLET +0x96 0x2013 #EN DASH +0x97 0x2014 #EM DASH +0x98 #UNDEFINED +0x99 0x2122 #TRADE MARK SIGN +0x9A 0x0161 #LATIN SMALL LETTER S WITH CARON +0x9B 0x203A #SINGLE RIGHT-POINTING ANGLE QUOTATION MARK +0x9C 0x015B #LATIN SMALL LETTER S WITH ACUTE +0x9D 0x0165 #LATIN SMALL LETTER T WITH CARON +0x9E 0x017E #LATIN SMALL LETTER Z WITH CARON +0x9F 0x017A #LATIN SMALL LETTER Z WITH ACUTE +0xA0 0x00A0 #NO-BREAK SPACE +0xA1 0x02C7 #CARON +0xA2 0x02D8 #BREVE +0xA3 0x0141 #LATIN CAPITAL LETTER L WITH STROKE +0xA4 0x00A4 #CURRENCY SIGN +0xA5 0x0104 #LATIN CAPITAL LETTER A WITH OGONEK +0xA6 0x00A6 #BROKEN BAR +0xA7 0x00A7 #SECTION SIGN +0xA8 0x00A8 #DIAERESIS +0xA9 0x00A9 #COPYRIGHT SIGN +0xAA 0x015E #LATIN CAPITAL LETTER S WITH CEDILLA +0xAB 0x00AB #LEFT-POINTING DOUBLE ANGLE QUOTATION MARK +0xAC 0x00AC #NOT SIGN +0xAD 0x00AD #SOFT HYPHEN +0xAE 0x00AE #REGISTERED SIGN +0xAF 0x017B #LATIN CAPITAL LETTER Z WITH DOT ABOVE +0xB0 0x00B0 #DEGREE SIGN +0xB1 0x00B1 #PLUS-MINUS SIGN +0xB2 0x02DB #OGONEK +0xB3 0x0142 #LATIN SMALL LETTER L WITH STROKE +0xB4 0x00B4 #ACUTE ACCENT +0xB5 0x00B5 #MICRO SIGN +0xB6 0x00B6 #PILCROW SIGN +0xB7 0x00B7 #MIDDLE DOT +0xB8 0x00B8 #CEDILLA +0xB9 0x0105 #LATIN SMALL LETTER A WITH OGONEK +0xBA 0x015F #LATIN SMALL LETTER S WITH CEDILLA +0xBB 0x00BB #RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK +0xBC 0x013D #LATIN CAPITAL LETTER L WITH CARON +0xBD 0x02DD #DOUBLE ACUTE ACCENT +0xBE 0x013E #LATIN SMALL LETTER L WITH CARON +0xBF 0x017C #LATIN SMALL LETTER Z WITH DOT ABOVE +0xC0 0x0154 #LATIN CAPITAL LETTER R WITH ACUTE +0xC1 0x00C1 #LATIN CAPITAL LETTER A WITH ACUTE +0xC2 0x00C2 #LATIN CAPITAL LETTER A WITH CIRCUMFLEX +0xC3 0x0102 #LATIN CAPITAL LETTER A WITH BREVE +0xC4 0x00C4 #LATIN CAPITAL LETTER A WITH DIAERESIS +0xC5 0x0139 #LATIN CAPITAL LETTER L WITH ACUTE +0xC6 0x0106 #LATIN CAPITAL LETTER C WITH ACUTE +0xC7 0x00C7 #LATIN CAPITAL LETTER C WITH CEDILLA +0xC8 0x010C #LATIN CAPITAL LETTER C WITH CARON +0xC9 0x00C9 #LATIN CAPITAL LETTER E WITH ACUTE +0xCA 0x0118 #LATIN CAPITAL LETTER E WITH OGONEK +0xCB 0x00CB #LATIN CAPITAL LETTER E WITH DIAERESIS +0xCC 0x011A #LATIN CAPITAL LETTER E WITH CARON +0xCD 0x00CD #LATIN CAPITAL LETTER I WITH ACUTE +0xCE 0x00CE #LATIN CAPITAL LETTER I WITH CIRCUMFLEX +0xCF 0x010E #LATIN CAPITAL LETTER D WITH CARON +0xD0 0x0110 #LATIN CAPITAL LETTER D WITH STROKE +0xD1 0x0143 #LATIN CAPITAL LETTER N WITH ACUTE +0xD2 0x0147 #LATIN CAPITAL LETTER N WITH CARON +0xD3 0x00D3 #LATIN CAPITAL LETTER O WITH ACUTE +0xD4 0x00D4 #LATIN CAPITAL LETTER O WITH CIRCUMFLEX +0xD5 0x0150 #LATIN CAPITAL LETTER O WITH DOUBLE ACUTE +0xD6 0x00D6 #LATIN CAPITAL LETTER O WITH DIAERESIS +0xD7 0x00D7 #MULTIPLICATION SIGN +0xD8 0x0158 #LATIN CAPITAL LETTER R WITH CARON +0xD9 0x016E #LATIN CAPITAL LETTER U WITH RING ABOVE +0xDA 0x00DA #LATIN CAPITAL LETTER U WITH ACUTE +0xDB 0x0170 #LATIN CAPITAL LETTER U WITH DOUBLE ACUTE +0xDC 0x00DC #LATIN CAPITAL LETTER U WITH DIAERESIS +0xDD 0x00DD #LATIN CAPITAL LETTER Y WITH ACUTE +0xDE 0x0162 #LATIN CAPITAL LETTER T WITH CEDILLA +0xDF 0x00DF #LATIN SMALL LETTER SHARP S +0xE0 0x0155 #LATIN SMALL LETTER R WITH ACUTE +0xE1 0x00E1 #LATIN SMALL LETTER A WITH ACUTE +0xE2 0x00E2 #LATIN SMALL LETTER A WITH CIRCUMFLEX +0xE3 0x0103 #LATIN SMALL LETTER A WITH BREVE +0xE4 0x00E4 #LATIN SMALL LETTER A WITH DIAERESIS +0xE5 0x013A #LATIN SMALL LETTER L WITH ACUTE +0xE6 0x0107 #LATIN SMALL LETTER C WITH ACUTE +0xE7 0x00E7 #LATIN SMALL LETTER C WITH CEDILLA +0xE8 0x010D #LATIN SMALL LETTER C WITH CARON +0xE9 0x00E9 #LATIN SMALL LETTER E WITH ACUTE +0xEA 0x0119 #LATIN SMALL LETTER E WITH OGONEK +0xEB 0x00EB #LATIN SMALL LETTER E WITH DIAERESIS +0xEC 0x011B #LATIN SMALL LETTER E WITH CARON +0xED 0x00ED #LATIN SMALL LETTER I WITH ACUTE +0xEE 0x00EE #LATIN SMALL LETTER I WITH CIRCUMFLEX +0xEF 0x010F #LATIN SMALL LETTER D WITH CARON +0xF0 0x0111 #LATIN SMALL LETTER D WITH STROKE +0xF1 0x0144 #LATIN SMALL LETTER N WITH ACUTE +0xF2 0x0148 #LATIN SMALL LETTER N WITH CARON +0xF3 0x00F3 #LATIN SMALL LETTER O WITH ACUTE +0xF4 0x00F4 #LATIN SMALL LETTER O WITH CIRCUMFLEX +0xF5 0x0151 #LATIN SMALL LETTER O WITH DOUBLE ACUTE +0xF6 0x00F6 #LATIN SMALL LETTER O WITH DIAERESIS +0xF7 0x00F7 #DIVISION SIGN +0xF8 0x0159 #LATIN SMALL LETTER R WITH CARON +0xF9 0x016F #LATIN SMALL LETTER U WITH RING ABOVE +0xFA 0x00FA #LATIN SMALL LETTER U WITH ACUTE +0xFB 0x0171 #LATIN SMALL LETTER U WITH DOUBLE ACUTE +0xFC 0x00FC #LATIN SMALL LETTER U WITH DIAERESIS +0xFD 0x00FD #LATIN SMALL LETTER Y WITH ACUTE +0xFE 0x0163 #LATIN SMALL LETTER T WITH CEDILLA +0xFF 0x02D9 #DOT ABOVE From 47df58008191423a54347bf365f61bf0bac74e10 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Thu, 5 Nov 2009 01:07:59 -0600 Subject: [PATCH 70/74] tools.time: overhaul --- basis/bootstrap/tools/tools.factor | 1 + basis/listener/listener.factor | 2 + basis/prettyprint/prettyprint.factor | 4 ++ .../known-words/known-words.factor | 6 +-- basis/tools/memory/memory-docs.factor | 14 +++++- basis/tools/memory/memory-tests.factor | 6 ++- basis/tools/memory/memory.factor | 46 ++++++++----------- basis/tools/profiler/profiler-docs.factor | 2 +- basis/tools/time/time-docs.factor | 23 ++++++++-- basis/tools/time/time.factor | 38 +++++---------- basis/vm/vm.factor | 29 ++++++++---- core/bootstrap/primitives.factor | 7 ++- vm/code_block.cpp | 2 +- vm/dispatch.cpp | 10 ++-- vm/dispatch.hpp | 12 +++++ vm/inline_cache.cpp | 35 +++----------- vm/master.hpp | 2 +- vm/primitives.cpp | 4 -- vm/vm.cpp | 4 +- vm/vm.hpp | 11 +---- 20 files changed, 127 insertions(+), 131 deletions(-) diff --git a/basis/bootstrap/tools/tools.factor b/basis/bootstrap/tools/tools.factor index 848e310d63..51f44025c9 100644 --- a/basis/bootstrap/tools/tools.factor +++ b/basis/bootstrap/tools/tools.factor @@ -12,6 +12,7 @@ IN: bootstrap.tools "tools.deploy" "tools.destructors" "tools.disassembler" + "tools.dispatch" "tools.memory" "tools.profiler" "tools.test" diff --git a/basis/listener/listener.factor b/basis/listener/listener.factor index 57d1fd3964..a42eada563 100644 --- a/basis/listener/listener.factor +++ b/basis/listener/listener.factor @@ -163,8 +163,10 @@ SYMBOL: interactive-vocabs "syntax" "tools.annotations" "tools.crossref" + "tools.deprecation" "tools.destructors" "tools.disassembler" + "tools.dispatch" "tools.errors" "tools.memory" "tools.profiler" diff --git a/basis/prettyprint/prettyprint.factor b/basis/prettyprint/prettyprint.factor index 718de7e84c..6cff399201 100644 --- a/basis/prettyprint/prettyprint.factor +++ b/basis/prettyprint/prettyprint.factor @@ -110,3 +110,7 @@ SYMBOL: pprint-string-cells? ] with-row ] each ] tabular-output nl ; + +: object-table. ( obj alist -- ) + [ [ nip first ] [ second call( obj -- str ) ] 2bi 2array ] with map + simple-table. ; diff --git a/basis/stack-checker/known-words/known-words.factor b/basis/stack-checker/known-words/known-words.factor index aea9c4b1ce..2c0ce853aa 100644 --- a/basis/stack-checker/known-words/known-words.factor +++ b/basis/stack-checker/known-words/known-words.factor @@ -13,7 +13,7 @@ words.private definitions assocs summary compiler.units system.private combinators combinators.short-circuit locals locals.backend locals.types combinators.private stack-checker.values generic.single generic.single.private -alien.libraries +alien.libraries tools.dispatch.private tools.profiler.private stack-checker.alien stack-checker.state stack-checker.errors @@ -705,8 +705,6 @@ M: bad-executable summary \ reset-dispatch-stats { } { } define-primitive \ dispatch-stats { } { array } define-primitive -\ reset-inline-cache-stats { } { } define-primitive -\ inline-cache-stats { } { array } define-primitive \ optimized? { word } { object } define-primitive @@ -716,3 +714,5 @@ M: bad-executable summary \ enable-gc-events { } { } define-primitive \ disable-gc-events { } { object } define-primitive + +\ profiling { object } { } define-primitive diff --git a/basis/tools/memory/memory-docs.factor b/basis/tools/memory/memory-docs.factor index 7ecbf402ab..f729e8945f 100644 --- a/basis/tools/memory/memory-docs.factor +++ b/basis/tools/memory/memory-docs.factor @@ -1,4 +1,4 @@ -USING: help.markup help.syntax memory sequences ; +USING: help.markup help.syntax memory sequences vm ; IN: tools.memory ARTICLE: "tools.memory" "Object memory tools" @@ -39,3 +39,15 @@ HELP: heap-stats. { $description "For each class, prints the number of instances and total memory consumed by those instances." } ; { heap-stats heap-stats. } related-words + +HELP: gc-events. +{ $description "Prints all garbage collection events that took place during the last call to " { $link collect-gc-events } "." } ; + +HELP: gc-stats. +{ $description "Prints a breakdown of different garbage collection events that took place during the last call to " { $link collect-gc-events } "." } ; + +HELP: gc-summary. +{ $description "Prints aggregate garbage collection statistics from the last call to " { $link collect-gc-events } "." } ; + +HELP: gc-events +{ $var-description "A sequence of " { $link gc-event } " instances, set by " { $link collect-gc-events } ". Can be inspected directly, or with the " { $link gc-events. } ", " { $link gc-stats. } " and " { $link gc-summary. } " words." } ; diff --git a/basis/tools/memory/memory-tests.factor b/basis/tools/memory/memory-tests.factor index 4b75cf0bfa..4711f472a3 100644 --- a/basis/tools/memory/memory-tests.factor +++ b/basis/tools/memory/memory-tests.factor @@ -1,5 +1,9 @@ -USING: tools.test tools.memory ; +USING: tools.test tools.memory memory ; IN: tools.memory.tests [ ] [ room. ] unit-test [ ] [ heap-stats. ] unit-test +[ ] [ [ gc gc ] collect-gc-events ] unit-test +[ ] [ gc-events. ] unit-test +[ ] [ gc-stats. ] unit-test +[ ] [ gc-summary. ] unit-test diff --git a/basis/tools/memory/memory.factor b/basis/tools/memory/memory.factor index 93151b403e..c147426a6f 100644 --- a/basis/tools/memory/memory.factor +++ b/basis/tools/memory/memory.factor @@ -22,16 +22,12 @@ IN: tools.memory : micros>string ( n -- str ) commas " µs" append ; -: fancy-table. ( obj alist -- ) - [ [ nip first ] [ second call( obj -- str ) ] 2bi 2array ] with map - simple-table. ; - : copying-room. ( copying-sizes -- ) { { "Size:" [ size>> kilobytes ] } { "Occupied:" [ occupied>> kilobytes ] } { "Free:" [ free>> kilobytes ] } - } fancy-table. ; + } object-table. ; : nursery-room. ( data-room -- ) "- Nursery space" print nursery>> copying-room. ; @@ -46,7 +42,7 @@ IN: tools.memory { "Total free:" [ total-free>> kilobytes ] } { "Contiguous free:" [ contiguous-free>> kilobytes ] } { "Free block count:" [ free-block-count>> number>string ] } - } fancy-table. ; + } object-table. ; : tenured-room. ( data-room -- ) "- Tenured space" print tenured>> mark-sweep-table. ; @@ -57,7 +53,7 @@ IN: tools.memory { "Card array:" [ cards>> kilobytes ] } { "Deck array:" [ decks>> kilobytes ] } { "Mark stack:" [ mark-stack>> kilobytes ] } - } fancy-table. ; + } object-table. ; : data-room. ( -- ) "== Data heap ==" print nl @@ -100,9 +96,12 @@ PRIVATE> ] each 2drop ] tabular-output nl ; -: collect-gc-events ( quot -- events ) - enable-gc-events [ ] [ disable-gc-events drop ] cleanup - disable-gc-events byte-array>gc-event-array ; inline +SYMBOL: gc-events + +: collect-gc-events ( quot -- ) + enable-gc-events + [ ] [ disable-gc-events drop ] cleanup + disable-gc-events byte-array>gc-event-array gc-events set ; inline > ] [ space-occupied-before ] bi 2array , ] - [ [ [ start-time>> ] [ total-time>> ] bi + ] [ space-occupied-after ] bi 2array , ] - bi - ] each - ] { } make ; - PRIVATE> : gc-event. ( event -- ) @@ -185,13 +175,16 @@ PRIVATE> { "Event type:" [ op>> gc-op-string ] } { "Total time:" [ total-time>> micros>string ] } { "Space reclaimed:" [ space-reclaimed kilobytes ] } - } fancy-table. ; + } object-table. ; -: gc-stats. ( events -- ) - compute-gc-stats gc-stats-table simple-table. ; +: gc-events. ( -- ) + gc-events get [ gc-event. nl ] each ; -: gc-summary. ( events -- ) - { +: gc-stats. ( -- ) + gc-events get compute-gc-stats gc-stats-table simple-table. ; + +: gc-summary. ( -- ) + gc-events get { { "Collections:" [ length commas ] } { "Cards scanned:" [ [ cards-scanned>> ] map-sum commas ] } { "Decks scanned:" [ [ decks-scanned>> ] map-sum commas ] } @@ -202,7 +195,4 @@ PRIVATE> { "Data heap sweep time:" [ [ data-sweep-time>> ] map-sum micros>string ] } { "Code heap sweep time:" [ [ code-sweep-time>> ] map-sum micros>string ] } { "Compaction time:" [ [ compaction-time>> ] map-sum micros>string ] } - } fancy-table. ; - -: heap-sizes. ( events -- ) - heap-sizes simple-table. ; + } object-table. ; diff --git a/basis/tools/profiler/profiler-docs.factor b/basis/tools/profiler/profiler-docs.factor index 0fda4a65e5..66ae5d7bd3 100644 --- a/basis/tools/profiler/profiler-docs.factor +++ b/basis/tools/profiler/profiler-docs.factor @@ -25,7 +25,7 @@ $nl method-profile. "profiler-limitations" } -{ $see-also "ui.tools.profiler" } ; +{ $see-also "ui.tools.profiler" "tools.annotations" "timing" } ; ABOUT: "profiling" diff --git a/basis/tools/time/time-docs.factor b/basis/tools/time/time-docs.factor index 3f8cbbf799..9e892c33ec 100644 --- a/basis/tools/time/time-docs.factor +++ b/basis/tools/time/time-docs.factor @@ -1,9 +1,12 @@ -USING: help.markup help.syntax memory system ; +USING: help.markup help.syntax memory system tools.dispatch +tools.memory quotations vm ; IN: tools.time -ARTICLE: "timing" "Timing code" +ARTICLE: "timing" "Timing code and collecting statistics" "You can time the execution of a quotation in the listener:" { $subsections time } +"This word also collects statistics about method dispatch and garbage collection:" +{ $subsections dispatch-stats. gc-events. gc-stats. gc-summary. } "A lower-level word puts timings on the stack, intead of printing:" { $subsections benchmark } "You can also read the system clock directly:" @@ -13,13 +16,23 @@ ARTICLE: "timing" "Timing code" ABOUT: "timing" HELP: benchmark -{ $values { "quot" "a quotation" } +{ $values { "quot" quotation } { "runtime" "the runtime in microseconds" } } { $description "Runs a quotation, measuring the total wall clock time." } { $notes "A nicer word for interactive use is " { $link time } "." } ; HELP: time -{ $values { "quot" "a quotation" } } -{ $description "Runs a quotation and then prints the total run time and some statistics." } ; +{ $values { "quot" quotation } } +{ $description "Runs a quotation, gathering statistics about method dispatch and garbage collection, and then prints the total run time." } ; { benchmark micros time } related-words + +HELP: collect-gc-events +{ $values { "quot" quotation } } +{ $description "Calls the quotation, storing an array of " { $link gc-event } " instances in the " { $link gc-events } " variable." } +{ $notes "The " { $link time } " combinator automatically calls this combinator." } ; + +HELP: collect-dispatch-stats +{ $values { "quot" quotation } } +{ $description "Calls the quotation, collecting method dispatch statistics and storing them in the " { $link last-dispatch-stats } " variable. " } +{ $notes "The " { $link time } " combinator automatically calls this combinator." } ; diff --git a/basis/tools/time/time.factor b/basis/tools/time/time.factor index af1b528051..3724a741b7 100644 --- a/basis/tools/time/time.factor +++ b/basis/tools/time/time.factor @@ -1,38 +1,22 @@ ! Copyright (C) 2003, 2009 Slava Pestov. ! See http://factorcode.org/license.txt for BSD license. -USING: kernel math memory io io.styles prettyprint -namespaces system sequences splitting grouping assocs strings -generic.single combinators ; +USING: system kernel math io prettyprint tools.memory +tools.dispatch ; IN: tools.time : benchmark ( quot -- runtime ) micros [ call micros ] dip - ; inline : time. ( time -- ) - "== Running time ==" print nl 1000000 /f pprint " seconds" print ; + "Running time: " write 1000000 /f pprint " seconds" print ; -: dispatch-stats. ( stats -- ) - "== Megamorphic caches ==" print nl - [ { "Hits" "Misses" } ] dip zip simple-table. ; - -: inline-cache-stats. ( stats -- ) - "== Polymorphic inline caches ==" print nl - 3 cut - [ - "- Transitions:" print - [ { "Cold to monomorphic" "Mono to polymorphic" "Poly to megamorphic" } ] dip zip - simple-table. nl - ] [ - "- Type check stubs:" print - [ { "Tag" "Tuple" } ] dip zip - simple-table. - ] bi* ; +: time-banner. ( -- ) + "Additional information was collected." print + "dispatch-stats. - Print method dispatch statistics" print + "gc-events. - Print all garbage collection events" print + "gc-stats. - Print breakdown of different garbage collection events" print + "gc-summary. - Print aggregate garbage collection statistics" print ; : time ( quot -- ) - reset-dispatch-stats - reset-inline-cache-stats - benchmark dispatch-stats inline-cache-stats - [ time. nl ] - [ dispatch-stats. nl ] - [ inline-cache-stats. ] - tri* ; inline + [ [ benchmark ] collect-dispatch-stats ] collect-gc-events + time. nl time-banner. ; inline diff --git a/basis/vm/vm.factor b/basis/vm/vm.factor index 94fa0fcaf5..ba057edffa 100644 --- a/basis/vm/vm.factor +++ b/basis/vm/vm.factor @@ -7,17 +7,17 @@ TYPEDEF: intptr_t cell C-TYPE: context STRUCT: zone - { start cell } - { here cell } - { size cell } - { end cell } ; +{ start cell } +{ here cell } +{ size cell } +{ end cell } ; STRUCT: vm - { stack_chain context* } - { nursery zone } - { cards_offset cell } - { decks_offset cell } - { userenv cell[70] } ; +{ stack_chain context* } +{ nursery zone } +{ cards_offset cell } +{ decks_offset cell } +{ userenv cell[70] } ; : vm-field-offset ( field -- offset ) vm offset-of ; inline @@ -66,3 +66,14 @@ STRUCT: gc-event { code-sweep-time cell } { compaction-time cell } { temp-time cell } ; + +STRUCT: dispatch-statistics +{ megamorphic-cache-hits cell } +{ megamorphic-cache-misses cell } + +{ cold-call-to-ic-transitions cell } +{ ic-to-pic-transitions cell } +{ pic-to-mega-transitions cell } + +{ pic-tag-count cell } +{ pic-tuple-count cell } ; diff --git a/core/bootstrap/primitives.factor b/core/bootstrap/primitives.factor index 92f6c6f551..5d4144e354 100644 --- a/core/bootstrap/primitives.factor +++ b/core/bootstrap/primitives.factor @@ -99,6 +99,7 @@ bootstrapping? on "system" "system.private" "threads.private" + "tools.dispatch.private" "tools.profiler.private" "words" "words.private" @@ -511,10 +512,8 @@ tuple { "inline-cache-miss-tail" "generic.single.private" (( generic methods index cache -- )) } { "mega-cache-miss" "generic.single.private" (( methods index cache -- method )) } { "lookup-method" "generic.single.private" (( object methods -- method )) } - { "reset-dispatch-stats" "generic.single" (( -- )) } - { "dispatch-stats" "generic.single" (( -- stats )) } - { "reset-inline-cache-stats" "generic.single" (( -- )) } - { "inline-cache-stats" "generic.single" (( -- stats )) } + { "reset-dispatch-stats" "tools.dispatch.private" (( -- )) } + { "dispatch-stats" "tools.dispatch.private" (( -- stats )) } { "optimized?" "words" (( word -- ? )) } { "quot-compiled?" "quotations" (( quot -- ? )) } { "vm-ptr" "vm" (( -- ptr )) } diff --git a/vm/code_block.cpp b/vm/code_block.cpp index 56dd27c6b7..dc6a488e26 100755 --- a/vm/code_block.cpp +++ b/vm/code_block.cpp @@ -178,7 +178,7 @@ cell factor_vm::compute_relocation(relocation_entry rel, cell index, code_block case RT_UNTAGGED: return untag_fixnum(ARG); case RT_MEGAMORPHIC_CACHE_HITS: - return (cell)&megamorphic_cache_hits; + return (cell)&dispatch_stats.megamorphic_cache_hits; case RT_VM: return (cell)this + untag_fixnum(ARG); case RT_CARDS_OFFSET: diff --git a/vm/dispatch.cpp b/vm/dispatch.cpp index 6bef970944..3eba483fe6 100755 --- a/vm/dispatch.cpp +++ b/vm/dispatch.cpp @@ -118,7 +118,7 @@ void factor_vm::update_method_cache(cell cache, cell klass, cell method) void factor_vm::primitive_mega_cache_miss() { - megamorphic_cache_misses++; + dispatch_stats.megamorphic_cache_misses++; cell cache = dpop(); fixnum index = untag_fixnum(dpop()); @@ -135,16 +135,12 @@ void factor_vm::primitive_mega_cache_miss() void factor_vm::primitive_reset_dispatch_stats() { - megamorphic_cache_hits = megamorphic_cache_misses = 0; + memset(&dispatch_stats,0,sizeof(dispatch_statistics)); } void factor_vm::primitive_dispatch_stats() { - growable_array stats(this); - stats.add(allot_cell(megamorphic_cache_hits)); - stats.add(allot_cell(megamorphic_cache_misses)); - stats.trim(); - dpush(stats.elements.value()); + dpush(tag(byte_array_from_value(&dispatch_stats))); } void quotation_jit::emit_mega_cache_lookup(cell methods_, fixnum index, cell cache_) diff --git a/vm/dispatch.hpp b/vm/dispatch.hpp index 412ef35bb4..87d08e2760 100644 --- a/vm/dispatch.hpp +++ b/vm/dispatch.hpp @@ -1,4 +1,16 @@ namespace factor { +struct dispatch_statistics { + cell megamorphic_cache_hits; + cell megamorphic_cache_misses; + + cell cold_call_to_ic_transitions; + cell ic_to_pic_transitions; + cell pic_to_mega_transitions; + + cell pic_tag_count; + cell pic_tuple_count; +}; + } diff --git a/vm/inline_cache.cpp b/vm/inline_cache.cpp index 21912966d0..469bb8bf2e 100755 --- a/vm/inline_cache.cpp +++ b/vm/inline_cache.cpp @@ -6,11 +6,6 @@ namespace factor void factor_vm::init_inline_caching(int max_size) { max_pic_size = max_size; - cold_call_to_ic_transitions = 0; - ic_to_pic_transitions = 0; - pic_to_mega_transitions = 0; - pic_counts[0] = 0; - pic_counts[1] = 0; } void factor_vm::deallocate_inline_cache(cell return_address) @@ -48,7 +43,10 @@ cell factor_vm::determine_inline_cache_type(array *cache_entries) void factor_vm::update_pic_count(cell type) { - pic_counts[type - PIC_TAG]++; + if(type == PIC_TAG) + dispatch_stats.pic_tag_count++; + else + dispatch_stats.pic_tuple_count++; } struct inline_cache_jit : public jit { @@ -167,11 +165,11 @@ cell factor_vm::add_inline_cache_entry(cell cache_entries_, cell klass_, cell me void factor_vm::update_pic_transitions(cell pic_size) { if(pic_size == max_pic_size) - pic_to_mega_transitions++; + dispatch_stats.pic_to_mega_transitions++; else if(pic_size == 0) - cold_call_to_ic_transitions++; + dispatch_stats.cold_call_to_ic_transitions++; else if(pic_size == 1) - ic_to_pic_transitions++; + dispatch_stats.ic_to_pic_transitions++; } /* The cache_entries parameter is empty (on cold call site) or has entries @@ -241,23 +239,4 @@ VM_C_API void *inline_cache_miss(cell return_address, factor_vm *parent) return parent->inline_cache_miss(return_address); } -void factor_vm::primitive_reset_inline_cache_stats() -{ - cold_call_to_ic_transitions = ic_to_pic_transitions = pic_to_mega_transitions = 0; - pic_counts[0] = 0; - pic_counts[1] = 0; -} - -void factor_vm::primitive_inline_cache_stats() -{ - growable_array stats(this); - stats.add(allot_cell(cold_call_to_ic_transitions)); - stats.add(allot_cell(ic_to_pic_transitions)); - stats.add(allot_cell(pic_to_mega_transitions)); - stats.add(allot_cell(pic_counts[0])); - stats.add(allot_cell(pic_counts[1])); - stats.trim(); - dpush(stats.elements.value()); -} - } diff --git a/vm/master.hpp b/vm/master.hpp index 0654e63bcb..39242a36af 100755 --- a/vm/master.hpp +++ b/vm/master.hpp @@ -71,6 +71,7 @@ namespace factor #include "image.hpp" #include "alien.hpp" #include "callbacks.hpp" +#include "dispatch.hpp" #include "vm.hpp" #include "allot.hpp" #include "tagged.hpp" @@ -93,7 +94,6 @@ namespace factor #include "byte_arrays.hpp" #include "jit.hpp" #include "quotations.hpp" -#include "dispatch.hpp" #include "inline_cache.hpp" #include "factor.hpp" #include "utilities.hpp" diff --git a/vm/primitives.cpp b/vm/primitives.cpp index f950e19e5a..957e6128ed 100644 --- a/vm/primitives.cpp +++ b/vm/primitives.cpp @@ -121,8 +121,6 @@ PRIMITIVE_FORWARD(mega_cache_miss) PRIMITIVE_FORWARD(lookup_method) PRIMITIVE_FORWARD(reset_dispatch_stats) PRIMITIVE_FORWARD(dispatch_stats) -PRIMITIVE_FORWARD(reset_inline_cache_stats) -PRIMITIVE_FORWARD(inline_cache_stats) PRIMITIVE_FORWARD(optimized_p) PRIMITIVE_FORWARD(quot_compiled_p) PRIMITIVE_FORWARD(vm_ptr) @@ -287,8 +285,6 @@ const primitive_type primitives[] = { primitive_lookup_method, primitive_reset_dispatch_stats, primitive_dispatch_stats, - primitive_reset_inline_cache_stats, - primitive_inline_cache_stats, primitive_optimized_p, primitive_quot_compiled_p, primitive_vm_ptr, diff --git a/vm/vm.cpp b/vm/vm.cpp index 2a41e96dfe..72c63292fd 100755 --- a/vm/vm.cpp +++ b/vm/vm.cpp @@ -11,6 +11,8 @@ factor_vm::factor_vm() : gc_events(NULL), fep_disabled(false), full_output(false) - { } +{ + primitive_reset_dispatch_stats(); +} } diff --git a/vm/vm.hpp b/vm/vm.hpp index aa04c8dffc..aa5a3051e6 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -80,14 +80,7 @@ struct factor_vm cell bignum_neg_one; /* Method dispatch statistics */ - cell megamorphic_cache_hits; - cell megamorphic_cache_misses; - - cell cold_call_to_ic_transitions; - cell ic_to_pic_transitions; - cell pic_to_mega_transitions; - /* Indexed by PIC_TAG, PIC_TUPLE */ - cell pic_counts[2]; + dispatch_statistics dispatch_stats; /* Number of entries in a polymorphic inline cache */ cell max_pic_size; @@ -646,8 +639,6 @@ struct factor_vm cell add_inline_cache_entry(cell cache_entries_, cell klass_, cell method_); void update_pic_transitions(cell pic_size); void *inline_cache_miss(cell return_address); - void primitive_reset_inline_cache_stats(); - void primitive_inline_cache_stats(); //factor void default_parameters(vm_parameters *p); From c6b0a91f3476493ae9bab1f4e6f5ab381d52e00f Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Thu, 5 Nov 2009 01:08:58 -0600 Subject: [PATCH 71/74] alien.c-types: use (byte-array) instead of in out parameter constructors --- basis/alien/c-types/c-types.factor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/basis/alien/c-types/c-types.factor b/basis/alien/c-types/c-types.factor index 119e437734..e06c543b54 100755 --- a/basis/alien/c-types/c-types.factor +++ b/basis/alien/c-types/c-types.factor @@ -292,7 +292,7 @@ M: long-long-type box-return ( c-type -- ) : define-out ( name -- ) [ "alien.c-types" constructor-word ] - [ dup c-setter '[ _ heap-size [ 0 @ ] keep ] ] bi + [ dup c-setter '[ _ heap-size (byte-array) [ 0 @ ] keep ] ] bi (( value -- c-ptr )) define-inline ; : define-primitive-type ( c-type name -- ) From fba6ddbc22fb5c40ba6fd35437f5d9e0be9482cc Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Thu, 5 Nov 2009 01:36:14 -0600 Subject: [PATCH 72/74] Move platform-specific c-type initialization out of cpu.* vocabularies and into alien.c-types so that the vm vocabulary, which is loaded before cpu.*, will have correct struct offsets --- basis/alien/c-types/c-types.factor | 52 ++++++++++++++++++++--------- basis/alien/data/data.factor | 6 ---- basis/cpu/ppc/ppc.factor | 11 ------ basis/cpu/x86/32/32.factor | 6 ---- basis/cpu/x86/64/winnt/winnt.factor | 6 ---- 5 files changed, 37 insertions(+), 44 deletions(-) diff --git a/basis/alien/c-types/c-types.factor b/basis/alien/c-types/c-types.factor index e06c543b54..cfbed5378d 100755 --- a/basis/alien/c-types/c-types.factor +++ b/basis/alien/c-types/c-types.factor @@ -230,6 +230,10 @@ M: byte-array byte-length length ; inline M: f byte-length drop 0 ; inline +: >c-bool ( ? -- int ) 1 0 ? ; inline + +: c-bool> ( int -- ? ) 0 = not ; inline + MIXIN: value-type : c-getter ( name -- quot ) @@ -256,6 +260,7 @@ PREDICATE: typedef-word < c-type-word "c-type" word-prop c-type-name? ; M: string typedef ( old new -- ) c-types get set-at ; + M: word typedef ( old new -- ) { [ nip define-symbol ] @@ -338,7 +343,7 @@ SYMBOLS: [ alien-signed-8 ] >>getter [ set-alien-signed-8 ] >>setter 8 >>size - 8 >>align + cpu x86.32? os windows? not and 4 8 ? >>align "box_signed_8" >>boxer "to_signed_8" >>unboxer \ longlong define-primitive-type @@ -349,7 +354,7 @@ SYMBOLS: [ alien-unsigned-8 ] >>getter [ set-alien-unsigned-8 ] >>setter 8 >>size - 8 >>align + cpu x86.32? os windows? not and 4 8 ? >>align "box_unsigned_8" >>boxer "to_unsigned_8" >>unboxer \ ulonglong define-primitive-type @@ -442,14 +447,24 @@ SYMBOLS: "to_cell" >>unboxer \ uchar define-primitive-type - - [ alien-unsigned-1 0 = not ] >>getter - [ [ 1 0 ? ] 2dip set-alien-unsigned-1 ] >>setter - 1 >>size - 1 >>align - "box_boolean" >>boxer - "to_boolean" >>unboxer - \ bool define-primitive-type + cpu ppc? [ + + [ alien-unsigned-4 c-bool> ] >>getter + [ [ >c-bool ] 2dip set-alien-unsigned-4 ] >>setter + 4 >>size + 4 >>align + "box_boolean" >>boxer + "to_boolean" >>unboxer + ] [ + + [ alien-unsigned-1 c-bool> ] >>getter + [ [ >c-bool ] 2dip set-alien-unsigned-1 ] >>setter + 1 >>size + 1 >>align + "box_boolean" >>boxer + "to_boolean" >>unboxer + \ bool define-primitive-type + ] if math:float >>class @@ -470,17 +485,24 @@ SYMBOLS: [ alien-double ] >>getter [ [ >float ] 2dip set-alien-double ] >>setter 8 >>size - 8 >>align + cpu x86.32? os windows? not and 4 8 ? >>align "box_double" >>boxer "to_double" >>unboxer double-rep >>rep [ >float ] >>unboxer-quot \ double define-primitive-type - \ long c-type \ ptrdiff_t typedef - \ long c-type \ intptr_t typedef - \ ulong c-type \ uintptr_t typedef - \ ulong c-type \ size_t typedef + cpu x86.64? os windows? and [ + \ longlong c-type \ ptrdiff_t typedef + \ longlong c-type \ intptr_t typedef + \ ulonglong c-type \ uintptr_t typedef + \ ulonglong c-type \ size_t typedef + ] [ + \ long c-type \ ptrdiff_t typedef + \ long c-type \ intptr_t typedef + \ ulong c-type \ uintptr_t typedef + \ ulong c-type \ size_t typedef + ] if ] with-compilation-unit M: char-16-rep rep-component-type drop char ; diff --git a/basis/alien/data/data.factor b/basis/alien/data/data.factor index fc18921ef1..93b1afd436 100644 --- a/basis/alien/data/data.factor +++ b/basis/alien/data/data.factor @@ -65,10 +65,6 @@ M: memory-stream stream-read : byte-array>memory ( byte-array base -- ) swap dup byte-length memcpy ; inline -: >c-bool ( ? -- int ) 1 0 ? ; inline - -: c-bool> ( int -- ? ) 0 = not ; inline - M: value-type c-type-rep drop int-rep ; M: value-type c-type-getter @@ -77,5 +73,3 @@ M: value-type c-type-getter M: value-type c-type-setter ( type -- quot ) [ c-type-getter ] [ c-type-unboxer-quot ] [ heap-size ] tri '[ @ swap @ _ memcpy ] ; - - diff --git a/basis/cpu/ppc/ppc.factor b/basis/cpu/ppc/ppc.factor index 92cea0d82f..0f33df8df7 100644 --- a/basis/cpu/ppc/ppc.factor +++ b/basis/cpu/ppc/ppc.factor @@ -737,14 +737,3 @@ USE: vocabs.loader } cond "complex-double" c-type t >>return-in-registers? drop - -[ - - [ alien-unsigned-4 c-bool> ] >>getter - [ [ >c-bool ] 2dip set-alien-unsigned-4 ] >>setter - 4 >>size - 4 >>align - "box_boolean" >>boxer - "to_boolean" >>unboxer - bool define-primitive-type -] with-compilation-unit diff --git a/basis/cpu/x86/32/32.factor b/basis/cpu/x86/32/32.factor index 8a29c82dad..8867ca6597 100755 --- a/basis/cpu/x86/32/32.factor +++ b/basis/cpu/x86/32/32.factor @@ -324,10 +324,4 @@ M: x86.32 dummy-fp-params? f ; ! Dreadful M: object flatten-value-type (flatten-int-type) ; -os windows? [ - cell longlong c-type (>>align) - cell ulonglong c-type (>>align) - 4 double c-type (>>align) -] unless - check-sse diff --git a/basis/cpu/x86/64/winnt/winnt.factor b/basis/cpu/x86/64/winnt/winnt.factor index 3ecd56bdd1..a398c6565c 100644 --- a/basis/cpu/x86/64/winnt/winnt.factor +++ b/basis/cpu/x86/64/winnt/winnt.factor @@ -24,9 +24,3 @@ M: x86.64 dummy-fp-params? t ; M: x86.64 temp-reg RAX ; -<< -longlong ptrdiff_t typedef -longlong intptr_t typedef -int c-type long define-primitive-type -uint c-type ulong define-primitive-type ->> From 9ca1ab7ccc742f8178d984c01bde23e4a8a0aa76 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Thu, 5 Nov 2009 01:36:26 -0600 Subject: [PATCH 73/74] tools.dispatch: split off method dispatch statistics from tools.time --- basis/tools/dispatch/authors.txt | 1 + basis/tools/dispatch/dispatch-docs.factor | 8 ++++++++ basis/tools/dispatch/dispatch.factor | 24 +++++++++++++++++++++++ 3 files changed, 33 insertions(+) create mode 100644 basis/tools/dispatch/authors.txt create mode 100644 basis/tools/dispatch/dispatch-docs.factor create mode 100644 basis/tools/dispatch/dispatch.factor diff --git a/basis/tools/dispatch/authors.txt b/basis/tools/dispatch/authors.txt new file mode 100644 index 0000000000..d4f5d6b3ae --- /dev/null +++ b/basis/tools/dispatch/authors.txt @@ -0,0 +1 @@ +Slava Pestov \ No newline at end of file diff --git a/basis/tools/dispatch/dispatch-docs.factor b/basis/tools/dispatch/dispatch-docs.factor new file mode 100644 index 0000000000..e93ea165c1 --- /dev/null +++ b/basis/tools/dispatch/dispatch-docs.factor @@ -0,0 +1,8 @@ +IN: tools.dispatch +USING: help.markup help.syntax vm quotations ; + +HELP: last-dispatch-stats +{ $var-description "A " { $link dispatch-statistics } " instance, set by " { $link collect-dispatch-stats } "." } ; + +HELP: dispatch-stats. +{ $description "Prints method dispatch statistics from the last call to " { $link collect-dispatch-stats } "." } ; diff --git a/basis/tools/dispatch/dispatch.factor b/basis/tools/dispatch/dispatch.factor new file mode 100644 index 0000000000..7d30dac36b --- /dev/null +++ b/basis/tools/dispatch/dispatch.factor @@ -0,0 +1,24 @@ +! Copyright (C) 2009 Slava Pestov. +! See http://factorcode.org/license.txt for BSD license. +USING: accessors kernel namespaces prettyprint classes.struct +vm tools.dispatch.private ; +IN: tools.dispatch + +SYMBOL: last-dispatch-stats + +: dispatch-stats. ( -- ) + last-dispatch-stats get { + { "Megamorphic hits" [ megamorphic-cache-hits>> ] } + { "Megamorphic misses" [ megamorphic-cache-misses>> ] } + { "Cold to monomorphic" [ cold-call-to-ic-transitions>> ] } + { "Mono to polymorphic" [ ic-to-pic-transitions>> ] } + { "Poly to megamorphic" [ pic-to-mega-transitions>> ] } + { "Tag check count" [ pic-tag-count>> ] } + { "Tuple check count" [ pic-tuple-count>> ] } + } object-table. ; + +: collect-dispatch-stats ( quot -- ) + reset-dispatch-stats + call + dispatch-stats dispatch-statistics memory>struct + last-dispatch-stats set ; inline From c5ecefc6ef4726260f3e7856de4636d457e742be Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Thu, 5 Nov 2009 01:54:49 -0600 Subject: [PATCH 74/74] game.input.dinput.keys-array: fix load error --- basis/game/input/dinput/keys-array/keys-array.factor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/basis/game/input/dinput/keys-array/keys-array.factor b/basis/game/input/dinput/keys-array/keys-array.factor index 3426b89141..b9f21f70a2 100755 --- a/basis/game/input/dinput/keys-array/keys-array.factor +++ b/basis/game/input/dinput/keys-array/keys-array.factor @@ -1,5 +1,5 @@ USING: sequences sequences.private math -accessors alien.data ; +accessors alien.c-types ; IN: game.input.dinput.keys-array TUPLE: keys-array