diff --git a/vm/code_block.cpp b/vm/code_block.cpp index d2337d71de..7e6892202a 100755 --- a/vm/code_block.cpp +++ b/vm/code_block.cpp @@ -346,7 +346,7 @@ void factor_vm::update_word_references(code_block *compiled) are referenced after this is done. So instead of polluting the code heap with dead PICs that will be freed on the next GC, we add them to the free list immediately. */ - else if(compiled->type() == PIC_TYPE) + else if(compiled->pic_p()) code->code_heap_free(compiled); else { @@ -437,9 +437,9 @@ void factor_vm::fixup_labels(array *labels, code_block *compiled) } /* Might GC */ -code_block *factor_vm::allot_code_block(cell size, cell type) +code_block *factor_vm::allot_code_block(cell size, code_block_type type) { - heap_block *block = code->heap_allot(size + sizeof(code_block),type); + heap_block *block = code->heap_allot(size + sizeof(code_block)); /* If allocation failed, do a full GC and compact the code heap. A full GC that occurs as a result of the data heap filling up does not @@ -449,7 +449,7 @@ code_block *factor_vm::allot_code_block(cell size, cell type) if(block == NULL) { primitive_compact_gc(); - block = code->heap_allot(size + sizeof(code_block),type); + block = code->heap_allot(size + sizeof(code_block)); /* Insufficient room even after code GC, give up */ if(block == NULL) @@ -465,11 +465,13 @@ code_block *factor_vm::allot_code_block(cell size, cell type) } } - return (code_block *)block; + code_block *compiled = (code_block *)block; + compiled->set_type(type); + return compiled; } /* Might GC */ -code_block *factor_vm::add_code_block(cell type, cell code_, cell labels_, cell owner_, cell relocation_, cell literals_) +code_block *factor_vm::add_code_block(code_block_type type, cell code_, cell labels_, cell owner_, cell relocation_, cell literals_) { gc_root code(code_,this); gc_root labels(labels_,this); diff --git a/vm/code_heap.cpp b/vm/code_heap.cpp index 756dfdbff6..19c9c87395 100755 --- a/vm/code_heap.cpp +++ b/vm/code_heap.cpp @@ -144,7 +144,7 @@ void factor_vm::primitive_modify_code_heap() cell code = array_nth(compiled_data,4); code_block *compiled = add_code_block( - WORD_TYPE, + code_block_optimized, code, labels, owner, diff --git a/vm/debug.cpp b/vm/debug.cpp index bcd9e6d4d6..a777c5f970 100755 --- a/vm/debug.cpp +++ b/vm/debug.cpp @@ -295,7 +295,7 @@ void factor_vm::dump_code_heap() while(scan != end) { const char *status; - if(scan->type() == FREE_BLOCK_TYPE) + if(scan->free_p()) status = "free"; else if(code->state->is_marked_p(scan)) { diff --git a/vm/heap.cpp b/vm/heap.cpp index 2132ba1a20..e13001ff4f 100644 --- a/vm/heap.cpp +++ b/vm/heap.cpp @@ -49,15 +49,16 @@ void heap::build_free_list(cell size) { clear_free_list(); free_heap_block *end = (free_heap_block *)(seg->start + size); - end->set_type(FREE_BLOCK_TYPE); + end->set_free(); end->set_size(seg->end - (cell)end); add_to_free_list(end); } void heap::assert_free_block(free_heap_block *block) { - if(block->type() != FREE_BLOCK_TYPE) - critical_error("Invalid block in free list",(cell)block); +#ifdef FACTOR_DEBUG + assert(block->free_p()); +#endif } free_heap_block *heap::find_free_block(cell size) @@ -102,11 +103,11 @@ free_heap_block *heap::find_free_block(cell size) free_heap_block *heap::split_free_block(free_heap_block *block, cell size) { - if(block->size() != size ) + if(block->size() != size) { /* split the block in two */ free_heap_block *split = (free_heap_block *)((cell)block + size); - split->set_type(FREE_BLOCK_TYPE); + split->set_free(); split->set_size(block->size() - size); split->next_free = block->next_free; block->set_size(size); @@ -116,27 +117,25 @@ free_heap_block *heap::split_free_block(free_heap_block *block, cell size) return block; } -/* Allocate a block of memory from the mark and sweep GC heap */ -heap_block *heap::heap_allot(cell size, cell type) +heap_block *heap::heap_allot(cell size) { - size = (size + block_size_increment - 1) & ~(block_size_increment - 1); + size = align(size,block_size_increment); free_heap_block *block = find_free_block(size); if(block) { block = split_free_block(block,size); - block->set_type(type); return block; } else return NULL; } -/* Deallocates a block manually */ void heap::heap_free(heap_block *block) { - block->set_type(FREE_BLOCK_TYPE); - add_to_free_list((free_heap_block *)block); + free_heap_block *free_block = (free_heap_block *)block; + free_block->set_free(); + add_to_free_list(free_block); } void heap::mark_block(heap_block *block) @@ -158,7 +157,7 @@ void heap::heap_usage(cell *used, cell *total_free, cell *max_free) { cell size = scan->size(); - if(scan->type() == FREE_BLOCK_TYPE) + if(scan->free_p()) { *total_free += size; if(size > *max_free) @@ -179,31 +178,19 @@ cell heap::heap_size() while(scan != end) { - if(scan->type() == FREE_BLOCK_TYPE) break; + if(scan->free_p()) break; else scan = scan->next(); } - assert(scan->type() == FREE_BLOCK_TYPE); - assert((cell)scan + scan->size() == seg->end); - - return (cell)scan - (cell)first_block(); -} - -heap_block *heap::free_allocated(heap_block *prev, heap_block *scan) -{ - if(secure_gc) - memset(scan + 1,0,scan->size() - sizeof(heap_block)); - - if(prev && prev->type() == FREE_BLOCK_TYPE) + if(scan != end) { - prev->set_size(prev->size() + scan->size()); - return prev; + assert(scan->free_p()); + assert((cell)scan + scan->size() == seg->end); + + return (cell)scan - (cell)first_block(); } else - { - scan->set_type(FREE_BLOCK_TYPE); - return scan; - } + return seg->size; } } diff --git a/vm/heap.hpp b/vm/heap.hpp index 70a4324798..7c3dca1eaf 100644 --- a/vm/heap.hpp +++ b/vm/heap.hpp @@ -34,15 +34,13 @@ struct heap { void assert_free_block(free_heap_block *block); free_heap_block *find_free_block(cell size); free_heap_block *split_free_block(free_heap_block *block, cell size); - heap_block *heap_allot(cell size, cell type); + heap_block *heap_allot(cell size); void heap_free(heap_block *block); void mark_block(heap_block *block); void heap_usage(cell *used, cell *total_free, cell *max_free); cell heap_size(); void compact_heap(); - heap_block *free_allocated(heap_block *prev, heap_block *scan); - template void sweep_heap(Iterator &iter); template void compact_heap(Iterator &iter); @@ -54,7 +52,7 @@ struct heap { while(scan != end) { heap_block *next = scan->next(); - if(scan->type() != FREE_BLOCK_TYPE) iter(scan,scan->size()); + if(!scan->free_p()) iter(scan,scan->size()); scan = next; } } @@ -72,27 +70,41 @@ template void heap::sweep_heap(Iterator &iter) while(scan != end) { - if(scan->type() == FREE_BLOCK_TYPE) + if(scan->free_p()) { - if(prev && prev->type() == FREE_BLOCK_TYPE) + if(prev && prev->free_p()) prev->set_size(prev->size() + scan->size()); else prev = scan; } else if(this->state->is_marked_p(scan)) { - if(prev && prev->type() == FREE_BLOCK_TYPE) + if(prev && prev->free_p()) this->add_to_free_list((free_heap_block *)prev); prev = scan; iter(scan,scan->size()); } else - prev = this->free_allocated(prev,scan); + { + if(secure_gc) + memset(scan + 1,0,scan->size() - sizeof(heap_block)); + + if(prev && prev->free_p()) + { + free_heap_block *free_prev = (free_heap_block *)prev; + free_prev->set_size(free_prev->size() + scan->size()); + } + else + { + scan->set_free(); + prev = scan; + } + } scan = scan->next(); } - if(prev && prev->type() == FREE_BLOCK_TYPE) + if(prev && prev->free_p()) this->add_to_free_list((free_heap_block *)prev); } diff --git a/vm/inline_cache.cpp b/vm/inline_cache.cpp index f6e756f758..772631d1ce 100755 --- a/vm/inline_cache.cpp +++ b/vm/inline_cache.cpp @@ -19,15 +19,9 @@ void factor_vm::deallocate_inline_cache(cell return_address) check_code_pointer((cell)old_xt); code_block *old_block = (code_block *)old_xt - 1; - cell old_type = old_block->type(); -#ifdef FACTOR_DEBUG - /* The call target was either another PIC, - or a compiled quotation (megamorphic stub) */ - assert(old_type == PIC_TYPE || old_type == QUOTATION_TYPE); -#endif - - if(old_type == PIC_TYPE) + /* Free the old PIC since we know its unreachable */ + if(old_block->pic_p()) code->code_heap_free(old_block); } @@ -78,7 +72,7 @@ void factor_vm::update_pic_count(cell type) struct inline_cache_jit : public jit { fixnum index; - explicit inline_cache_jit(cell generic_word_,factor_vm *vm) : jit(PIC_TYPE,generic_word_,vm) {}; + explicit inline_cache_jit(cell generic_word_,factor_vm *vm) : jit(code_block_pic,generic_word_,vm) {}; void emit_check(cell klass); void compile_inline_cache(fixnum index, diff --git a/vm/jit.cpp b/vm/jit.cpp index ced487e659..98212d2efe 100644 --- a/vm/jit.cpp +++ b/vm/jit.cpp @@ -10,7 +10,7 @@ namespace factor - polymorphic inline caches (inline_cache.cpp) */ /* Allocates memory */ -jit::jit(cell type_, cell owner_, factor_vm *vm) +jit::jit(code_block_type type_, cell owner_, factor_vm *vm) : type(type_), owner(owner_,vm), code(vm), diff --git a/vm/jit.hpp b/vm/jit.hpp index d69f44d05d..4928962fc6 100644 --- a/vm/jit.hpp +++ b/vm/jit.hpp @@ -2,7 +2,7 @@ namespace factor { struct jit { - cell type; + code_block_type type; gc_root owner; growable_byte_array code; growable_byte_array relocation; @@ -12,7 +12,7 @@ struct jit { cell offset; factor_vm *parent; - explicit jit(cell jit_type, cell owner, factor_vm *vm); + explicit jit(code_block_type type, cell owner, factor_vm *parent); void compute_position(cell offset); void emit_relocation(cell code_template); diff --git a/vm/layouts.hpp b/vm/layouts.hpp index 5b94ddfaf5..3249aac946 100644 --- a/vm/layouts.hpp +++ b/vm/layouts.hpp @@ -62,8 +62,14 @@ inline static cell align8(cell a) #define TYPE_COUNT 15 /* Not real types, but code_block's type can be set to this */ -#define PIC_TYPE 16 -#define FREE_BLOCK_TYPE 17 + +enum code_block_type +{ + code_block_unoptimized, + code_block_optimized, + code_block_profiling, + code_block_pic +}; /* Constants used when floating-point trap exceptions are thrown */ enum @@ -201,16 +207,29 @@ struct heap_block { cell header; - cell type() { return (header >> 1) & 0x1f; } - void set_type(cell type) + bool free_p() { - header = ((header & ~(0x1f << 1)) | (type << 1)); + return header & 1 == 1; + } + + void set_free() + { + header |= 1; + } + + void clear_free() + { + header &= ~1; + } + + cell size() + { + return header >> 3; } - cell size() { return (header >> 6); } void set_size(cell size) { - header = (header & 0x2f) | (size << 6); + header = (header & 0x7) | (size << 3); } inline heap_block *next() @@ -230,7 +249,30 @@ struct code_block : public heap_block cell literals; /* tagged pointer to array or f */ cell relocation; /* tagged pointer to byte-array or f */ - void *xt() { return (void *)(this + 1); } + void *xt() + { + return (void *)(this + 1); + } + + cell type() + { + return (header >> 1) & 0x3; + } + + void set_type(code_block_type type) + { + header = ((header & ~0x7) | (type << 1)); + } + + bool pic_p() + { + return type() == code_block_pic; + } + + bool optimized_p() + { + return type() == code_block_optimized; + } }; /* Assembly code makes assumptions about the layout of this struct */ diff --git a/vm/mark_bits.hpp b/vm/mark_bits.hpp index ad3eda89df..a4dc715c50 100644 --- a/vm/mark_bits.hpp +++ b/vm/mark_bits.hpp @@ -8,7 +8,6 @@ template struct mark_bits { cell size; cell bits_size; u64 *marked; - u64 *allocated; cell *forwarding; void clear_mark_bits() @@ -16,11 +15,6 @@ template struct mark_bits { memset(marked,0,bits_size * sizeof(u64)); } - void clear_allocated_bits() - { - memset(allocated,0,bits_size * sizeof(u64)); - } - void clear_forwarding() { memset(forwarding,0,bits_size * sizeof(cell)); @@ -31,11 +25,9 @@ template struct mark_bits { size(size_), bits_size(size / Granularity / forwarding_granularity), marked(new u64[bits_size]), - allocated(new u64[bits_size]), forwarding(new cell[bits_size]) { clear_mark_bits(); - clear_allocated_bits(); clear_forwarding(); } @@ -43,8 +35,6 @@ template struct mark_bits { { delete[] marked; marked = NULL; - delete[] allocated; - allocated = NULL; delete[] forwarding; forwarding = NULL; } @@ -109,16 +99,6 @@ template struct mark_bits { set_bitmap_range(marked,address); } - bool is_allocated_p(Block *address) - { - return bitmap_elt(allocated,address); - } - - void set_allocated_p(Block *address) - { - set_bitmap_range(allocated,address); - } - /* From http://chessprogramming.wikispaces.com/Population+Count */ cell popcount(u64 x) { diff --git a/vm/profiler.cpp b/vm/profiler.cpp index 4674b726b1..df9d9ee67b 100755 --- a/vm/profiler.cpp +++ b/vm/profiler.cpp @@ -13,7 +13,7 @@ code_block *factor_vm::compile_profiling_stub(cell word_) { gc_root word(word_,this); - jit jit(WORD_TYPE,word.value(),this); + jit jit(code_block_profiling,word.value(),this); jit.emit_with(userenv[JIT_PROFILING],word.value()); return jit.to_code_block(); diff --git a/vm/quotations.cpp b/vm/quotations.cpp index d75d1c680c..e06b5c23d5 100755 --- a/vm/quotations.cpp +++ b/vm/quotations.cpp @@ -335,7 +335,7 @@ void factor_vm::compile_all_words() { gc_root word(array_nth(words.untagged(),i),this); - if(!word->code || !word_optimized_p(word.untagged())) + if(!word->code || !word->code->optimized_p()) jit_compile_word(word.value(),word->def,false); update_word_xt(word.value()); diff --git a/vm/quotations.hpp b/vm/quotations.hpp index feb2af1ce4..e6e6afcd0b 100755 --- a/vm/quotations.hpp +++ b/vm/quotations.hpp @@ -6,7 +6,7 @@ struct quotation_jit : public jit { bool compiling, relocate; explicit quotation_jit(cell quot, bool compiling_, bool relocate_, factor_vm *vm) - : jit(QUOTATION_TYPE,quot,vm), + : jit(code_block_unoptimized,quot,vm), elements(owner.as().untagged()->array,vm), compiling(compiling_), relocate(relocate_){}; diff --git a/vm/tagged.hpp b/vm/tagged.hpp index a61c599aeb..02fcdee26c 100755 --- a/vm/tagged.hpp +++ b/vm/tagged.hpp @@ -27,23 +27,34 @@ struct tagged return tag; } - bool type_p(cell type_) const { return type() == type_; } + bool type_p(cell type_) const + { + return type() == type_; + } + + bool type_p() const + { + if(Type::type_number == TYPE_COUNT) + return true; + else + return type_p(Type::type_number); + } Type *untag_check(factor_vm *parent) const { - if(Type::type_number != TYPE_COUNT && !type_p(Type::type_number)) + if(!type_p()) parent->type_error(Type::type_number,value_); return untagged(); } explicit tagged(cell tagged) : value_(tagged) { #ifdef FACTOR_DEBUG - untag_check(tls_vm()); + assert(type_p()); #endif } explicit tagged(Type *untagged) : value_(factor::tag(untagged)) { #ifdef FACTOR_DEBUG - untag_check(tls_vm()); + assert(type_p()); #endif } diff --git a/vm/vm.hpp b/vm/vm.hpp index 05a918c5e9..921e829bda 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -498,8 +498,8 @@ struct factor_vm void check_code_address(cell address); void relocate_code_block(code_block *compiled); void fixup_labels(array *labels, code_block *compiled); - code_block *allot_code_block(cell size, cell type); - code_block *add_code_block(cell type, cell code_, cell labels_, cell owner_, cell relocation_, cell literals_); + code_block *allot_code_block(cell size, code_block_type type); + code_block *add_code_block(code_block_type type, cell code_, cell labels_, cell owner_, cell relocation_, cell literals_); //code heap inline void check_code_pointer(cell ptr) diff --git a/vm/words.cpp b/vm/words.cpp index 6193a5c93c..9d3ccff3c3 100644 --- a/vm/words.cpp +++ b/vm/words.cpp @@ -82,7 +82,8 @@ void factor_vm::update_word_xt(cell w_) void factor_vm::primitive_optimized_p() { - drepl(tag_boolean(word_optimized_p(untag_check(dpeek())))); + word *w = untag_check(dpeek()); + drepl(tag_boolean(w->code->optimized_p())); } void factor_vm::primitive_wrapper() diff --git a/vm/words.hpp b/vm/words.hpp index 1701def6dc..412ef35bb4 100644 --- a/vm/words.hpp +++ b/vm/words.hpp @@ -1,9 +1,4 @@ namespace factor { -inline bool word_optimized_p(word *word) -{ - return word->code->type() == WORD_TYPE; -} - }