From 1fc48b88942c9175f905401b3a96a06730f92415 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Fri, 9 Oct 2009 09:47:54 -0500 Subject: [PATCH 01/10] vm: remove old_data_heap from gc_state --- vm/full_collector.cpp | 9 +++++---- vm/gc.hpp | 1 - 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index 15a76590f1..dd142a2cb0 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -95,13 +95,15 @@ void full_collector::cheneys_algorithm() void factor_vm::collect_full(cell requested_bytes, bool trace_contexts_p) { + data_heap *old; if(current_gc->growing_data_heap) { - current_gc->old_data_heap = data; - set_data_heap(grow_data_heap(current_gc->old_data_heap,requested_bytes)); + old = data; + set_data_heap(grow_data_heap(data,requested_bytes)); } else { + old = NULL; std::swap(data->tenured,data->tenured_semispace); reset_generation(data->tenured); } @@ -121,8 +123,7 @@ void factor_vm::collect_full(cell requested_bytes, bool trace_contexts_p) reset_generation(data->aging); nursery.here = nursery.start; - if(current_gc->growing_data_heap) - delete current_gc->old_data_heap; + if(old) delete old; } } diff --git a/vm/gc.hpp b/vm/gc.hpp index d95d8df833..02f54414fd 100755 --- a/vm/gc.hpp +++ b/vm/gc.hpp @@ -24,7 +24,6 @@ struct gc_state { /* sometimes we grow the heap */ bool growing_data_heap; - data_heap *old_data_heap; /* Which generation is being collected */ cell collecting_gen; From 0bf31e42e6222b7df4b1fe25645970c85d42f140 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Fri, 9 Oct 2009 11:16:18 -0500 Subject: [PATCH 02/10] vm: refactoring card logic a bit --- vm/copying_collector.hpp | 53 ++++++++++++++++++++++++++++------------ vm/data_heap.cpp | 6 ++--- vm/data_heap.hpp | 10 +++++--- vm/debug.cpp | 14 ----------- vm/old_space.hpp | 17 ++++++++++++- vm/os-unix.cpp | 1 - 6 files changed, 63 insertions(+), 38 deletions(-) diff --git a/vm/copying_collector.hpp b/vm/copying_collector.hpp index c2a3269f3b..e7d2207fd5 100644 --- a/vm/copying_collector.hpp +++ b/vm/copying_collector.hpp @@ -16,7 +16,8 @@ struct complex_unmarker { complex_unmarker(card unmask_none_, card unmask_some_) : unmask_none(unmask_none_), unmask_some(unmask_some_) {} - void operator()(bool result, card *ptr) { + void operator()(bool result, card *ptr) + { *ptr &= (result ? ~unmask_some : ~unmask_none); } }; @@ -42,15 +43,30 @@ struct copying_collector : collector { return copied; } - template - bool trace_card(SourceGeneration *gen, card *ptr, Unmarker unmarker) + inline cell card_index(cell deck) { - cell card_start = this->myvm->card_to_addr(ptr); + return deck << (deck_bits - card_bits); + } + + inline cell card_deck_index(cell a) + { + return (a - this->data->start) >> deck_bits; + } + + inline cell card_start_address(cell card) + { + return (card << card_bits) + this->data->start; + } + + template + bool trace_card(SourceGeneration *gen, card *cards, cell card_index, Unmarker unmarker) + { + cell card_start = card_start_address(card_index); cell card_scan = card_start + gen->first_object_in_card(card_start); - cell card_end = this->myvm->card_to_addr(ptr + 1); + cell card_end = card_start_address(card_index + 1); bool result = this->trace_objects_between(gen,card_scan,&card_end); - unmarker(result,ptr); + unmarker(result,&cards[card_index]); this->myvm->gc_stats.cards_scanned++; @@ -58,15 +74,18 @@ struct copying_collector : collector { } template - bool trace_card_deck(SourceGeneration *gen, card_deck *deck, card mask, Unmarker unmarker) + bool trace_card_deck(SourceGeneration *gen, cell deck_index, card mask, Unmarker unmarker) { - card *first_card = this->myvm->deck_to_card(deck); - card *last_card = this->myvm->deck_to_card(deck + 1); + cell first_card = card_index(deck_index); + cell last_card = card_index(deck_index + 1); bool copied = false; - for(card *ptr = first_card; ptr < last_card; ptr++) - if(*ptr & mask) copied |= trace_card(gen,ptr,unmarker); + card *cards = this->data->cards; + for(cell i = first_card; i < last_card; i++) + { + if(cards[i] & mask) copied |= trace_card(gen,cards,i,unmarker); + } this->myvm->gc_stats.decks_scanned++; @@ -78,11 +97,15 @@ struct copying_collector : collector { { u64 start = current_micros(); - card_deck *first_deck = this->myvm->addr_to_deck(gen->start); - card_deck *last_deck = this->myvm->addr_to_deck(gen->end); + cell first_deck = card_deck_index(gen->start); + cell last_deck = card_deck_index(gen->end); - for(card_deck *ptr = first_deck; ptr < last_deck; ptr++) - if(*ptr & mask) unmarker(trace_card_deck(gen,ptr,mask,unmarker),ptr); + card_deck *decks = this->data->decks; + for(cell i = first_deck; i < last_deck; i++) + { + if(decks[i] & mask) + unmarker(trace_card_deck(gen,i,mask,unmarker),&decks[i]); + } this->myvm->gc_stats.card_scan_time += (current_micros() - start); } diff --git a/vm/data_heap.cpp b/vm/data_heap.cpp index 52f815c139..f028dd7a00 100755 --- a/vm/data_heap.cpp +++ b/vm/data_heap.cpp @@ -28,14 +28,14 @@ data_heap::data_heap(factor_vm *myvm, cell young_size_, cell aging_size_, cell t cell cards_size = total_size >> card_bits; - cards = new char[cards_size]; + cards = new card[cards_size]; cards_end = cards + cards_size; cell decks_size = total_size >> deck_bits; - decks = new char[decks_size]; + decks = new card_deck[decks_size]; decks_end = decks + decks_size; - cell start = align(seg->start,deck_size); + start = align(seg->start,deck_size); tenured = new tenured_space(tenured_size,start); tenured_semispace = new tenured_space(tenured_size,tenured->end); diff --git a/vm/data_heap.hpp b/vm/data_heap.hpp index 36c75e23b9..fdc2cbfbf6 100755 --- a/vm/data_heap.hpp +++ b/vm/data_heap.hpp @@ -2,6 +2,8 @@ namespace factor { struct data_heap { + cell start; + cell young_size; cell aging_size; cell tenured_size; @@ -14,11 +16,11 @@ struct data_heap { tenured_space *tenured; tenured_space *tenured_semispace; - char *cards; - char *cards_end; + card *cards; + card *cards_end; - char *decks; - char *decks_end; + card_deck *decks; + card_deck *decks_end; explicit data_heap(factor_vm *myvm, cell young_size, cell aging_size, cell tenured_size); ~data_heap(); diff --git a/vm/debug.cpp b/vm/debug.cpp index 5460a309fd..64514b9261 100755 --- a/vm/debug.cpp +++ b/vm/debug.cpp @@ -346,8 +346,6 @@ void factor_vm::factorbug() print_string(".s .r .c -- print data, retain, call stacks\n"); print_string("e -- dump environment\n"); print_string("g -- dump generations\n"); - print_string("card -- print card containing address\n"); - print_string("addr -- print address containing card\n"); print_string("data -- data heap dump\n"); print_string("words -- words dump\n"); print_string("tuples -- tuples dump\n"); @@ -423,18 +421,6 @@ void factor_vm::factorbug() } else if(strcmp(cmd,"g") == 0) dump_generations(); - else if(strcmp(cmd,"card") == 0) - { - cell addr = read_cell_hex(); - print_cell_hex((cell)addr_to_card(addr)); - nl(); - } - else if(strcmp(cmd,"addr") == 0) - { - card *ptr = (card *)read_cell_hex(); - print_cell_hex(card_to_addr(ptr)); - nl(); - } else if(strcmp(cmd,"q") == 0) return; else if(strcmp(cmd,"x") == 0) diff --git a/vm/old_space.hpp b/vm/old_space.hpp index 1f136e8c6c..6e2664c758 100644 --- a/vm/old_space.hpp +++ b/vm/old_space.hpp @@ -10,9 +10,24 @@ struct old_space : zone { old_space(cell size_, cell start_); ~old_space(); + cell object_start_map_index(cell address) + { + return (address - start) >> card_bits; + } + + /* Find the first object starting on or after the given address */ cell first_object_in_card(cell address) { - return object_start_offsets[(address - start) >> card_bits]; + return object_start_offsets[object_start_map_index(address)]; + } + + /* Find the card which contains the header of the object which contains + the given address */ + cell find_object_start_card(cell address) + { + cell i = object_start_map_index(address); + while(i >= 0 && object_start_offsets[i] == card_starts_inside_object) i--; + return i; } void record_object_start_offset(object *obj); diff --git a/vm/os-unix.cpp b/vm/os-unix.cpp index e792483e0d..b5dbbfbc03 100644 --- a/vm/os-unix.cpp +++ b/vm/os-unix.cpp @@ -211,7 +211,6 @@ void unix_init_signals() misc_sigaction.sa_sigaction = misc_signal_handler; misc_sigaction.sa_flags = SA_SIGINFO; - sigaction_safe(SIGABRT,&misc_sigaction,NULL); sigaction_safe(SIGQUIT,&misc_sigaction,NULL); sigaction_safe(SIGILL,&misc_sigaction,NULL); From d17c63efa5b051c759b315840ef061ea8316bd90 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Fri, 9 Oct 2009 23:37:35 -0500 Subject: [PATCH 03/10] vm: rename --- vm/old_space.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/old_space.hpp b/vm/old_space.hpp index 6e2664c758..8d77e8499a 100644 --- a/vm/old_space.hpp +++ b/vm/old_space.hpp @@ -23,7 +23,7 @@ struct old_space : zone { /* Find the card which contains the header of the object which contains the given address */ - cell find_object_start_card(cell address) + cell find_card_containing_header(cell address) { cell i = object_start_map_index(address); while(i >= 0 && object_start_offsets[i] == card_starts_inside_object) i--; From d689be57a515e24ecb2134ee5e2f7b79e74c74a9 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Tue, 13 Oct 2009 21:16:04 -0500 Subject: [PATCH 04/10] vm: new card marking implementation supports marking partial objects --- vm/arrays.cpp | 3 +- vm/arrays.hpp | 5 +- vm/copying_collector.hpp | 155 +++++++++++++++++++++++++++------------ vm/data_heap.cpp | 37 ++++------ vm/data_heap.hpp | 3 +- vm/full_collector.cpp | 2 +- vm/gc.cpp | 4 +- vm/generic_arrays.hpp | 6 +- vm/old_space.cpp | 39 ++++++++-- vm/old_space.hpp | 22 +----- vm/run.cpp | 5 +- vm/strings.cpp | 4 +- vm/vm.hpp | 33 +-------- vm/write_barrier.hpp | 9 +++ 14 files changed, 186 insertions(+), 141 deletions(-) diff --git a/vm/arrays.cpp b/vm/arrays.cpp index 95a435eecd..af8e0c82d5 100644 --- a/vm/arrays.cpp +++ b/vm/arrays.cpp @@ -16,8 +16,7 @@ array *factor_vm::allot_array(cell capacity, cell fill_) /* No need for write barrier here. Either the object is in the nursery, or it was allocated directly in tenured space and the write barrier is already hit for us in that case. */ - cell i; - for(i = 0; i < capacity; i++) + for(cell i = 0; i < capacity; i++) new_array->data()[i] = fill.value(); } return new_array.untagged(); diff --git a/vm/arrays.hpp b/vm/arrays.hpp index c3815c9f60..79f31573f4 100755 --- a/vm/arrays.hpp +++ b/vm/arrays.hpp @@ -17,8 +17,9 @@ inline void factor_vm::set_array_nth(array *array, cell slot, cell value) assert(array->h.hi_tag() == ARRAY_TYPE); check_tagged_pointer(value); #endif - array->data()[slot] = value; - write_barrier(array); + cell *slot_ptr = &array->data()[slot]; + *slot_ptr = value; + write_barrier(slot_ptr); } struct growable_array { diff --git a/vm/copying_collector.hpp b/vm/copying_collector.hpp index e7d2207fd5..0d11ba04bc 100644 --- a/vm/copying_collector.hpp +++ b/vm/copying_collector.hpp @@ -29,28 +29,24 @@ struct copying_collector : collector { explicit copying_collector(factor_vm *myvm_, TargetGeneration *target_, Policy policy_) : collector(myvm_,target_,policy_), scan(target_->here) {} - template - bool trace_objects_between(SourceGeneration *gen, cell scan, cell *end) - { - bool copied = false; - - while(scan && scan < *end) - { - copied |= this->trace_slots((object *)scan); - scan = gen->next_object_after(this->myvm,scan); - } - - return copied; - } - - inline cell card_index(cell deck) + inline cell first_card_in_deck(cell deck) { return deck << (deck_bits - card_bits); } - inline cell card_deck_index(cell a) + inline cell last_card_in_deck(cell deck) { - return (a - this->data->start) >> deck_bits; + return first_card_in_deck(deck + 1); + } + + inline cell card_to_addr(cell c) + { + return c << card_bits + this->data->start; + } + + inline cell card_deck_for_address(cell a) + { + return addr_to_deck(a - this->data->start); } inline cell card_start_address(cell card) @@ -58,36 +54,31 @@ struct copying_collector : collector { return (card << card_bits) + this->data->start; } - template - bool trace_card(SourceGeneration *gen, card *cards, cell card_index, Unmarker unmarker) + inline cell card_end_address(cell card) { - cell card_start = card_start_address(card_index); - cell card_scan = card_start + gen->first_object_in_card(card_start); - cell card_end = card_start_address(card_index + 1); - - bool result = this->trace_objects_between(gen,card_scan,&card_end); - unmarker(result,&cards[card_index]); - - this->myvm->gc_stats.cards_scanned++; - - return result; + return ((card + 1) << card_bits) + this->data->start; } - template - bool trace_card_deck(SourceGeneration *gen, cell deck_index, card mask, Unmarker unmarker) + bool trace_partial_objects(cell start, cell end, cell card_start, cell card_end) { - cell first_card = card_index(deck_index); - cell last_card = card_index(deck_index + 1); - bool copied = false; - card *cards = this->data->cards; - for(cell i = first_card; i < last_card; i++) + if(card_start < end) { - if(cards[i] & mask) copied |= trace_card(gen,cards,i,unmarker); - } + start += sizeof(cell); - this->myvm->gc_stats.decks_scanned++; + if(start < card_start) start = card_start; + if(end > card_end) end = card_end; + + cell *slot_ptr = (cell *)start; + cell *end_ptr = (cell *)end; + + if(slot_ptr != end_ptr) + { + for(; slot_ptr < end_ptr; slot_ptr++) + copied |= this->trace_handle(slot_ptr); + } + } return copied; } @@ -95,19 +86,73 @@ struct copying_collector : collector { template void trace_cards(SourceGeneration *gen, card mask, Unmarker unmarker) { - u64 start = current_micros(); - - cell first_deck = card_deck_index(gen->start); - cell last_deck = card_deck_index(gen->end); - + u64 start_time = current_micros(); + card_deck *decks = this->data->decks; - for(cell i = first_deck; i < last_deck; i++) + card_deck *cards = this->data->cards; + + cell gen_start_card = addr_to_card(gen->start - this->data->start); + + cell first_deck = card_deck_for_address(gen->start); + cell last_deck = card_deck_for_address(gen->end); + + cell start = 0, binary_start = 0, end = 0; + + for(cell deck_index = first_deck; deck_index < last_deck; deck_index++) { - if(decks[i] & mask) - unmarker(trace_card_deck(gen,i,mask,unmarker),&decks[i]); + if(decks[deck_index] & mask) + { + cell first_card = first_card_in_deck(deck_index); + cell last_card = last_card_in_deck(deck_index); + + bool deck_dirty = false; + + for(cell card_index = first_card; card_index < last_card; card_index++) + { + if(cards[card_index] & mask) + { + if(card_start_address(card_index) >= end) + { + start = gen->find_object_containing_card(card_index - gen_start_card); + binary_start = start + this->myvm->binary_payload_start((object *)start); + end = start + this->myvm->untagged_object_size((object *)start); + } + + bool card_dirty = false; + +#ifdef FACTOR_DEBUG + assert(addr_to_card(start - this->data->start) <= card_index); + assert(start < card_end_address(card_index)); +#endif + +scan_next_object: { + card_dirty |= trace_partial_objects( + start, + binary_start, + card_start_address(card_index), + card_end_address(card_index)); + if(end < card_end_address(card_index)) + { + start = gen->next_object_after(this->myvm,start); + if(!start) goto end; + binary_start = start + this->myvm->binary_payload_start((object *)start); + end = start + this->myvm->untagged_object_size((object *)start); + goto scan_next_object; + } + } + + unmarker(card_dirty,&cards[card_index]); + this->myvm->gc_stats.cards_scanned++; + + deck_dirty |= card_dirty; + } + } + + unmarker(deck_dirty,&decks[deck_index]); + } } - this->myvm->gc_stats.card_scan_time += (current_micros() - start); +end: this->myvm->gc_stats.card_scan_time += (current_micros() - start_time); } /* Trace all literals referenced from a code block. Only for aging and nursery collections */ @@ -127,6 +172,20 @@ struct copying_collector : collector { for(; iter != end; iter++) trace_literal_references(*iter); } + template + bool trace_objects_between(SourceGeneration *gen, cell scan, cell *end) + { + bool copied = false; + + while(scan && scan < *end) + { + copied |= this->trace_slots((object *)scan); + scan = gen->next_object_after(this->myvm,scan); + } + + return copied; + } + void cheneys_algorithm() { trace_objects_between(this->target,scan,&this->target->here); diff --git a/vm/data_heap.cpp b/vm/data_heap.cpp index f028dd7a00..d93c121db0 100755 --- a/vm/data_heap.cpp +++ b/vm/data_heap.cpp @@ -5,12 +5,11 @@ namespace factor void factor_vm::init_card_decks() { - cell start = align(data->seg->start,deck_size); - cards_offset = (cell)data->cards - (start >> card_bits); - decks_offset = (cell)data->decks - (start >> deck_bits); + cards_offset = (cell)data->cards - addr_to_card(data->start); + decks_offset = (cell)data->decks - addr_to_deck(data->start); } -data_heap::data_heap(factor_vm *myvm, cell young_size_, cell aging_size_, cell tenured_size_) +data_heap::data_heap(cell young_size_, cell aging_size_, cell tenured_size_) { young_size_ = align(young_size_,deck_size); aging_size_ = align(aging_size_,deck_size); @@ -26,12 +25,12 @@ data_heap::data_heap(factor_vm *myvm, cell young_size_, cell aging_size_, cell t seg = new segment(total_size); - cell cards_size = total_size >> card_bits; + cell cards_size = addr_to_card(total_size); cards = new card[cards_size]; cards_end = cards + cards_size; - cell decks_size = total_size >> deck_bits; + cell decks_size = addr_to_deck(total_size); decks = new card_deck[decks_size]; decks_end = decks + decks_size; @@ -60,30 +59,24 @@ data_heap::~data_heap() delete[] decks; } -data_heap *factor_vm::grow_data_heap(data_heap *data, cell requested_bytes) +data_heap *data_heap::grow(cell requested_bytes) { - cell new_tenured_size = (data->tenured_size * 2) + requested_bytes; - - return new data_heap(this, - data->young_size, - data->aging_size, - new_tenured_size); + cell new_tenured_size = (tenured_size * 2) + requested_bytes; + return new data_heap(young_size,aging_size,new_tenured_size); } void factor_vm::clear_cards(old_space *gen) { - /* NOTE: reverse order due to heap layout. */ - card *first_card = addr_to_card(gen->start); - card *last_card = addr_to_card(gen->end); - memset(first_card,0,last_card - first_card); + cell first_card = addr_to_card(gen->start - data->start); + cell last_card = addr_to_card(gen->end - data->start); + memset(&data->cards[first_card],0,last_card - first_card); } void factor_vm::clear_decks(old_space *gen) { - /* NOTE: reverse order due to heap layout. */ - card_deck *first_deck = addr_to_deck(gen->start); - card_deck *last_deck = addr_to_deck(gen->end); - memset(first_deck,0,last_deck - first_deck); + cell first_deck = addr_to_deck(gen->start - data->start); + cell last_deck = addr_to_deck(gen->end - data->start); + memset(&data->decks[first_deck],0,last_deck - first_deck); } /* After garbage collection, any generations which are now empty need to have @@ -110,7 +103,7 @@ void factor_vm::set_data_heap(data_heap *data_) void factor_vm::init_data_heap(cell young_size, cell aging_size, cell tenured_size, bool secure_gc_) { - set_data_heap(new data_heap(this,young_size,aging_size,tenured_size)); + set_data_heap(new data_heap(young_size,aging_size,tenured_size)); secure_gc = secure_gc_; } diff --git a/vm/data_heap.hpp b/vm/data_heap.hpp index fdc2cbfbf6..2370325cad 100755 --- a/vm/data_heap.hpp +++ b/vm/data_heap.hpp @@ -22,8 +22,9 @@ struct data_heap { card_deck *decks; card_deck *decks_end; - explicit data_heap(factor_vm *myvm, cell young_size, cell aging_size, cell tenured_size); + explicit data_heap(cell young_size, cell aging_size, cell tenured_size); ~data_heap(); + data_heap *grow(cell requested_size); }; static const cell nursery_gen = 0; diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index dd142a2cb0..c79b8d4d31 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -99,7 +99,7 @@ void factor_vm::collect_full(cell requested_bytes, bool trace_contexts_p) if(current_gc->growing_data_heap) { old = data; - set_data_heap(grow_data_heap(data,requested_bytes)); + set_data_heap(data->grow(requested_bytes)); } else { diff --git a/vm/gc.cpp b/vm/gc.cpp index 3850dc642e..3b455b2a3f 100755 --- a/vm/gc.cpp +++ b/vm/gc.cpp @@ -249,7 +249,9 @@ object *factor_vm::allot_object(header header, cell size) /* Allows initialization code to store old->new pointers without hitting the write barrier in the common case of a nursery allocation */ - write_barrier(obj); + char *start = (char *)obj; + for(cell offset = 0; offset < size; offset += card_size) + write_barrier((cell *)(start + offset)); } obj->h = header; diff --git a/vm/generic_arrays.hpp b/vm/generic_arrays.hpp index 07e876171b..0ba6d11da2 100755 --- a/vm/generic_arrays.hpp +++ b/vm/generic_arrays.hpp @@ -45,13 +45,13 @@ template Array *factor_vm::reallot_array(Array *array_, cell cap cell to_copy = array_capacity(array.untagged()); if(capacity < to_copy) to_copy = capacity; - + Array *new_array = allot_array_internal(capacity); - + memcpy(new_array + 1,array.untagged() + 1,to_copy * Array::element_size); memset((char *)(new_array + 1) + to_copy * Array::element_size, 0,(capacity - to_copy) * Array::element_size); - + return new_array; } } diff --git a/vm/old_space.cpp b/vm/old_space.cpp index 49517dc9a6..6bd8d6db0a 100644 --- a/vm/old_space.cpp +++ b/vm/old_space.cpp @@ -5,9 +5,8 @@ namespace factor old_space::old_space(cell size_, cell start_) : zone(size_,start_) { - cell cards_size = size_ >> card_bits; - object_start_offsets = new card[cards_size]; - object_start_offsets_end = object_start_offsets + cards_size; + object_start_offsets = new card[addr_to_card(size_)]; + object_start_offsets_end = object_start_offsets + addr_to_card(size_); } old_space::~old_space() @@ -15,12 +14,38 @@ old_space::~old_space() delete[] object_start_offsets; } +cell old_space::first_object_in_card(cell card_index) +{ + return object_start_offsets[card_index]; +} + +cell old_space::find_object_containing_card(cell card_index) +{ + if(card_index == 0) + return start; + else + { + card_index--; + + while(first_object_in_card(card_index) == card_starts_inside_object) + { +#ifdef FACTOR_DEBUG + /* First card should start with an object */ + assert(card_index > 0); +#endif + card_index--; + } + + return start + (card_index << card_bits) + first_object_in_card(card_index); + } +} + /* we need to remember the first object allocated in the card */ void old_space::record_object_start_offset(object *obj) { - card *ptr = (card *)((((cell)obj - start) >> card_bits) + (cell)object_start_offsets); - if(*ptr == card_starts_inside_object) - *ptr = ((cell)obj & addr_card_mask); + cell idx = addr_to_card((cell)obj - start); + if(object_start_offsets[idx] == card_starts_inside_object) + object_start_offsets[idx] = ((cell)obj & addr_card_mask); } object *old_space::allot(cell size) @@ -34,7 +59,7 @@ object *old_space::allot(cell size) void old_space::clear_object_start_offsets() { - memset(object_start_offsets,card_starts_inside_object,size >> card_bits); + memset(object_start_offsets,card_starts_inside_object,addr_to_card(size)); } cell old_space::next_object_after(factor_vm *myvm, cell scan) diff --git a/vm/old_space.hpp b/vm/old_space.hpp index 8d77e8499a..410c87f6b1 100644 --- a/vm/old_space.hpp +++ b/vm/old_space.hpp @@ -10,26 +10,8 @@ struct old_space : zone { old_space(cell size_, cell start_); ~old_space(); - cell object_start_map_index(cell address) - { - return (address - start) >> card_bits; - } - - /* Find the first object starting on or after the given address */ - cell first_object_in_card(cell address) - { - return object_start_offsets[object_start_map_index(address)]; - } - - /* Find the card which contains the header of the object which contains - the given address */ - cell find_card_containing_header(cell address) - { - cell i = object_start_map_index(address); - while(i >= 0 && object_start_offsets[i] == card_starts_inside_object) i--; - return i; - } - + cell old_space::first_object_in_card(cell card_index); + cell find_object_containing_card(cell card_index); void record_object_start_offset(object *obj); object *allot(cell size); void clear_object_start_offsets(); diff --git a/vm/run.cpp b/vm/run.cpp index 1a24d1d910..79aca937ca 100755 --- a/vm/run.cpp +++ b/vm/run.cpp @@ -37,8 +37,9 @@ void factor_vm::primitive_set_slot() object *obj = untag(dpop()); cell value = dpop(); - obj->slots()[slot] = value; - write_barrier(obj); + cell *slot_ptr = &obj->slots()[slot]; + *slot_ptr = value; + write_barrier(slot_ptr); } void factor_vm::primitive_load_locals() diff --git a/vm/strings.cpp b/vm/strings.cpp index fa7a775760..ecfc84ebef 100644 --- a/vm/strings.cpp +++ b/vm/strings.cpp @@ -45,8 +45,8 @@ void factor_vm::set_string_nth_slow(string *str_, cell index, cell ch) the bits are clear. */ aux = allot_array_internal(untag_fixnum(str->length) * sizeof(u16)); - write_barrier(str.untagged()); str->aux = tag(aux); + write_barrier(&str->aux); } else aux = untag(str->aux); @@ -143,8 +143,8 @@ string* factor_vm::reallot_string(string *str_, cell capacity) { byte_array *new_aux = allot_byte_array(capacity * sizeof(u16)); - write_barrier(new_str.untagged()); new_str->aux = tag(new_aux); + write_barrier(&new_str->aux); byte_array *aux = untag(str->aux); memcpy(new_aux->data(),aux->data(),to_copy * sizeof(u16)); diff --git a/vm/vm.hpp b/vm/vm.hpp index c89268f9b6..135acd58ab 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -203,7 +203,6 @@ struct factor_vm //data heap void init_card_decks(); - data_heap *grow_data_heap(data_heap *data, cell requested_bytes); void clear_cards(old_space *gen); void clear_decks(old_space *gen); void reset_generation(old_space *gen); @@ -224,38 +223,12 @@ struct factor_vm cell find_all_words(); cell object_size(cell tagged); - //write barrier - inline card *addr_to_card(cell a) - { - return (card*)(((cell)(a) >> card_bits) + cards_offset); - } - - inline cell card_to_addr(card *c) - { - return ((cell)c - cards_offset) << card_bits; - } - - inline card_deck *addr_to_deck(cell a) - { - return (card_deck *)(((cell)a >> deck_bits) + decks_offset); - } - - inline cell deck_to_addr(card_deck *c) - { - return ((cell)c - decks_offset) << deck_bits; - } - - inline card *deck_to_card(card_deck *d) - { - return (card *)((((cell)d - decks_offset) << (deck_bits - card_bits)) + cards_offset); - } - /* the write barrier must be called any time we are potentially storing a pointer from an older generation to a younger one */ - inline void write_barrier(object *obj) + inline void write_barrier(cell *slot_ptr) { - *addr_to_card((cell)obj) = card_mark_mask; - *addr_to_deck((cell)obj) = card_mark_mask; + data->cards[addr_to_card((cell)slot_ptr - data->start)] = card_mark_mask; + data->decks[addr_to_deck((cell)slot_ptr - data->start)] = card_mark_mask; } // gc diff --git a/vm/write_barrier.hpp b/vm/write_barrier.hpp index bcfc0531d3..d434bf2d98 100755 --- a/vm/write_barrier.hpp +++ b/vm/write_barrier.hpp @@ -25,4 +25,13 @@ static const cell deck_bits = (card_bits + 10); static const cell deck_size = (1<> card_bits; +} + +inline cell addr_to_deck(cell a) +{ + return a >> deck_bits; +} } From 1ce39963fdeaedbae1cb806f879b6c180ec411c6 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Wed, 14 Oct 2009 02:06:01 -0500 Subject: [PATCH 05/10] Working on adding support for the new write barrier to optimized code --- basis/compiler/cfg/dce/dce.factor | 8 + .../cfg/instructions/instructions.factor | 7 +- .../cfg/intrinsics/slots/slots.factor | 26 ++-- .../cfg/write-barrier/write-barrier.factor | 146 +++--------------- basis/compiler/codegen/codegen.factor | 1 + basis/compiler/codegen/fixup/fixup.factor | 9 ++ basis/compiler/constants/constants.factor | 2 + basis/cpu/architecture/architecture.factor | 3 +- basis/cpu/x86/32/32.factor | 6 +- basis/cpu/x86/64/64.factor | 7 +- basis/cpu/x86/x86.factor | 52 ++++--- vm/code_block.cpp | 6 + vm/code_block.hpp | 4 + vm/full_collector.cpp | 64 ++++++-- vm/gc.cpp | 31 ++-- vm/vm.hpp | 6 +- vm/zone.hpp | 9 +- 17 files changed, 179 insertions(+), 208 deletions(-) diff --git a/basis/compiler/cfg/dce/dce.factor b/basis/compiler/cfg/dce/dce.factor index 363cea7852..b8735e224c 100644 --- a/basis/compiler/cfg/dce/dce.factor +++ b/basis/compiler/cfg/dce/dce.factor @@ -42,6 +42,9 @@ M: ##set-slot-imm build-liveness-graph M: ##write-barrier build-liveness-graph dup src>> setter-liveness-graph ; +M: ##write-barrier-imm build-liveness-graph + dup src>> setter-liveness-graph ; + M: ##allot build-liveness-graph [ dst>> allocations get conjoin ] [ call-next-method ] bi ; @@ -74,6 +77,9 @@ M: ##set-slot-imm compute-live-vregs M: ##write-barrier compute-live-vregs dup src>> setter-live-vregs ; +M: ##write-barrier-imm compute-live-vregs + dup src>> setter-live-vregs ; + M: ##fixnum-add compute-live-vregs record-live ; M: ##fixnum-sub compute-live-vregs record-live ; @@ -91,6 +97,8 @@ M: ##set-slot-imm live-insn? obj>> live-vreg? ; M: ##write-barrier live-insn? src>> live-vreg? ; +M: ##write-barrier-imm live-insn? src>> live-vreg? ; + M: ##fixnum-add live-insn? drop t ; M: ##fixnum-sub live-insn? drop t ; diff --git a/basis/compiler/cfg/instructions/instructions.factor b/basis/compiler/cfg/instructions/instructions.factor index 119af6d0b1..bffa0e59d0 100644 --- a/basis/compiler/cfg/instructions/instructions.factor +++ b/basis/compiler/cfg/instructions/instructions.factor @@ -619,8 +619,13 @@ literal: size class temp: temp/int-rep ; INSN: ##write-barrier +use: src/int-rep slot/int-rep +temp: temp1/int-rep temp2/int-rep ; + +INSN: ##write-barrier-imm use: src/int-rep -temp: card#/int-rep table/int-rep ; +literal: slot +temp: temp1/int-rep temp2/int-rep ; INSN: ##alien-global def: dst/int-rep diff --git a/basis/compiler/cfg/intrinsics/slots/slots.factor b/basis/compiler/cfg/intrinsics/slots/slots.factor index 8ee1c41cfb..a28c95f81f 100644 --- a/basis/compiler/cfg/intrinsics/slots/slots.factor +++ b/basis/compiler/cfg/intrinsics/slots/slots.factor @@ -1,7 +1,7 @@ ! Copyright (C) 2008, 2009 Slava Pestov. ! See http://factorcode.org/license.txt for BSD license. USING: layouts namespaces kernel accessors sequences classes.algebra -compiler.tree.propagation.info compiler.cfg.stacks compiler.cfg.hats +fry compiler.tree.propagation.info compiler.cfg.stacks compiler.cfg.hats compiler.cfg.registers compiler.cfg.instructions compiler.cfg.utilities compiler.cfg.builder.blocks ; IN: compiler.cfg.intrinsics.slots @@ -30,25 +30,25 @@ IN: compiler.cfg.intrinsics.slots ds-push ] [ drop emit-primitive ] if ; -: (emit-set-slot) ( infos -- obj-reg ) - [ 3inputs ] [ second value-tag ] bi* - ^^tag-offset>slot over [ ##set-slot ] dip ; +: (emit-set-slot) ( infos -- ) + [ first class>> immediate class<= ] + [ [ 3inputs ] [ second value-tag ] bi* ^^tag-offset>slot ] bi + [ ##set-slot ] + [ '[ _ drop _ _ next-vreg next-vreg ##write-barrier ] unless ] 3bi ; -: (emit-set-slot-imm) ( infos -- obj-reg ) +: (emit-set-slot-imm) ( infos -- ) ds-drop - [ 2inputs ] - [ [ third literal>> ] [ second value-tag ] bi ] bi* - pick [ ##set-slot-imm ] dip ; + [ first class>> immediate class<= ] + [ [ 2inputs ] [ [ third literal>> ] [ second value-tag ] bi ] bi* ] bi + '[ _ ##set-slot-imm ] + [ '[ _ drop _ _ cells next-vreg next-vreg ##write-barrier-imm ] unless ] 3bi ; : emit-set-slot ( node -- ) dup node-input-infos dup second value-tag [ nip - [ - dup third value-info-small-fixnum? - [ (emit-set-slot-imm) ] [ (emit-set-slot) ] if - ] [ first class>> immediate class<= ] bi - [ drop ] [ next-vreg next-vreg ##write-barrier ] if + dup third value-info-small-fixnum? + [ (emit-set-slot-imm) ] [ (emit-set-slot) ] if ] [ drop emit-primitive ] if ; : emit-string-nth ( -- ) diff --git a/basis/compiler/cfg/write-barrier/write-barrier.factor b/basis/compiler/cfg/write-barrier/write-barrier.factor index 778d0526d5..0217055923 100644 --- a/basis/compiler/cfg/write-barrier/write-barrier.factor +++ b/basis/compiler/cfg/write-barrier/write-barrier.factor @@ -1,139 +1,43 @@ ! Copyright (C) 2008, 2009 Slava Pestov, Daniel Ehrenberg. ! See http://factorcode.org/license.txt for BSD license. -USING: kernel accessors namespaces assocs sets sequences -fry combinators.short-circuit locals make arrays -compiler.cfg -compiler.cfg.dominance -compiler.cfg.predecessors -compiler.cfg.loop-detection -compiler.cfg.rpo -compiler.cfg.instructions -compiler.cfg.registers -compiler.cfg.dataflow-analysis -compiler.cfg.utilities ; +USING: accessors assocs combinators.short-circuit +compiler.cfg.instructions compiler.cfg.rpo kernel namespaces +sequences sets ; IN: compiler.cfg.write-barrier -! Eliminate redundant write barrier hits. +SYMBOL: fresh-allocations -! Objects which have already been marked, as well as -! freshly-allocated objects -SYMBOL: safe - -! Objects which have been mutated -SYMBOL: mutated +SYMBOL: mutated-objects GENERIC: eliminate-write-barrier ( insn -- ? ) M: ##allot eliminate-write-barrier - dst>> safe get conjoin t ; + dst>> fresh-allocations get conjoin t ; + +M: ##set-slot eliminate-write-barrier + obj>> mutated-objects get conjoin t ; + +M: ##set-slot-imm eliminate-write-barrier + obj>> mutated-objects get conjoin t ; + +: needs-write-barrier? ( insn -- ? ) + { [ fresh-allocations get key? not ] [ mutated-objects get key? ] } 1&& ; M: ##write-barrier eliminate-write-barrier - src>> dup safe get key? not - [ safe get conjoin t ] [ drop f ] if ; + src>> needs-write-barrier? ; + +M: ##write-barrier-imm eliminate-write-barrier + src>> needs-write-barrier? ; + +M: ##copy eliminate-write-barrier + "Run copy propagation first" throw ; M: insn eliminate-write-barrier drop t ; -! This doesn't actually benefit from being a dataflow analysis -! might as well be dominator-based -! Dealing with phi functions would help, though -FORWARD-ANALYSIS: safe - -: has-allocation? ( bb -- ? ) - instructions>> [ { [ ##allocation? ] [ ##call? ] } 1|| ] any? ; - -M: safe-analysis transfer-set - drop [ H{ } assoc-clone-like safe set ] dip - instructions>> [ - eliminate-write-barrier drop - ] each safe get ; - -M: safe-analysis join-sets - drop has-allocation? [ drop H{ } clone ] [ assoc-refine ] if ; - : write-barriers-step ( bb -- ) - dup safe-in H{ } assoc-clone-like safe set + H{ } clone fresh-allocations set + H{ } clone mutated-objects set instructions>> [ eliminate-write-barrier ] filter-here ; -GENERIC: remove-dead-barrier ( insn -- ? ) - -M: ##write-barrier remove-dead-barrier - src>> mutated get key? ; - -M: ##set-slot remove-dead-barrier - obj>> mutated get conjoin t ; - -M: ##set-slot-imm remove-dead-barrier - obj>> mutated get conjoin t ; - -M: insn remove-dead-barrier drop t ; - -: remove-dead-barriers ( bb -- ) - H{ } clone mutated set - instructions>> [ remove-dead-barrier ] filter-here ; - -! Availability of slot -! Anticipation of this and set-slot would help too, maybe later -FORWARD-ANALYSIS: slot - -UNION: access ##slot ##slot-imm ##set-slot ##set-slot-imm ; - -M: slot-analysis transfer-set - drop [ H{ } assoc-clone-like ] dip - instructions>> over '[ - dup access? [ - obj>> _ conjoin - ] [ drop ] if - ] each ; - -: slot-available? ( vreg bb -- ? ) - slot-in key? ; - -: make-barriers ( vregs -- bb ) - [ [ next-vreg next-vreg ##write-barrier ] each ] V{ } make ; - -: emit-barriers ( vregs loop -- ) - swap [ - [ [ header>> predecessors>> ] [ ends>> keys ] bi diff ] - [ header>> ] bi - ] [ make-barriers ] bi* - insert-basic-block ; - -: write-barriers ( bbs -- bb=>barriers ) - [ - dup instructions>> - [ ##write-barrier? ] filter - [ src>> ] map - ] { } map>assoc - [ nip empty? not ] assoc-filter ; - -: filter-dominant ( bb=>barriers bbs -- barriers ) - '[ drop _ [ dominates? ] with all? ] assoc-filter - values concat prune ; - -: dominant-write-barriers ( loop -- vregs ) - [ blocks>> values write-barriers ] [ ends>> keys ] bi filter-dominant ; - -: safe-loops ( -- loops ) - loops get values - [ blocks>> keys [ has-allocation? not ] all? ] filter ; - -:: insert-extra-barriers ( cfg -- ) - safe-loops [| loop | - cfg needs-dominance needs-predecessors drop - loop dominant-write-barriers - loop header>> '[ _ slot-available? ] filter - [ loop emit-barriers cfg cfg-changed drop ] unless-empty - ] each ; - -: contains-write-barrier? ( cfg -- ? ) - post-order [ instructions>> [ ##write-barrier? ] any? ] any? ; - : eliminate-write-barriers ( cfg -- cfg' ) - dup contains-write-barrier? [ - needs-loops - dup [ remove-dead-barriers ] each-basic-block - dup compute-slot-sets - dup insert-extra-barriers - dup compute-safe-sets - dup [ write-barriers-step ] each-basic-block - ] when ; + dup [ write-barriers-step ] each-basic-block ; diff --git a/basis/compiler/codegen/codegen.factor b/basis/compiler/codegen/codegen.factor index 938219af22..467e8d9b8d 100755 --- a/basis/compiler/codegen/codegen.factor +++ b/basis/compiler/codegen/codegen.factor @@ -218,6 +218,7 @@ CODEGEN: ##set-alien-double %set-alien-double CODEGEN: ##set-alien-vector %set-alien-vector CODEGEN: ##allot %allot CODEGEN: ##write-barrier %write-barrier +CODEGEN: ##write-barrier-imm %write-barrier-imm CODEGEN: ##compare %compare CODEGEN: ##compare-imm %compare-imm CODEGEN: ##compare-float-ordered %compare-float-ordered diff --git a/basis/compiler/codegen/fixup/fixup.factor b/basis/compiler/codegen/fixup/fixup.factor index 60dd055a5f..f56d654c77 100755 --- a/basis/compiler/codegen/fixup/fixup.factor +++ b/basis/compiler/codegen/fixup/fixup.factor @@ -77,6 +77,15 @@ SYMBOL: relocation-table : rel-here ( offset class -- ) [ add-literal ] dip rt-here rel-fixup ; +: rel-vm ( class -- ) + rt-vm rel-fixup ; + +: rel-cards-offset ( class -- ) + rt-cards-offset rel-fixup ; + +: rel-decks-offset ( class -- ) + rt-decks-offset rel-fixup ; + ! And the rest : resolve-offset ( label-fixup -- offset ) label>> offset>> [ "Unresolved label" throw ] unless* ; diff --git a/basis/compiler/constants/constants.factor b/basis/compiler/constants/constants.factor index cc6003b89c..f6c6573be1 100644 --- a/basis/compiler/constants/constants.factor +++ b/basis/compiler/constants/constants.factor @@ -51,6 +51,8 @@ CONSTANT: rt-stack-chain 9 CONSTANT: rt-untagged 10 CONSTANT: rt-megamorphic-cache-hits 11 CONSTANT: rt-vm 12 +CONSTANT: rt-cards-offset 13 +CONSTANT: rt-decks-offset 14 : rc-absolute? ( n -- ? ) ${ rc-absolute-ppc-2/2 rc-absolute-cell rc-absolute } member? ; diff --git a/basis/cpu/architecture/architecture.factor b/basis/cpu/architecture/architecture.factor index 19b38fd8f8..d5b84b7002 100644 --- a/basis/cpu/architecture/architecture.factor +++ b/basis/cpu/architecture/architecture.factor @@ -397,7 +397,8 @@ HOOK: %alien-global cpu ( dst symbol library -- ) HOOK: %vm-field-ptr cpu ( dst fieldname -- ) HOOK: %allot cpu ( dst size class temp -- ) -HOOK: %write-barrier cpu ( src card# table -- ) +HOOK: %write-barrier cpu ( src slot temp1 temp2 -- ) +HOOK: %write-barrier-imm cpu ( src slot temp1 temp2 -- ) ! GC checks HOOK: %check-nursery cpu ( label size temp1 temp2 -- ) diff --git a/basis/cpu/x86/32/32.factor b/basis/cpu/x86/32/32.factor index 414249f88e..55e9448c7c 100755 --- a/basis/cpu/x86/32/32.factor +++ b/basis/cpu/x86/32/32.factor @@ -50,7 +50,7 @@ M: x86.32 reserved-area-size 0 ; M: x86.32 %alien-invoke 0 CALL rc-relative rel-dlsym ; : push-vm-ptr ( -- ) - 0 PUSH rc-absolute-cell rt-vm rel-fixup ; ! push the vm ptr as an argument + 0 PUSH rc-absolute-cell rel-vm ; ! push the vm ptr as an argument M: x86.32 return-struct-in-registers? ( c-type -- ? ) c-type @@ -263,7 +263,7 @@ M: x86.32 %alien-callback ( quot -- ) 4 [ EAX swap %load-reference EAX PUSH - param-reg-2 0 MOV rc-absolute-cell rt-vm rel-fixup + param-reg-2 %mov-vm-ptr "c_to_factor" f %alien-invoke ] with-aligned-stack ; @@ -348,7 +348,7 @@ M:: x86.32 %call-gc ( gc-root-count temp -- ) temp gc-root-base param@ LEA 12 [ ! Pass the VM ptr as the third parameter - 0 PUSH rc-absolute-cell rt-vm rel-fixup + push-vm-ptr ! Pass number of roots as second parameter gc-root-count PUSH ! Pass pointer to start of GC roots as first parameter diff --git a/basis/cpu/x86/64/64.factor b/basis/cpu/x86/64/64.factor index 805dda982b..c15169dd89 100644 --- a/basis/cpu/x86/64/64.factor +++ b/basis/cpu/x86/64/64.factor @@ -75,9 +75,6 @@ M: x86.64 %prepare-unbox ( -- ) param-reg-1 R14 [] MOV R14 cell SUB ; -: %mov-vm-ptr ( reg -- ) - 0 MOV rc-absolute-cell rt-vm rel-fixup ; - M:: x86.64 %unbox ( n rep func -- ) param-reg-2 %mov-vm-ptr ! Call the unboxer @@ -183,11 +180,11 @@ M: x86.64 %alien-invoke R11 CALL ; M: x86.64 %nest-stacks ( -- ) - param-reg-1 0 MOV rc-absolute-cell rt-vm rel-fixup + param-reg-1 %mov-vm-ptr "nest_stacks" f %alien-invoke ; M: x86.64 %unnest-stacks ( -- ) - param-reg-1 0 MOV rc-absolute-cell rt-vm rel-fixup + param-reg-1 %mov-vm-ptr "unnest_stacks" f %alien-invoke ; M: x86.64 %prepare-alien-indirect ( -- ) diff --git a/basis/cpu/x86/x86.factor b/basis/cpu/x86/x86.factor index 8612acdcff..9b063d59e8 100644 --- a/basis/cpu/x86/x86.factor +++ b/basis/cpu/x86/x86.factor @@ -369,19 +369,17 @@ M: x86 %shl int-rep two-operand [ SHL ] emit-shift ; M: x86 %shr int-rep two-operand [ SHR ] emit-shift ; M: x86 %sar int-rep two-operand [ SAR ] emit-shift ; -M: x86 %vm-field-ptr ( dst field -- ) - [ drop 0 MOV rc-absolute-cell rt-vm rel-fixup ] - [ vm-field-offset ADD ] 2bi ; +: %mov-vm-ptr ( reg -- ) + 0 MOV rc-absolute-cell rel-vm ; -: load-zone-ptr ( reg -- ) - #! Load pointer to start of zone array - "nursery" %vm-field-ptr ; +M: x86 %vm-field-ptr ( dst field -- ) + [ drop %mov-vm-ptr ] [ vm-field-offset ADD ] 2bi ; : load-allot-ptr ( nursery-ptr allot-ptr -- ) - [ drop load-zone-ptr ] [ swap cell [+] MOV ] 2bi ; + [ drop "nursery" %vm-field-ptr ] [ swap [] MOV ] 2bi ; : inc-allot-ptr ( nursery-ptr n -- ) - [ cell [+] ] dip 8 align ADD ; + [ [] ] dip 8 align ADD ; : store-header ( temp class -- ) [ [] ] [ type-number tag-fixnum ] bi* MOV ; @@ -395,26 +393,32 @@ M:: x86 %allot ( dst size class nursery-ptr -- ) dst class store-tagged nursery-ptr size inc-allot-ptr ; -M:: x86 %write-barrier ( src card# table -- ) - #! Mark the card pointed to by vreg. +:: (%write-barrier) ( src slot temp1 temp2 -- ) + ! Compute slot address. + temp1 src MOV + temp1 slot ADD + ! Mark the card - card# src MOV - card# card-bits SHR - table "cards_offset" %vm-field-ptr - table table [] MOV - table card# [+] card-mark MOV + temp1 card-bits SHR + temp2 0 MOV rc-absolute-cell rel-cards-offset + temp2 temp1 [+] card-mark MOV ! Mark the card deck - card# deck-bits card-bits - SHR - table "decks_offset" %vm-field-ptr - table table [] MOV - table card# [+] card-mark MOV ; + temp1 deck-bits card-bits - SHR + temp2 0 MOV rc-absolute-cell rel-decks-offset + temp2 temp1 [+] card-mark MOV ; + +M: x86 %write-barrier ( src slot temp1 temp2 -- ) (%write-barrier) ; + +M: x86 %write-barrier-imm ( src slot temp1 temp2 -- ) (%write-barrier) ; M:: x86 %check-nursery ( label size temp1 temp2 -- ) - temp1 load-zone-ptr - temp2 temp1 cell [+] MOV + temp1 "nursery" %vm-field-ptr + ! Load 'here' into temp2 + temp2 temp1 [] MOV temp2 size ADD - temp1 temp1 3 cells [+] MOV + ! Load 'end' into temp1 + temp1 temp1 2 cells [+] MOV temp2 temp1 CMP label JLE ; @@ -1327,8 +1331,8 @@ M:: x86 %save-context ( temp1 temp2 callback-allowed? -- ) #! Save Factor stack pointers in case the C code calls a #! callback which does a GC, which must reliably trace #! all roots. - temp1 0 MOV rc-absolute-cell rt-vm rel-fixup - temp1 temp1 "stack_chain" vm-field-offset [+] MOV + temp1 "stack_chain" %vm-field-ptr + temp1 temp1 [] MOV temp2 stack-reg cell neg [+] LEA temp1 [] temp2 MOV callback-allowed? [ diff --git a/vm/code_block.cpp b/vm/code_block.cpp index 24c68c90ce..f693f8e72c 100755 --- a/vm/code_block.cpp +++ b/vm/code_block.cpp @@ -41,6 +41,8 @@ int factor_vm::number_of_parameters(relocation_type type) case RT_STACK_CHAIN: case RT_MEGAMORPHIC_CACHE_HITS: case RT_VM: + case RT_CARDS_OFFSET: + case RT_DECKS_OFFSET: return 0; default: critical_error("Bad rel type",type); @@ -180,6 +182,10 @@ cell factor_vm::compute_relocation(relocation_entry rel, cell index, code_block return (cell)&megamorphic_cache_hits; case RT_VM: return (cell)this; + case RT_CARDS_OFFSET: + return cards_offset; + case RT_DECKS_OFFSET: + return decks_offset; default: critical_error("Bad rel type",rel); return 0; /* Can't happen */ diff --git a/vm/code_block.hpp b/vm/code_block.hpp index ecdd22864f..e3d392001b 100644 --- a/vm/code_block.hpp +++ b/vm/code_block.hpp @@ -28,6 +28,10 @@ enum relocation_type { RT_MEGAMORPHIC_CACHE_HITS, /* address of vm object */ RT_VM, + /* value of vm->cards_offset */ + RT_CARDS_OFFSET, + /* value of vm->decks_offset */ + RT_DECKS_OFFSET, }; enum relocation_class { diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index c79b8d4d31..56fd34bba0 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -93,21 +93,49 @@ void full_collector::cheneys_algorithm() } } -void factor_vm::collect_full(cell requested_bytes, bool trace_contexts_p) -{ - data_heap *old; - if(current_gc->growing_data_heap) +struct full_updater { + factor_vm *myvm; + + full_updater(factor_vm *myvm_) : myvm(myvm_) {} + + void operator()(heap_block *block) { - old = data; - set_data_heap(data->grow(requested_bytes)); + myvm->relocate_code_block((code_block *)block); + } +}; + +struct literal_and_word_reference_updater { + factor_vm *myvm; + + literal_and_word_reference_updater(factor_vm *myvm_) : myvm(myvm_) {} + + void operator()(heap_block *block) + { + code_block *compiled = (code_block *)block; + myvm->update_literal_references(compiled); + myvm->update_word_references(compiled); + } +}; + +void factor_vm::free_unmarked_code_blocks(bool growing_data_heap) +{ + if(growing_data_heap) + { + full_updater updater(this); + code->free_unmarked(updater); } else { - old = NULL; - std::swap(data->tenured,data->tenured_semispace); - reset_generation(data->tenured); + literal_and_word_reference_updater updater(this); + code->free_unmarked(updater); } + code->points_to_nursery.clear(); + code->points_to_aging.clear(); +} + +void factor_vm::collect_full_impl(bool trace_contexts_p) +{ full_collector collector(this); collector.trace_roots(); @@ -118,12 +146,26 @@ void factor_vm::collect_full(cell requested_bytes, bool trace_contexts_p) } collector.cheneys_algorithm(); - free_unmarked_code_blocks(); reset_generation(data->aging); nursery.here = nursery.start; +} - if(old) delete old; +void factor_vm::collect_growing_heap(cell requested_bytes, bool trace_contexts_p) +{ + data_heap *old = data; + set_data_heap(data->grow(requested_bytes)); + collect_full(trace_contexts_p); + free_unmarked_code_blocks(true); + delete old; +} + +void factor_vm::collect_full(bool trace_contexts_p) +{ + std::swap(data->tenured,data->tenured_semispace); + reset_generation(data->tenured); + collect_full_impl(trace_contexts_p); + free_unmarked_code_blocks(false); } } diff --git a/vm/gc.cpp b/vm/gc.cpp index 3b455b2a3f..10aee9a736 100755 --- a/vm/gc.cpp +++ b/vm/gc.cpp @@ -12,27 +12,6 @@ gc_state::gc_state(data_heap *data_, bool growing_data_heap_, cell collecting_ge gc_state::~gc_state() { } -struct literal_and_word_reference_updater { - factor_vm *myvm; - - literal_and_word_reference_updater(factor_vm *myvm_) : myvm(myvm_) {} - - void operator()(heap_block *block) - { - code_block *compiled = (code_block *)block; - myvm->update_literal_references(compiled); - myvm->update_word_references(compiled); - } -}; - -void factor_vm::free_unmarked_code_blocks() -{ - literal_and_word_reference_updater updater(this); - code->free_unmarked(updater); - code->points_to_nursery.clear(); - code->points_to_aging.clear(); -} - void factor_vm::update_dirty_code_blocks(std::set *remembered_set) { /* The youngest generation that any code block can now reference */ @@ -75,6 +54,7 @@ void factor_vm::garbage_collection(cell collecting_gen_, bool growing_data_heap_ resort to growing the data heap */ if(current_gc->collecting_tenured_p()) { + assert(!current_gc->growing_data_heap); current_gc->growing_data_heap = true; /* Since we start tracing again, any previously @@ -105,7 +85,14 @@ void factor_vm::garbage_collection(cell collecting_gen_, bool growing_data_heap_ collect_aging(); } else if(current_gc->collecting_tenured_p()) - collect_full(requested_bytes,trace_contexts_p); + { + if(current_gc->growing_data_heap) + collect_growing_heap(requested_bytes,trace_contexts_p); + else + collect_full(trace_contexts_p); + } + else + critical_error("Bug in GC",0); record_gc_stats(); diff --git a/vm/vm.hpp b/vm/vm.hpp index 135acd58ab..ae8e882b6b 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -232,12 +232,14 @@ struct factor_vm } // gc - void free_unmarked_code_blocks(); void update_dirty_code_blocks(std::set *remembered_set); void collect_nursery(); void collect_aging(); void collect_to_tenured(); - void collect_full(cell requested_bytes, bool trace_contexts_p); + void free_unmarked_code_blocks(bool growing_data_heap); + void collect_full_impl(bool trace_contexts_p); + void collect_growing_heap(cell requested_bytes, bool trace_contexts_p); + void collect_full(bool trace_contexts_p); void record_gc_stats(); void garbage_collection(cell gen, bool growing_data_heap, bool trace_contexts_p, cell requested_bytes); void gc(); diff --git a/vm/zone.hpp b/vm/zone.hpp index 25859fe500..4fe4ae9b6b 100644 --- a/vm/zone.hpp +++ b/vm/zone.hpp @@ -2,14 +2,13 @@ namespace factor { struct zone { - /* allocation pointer is 'here'; its offset is hardcoded in the - compiler backends */ - cell start; + /* offset of 'here' and 'end' is hardcoded in compiler backends */ cell here; - cell size; + cell start; cell end; + cell size; - zone(cell size_, cell start_) : start(start_), here(0), size(size_), end(start_ + size_) {} + zone(cell size_, cell start_) : here(0), start(start_), end(start_ + size_), size(size_) {} inline bool contains_p(object *pointer) { From 236efe4881d4d2de066b6ac6c9983a72b8e91ef3 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Wed, 14 Oct 2009 04:59:28 -0500 Subject: [PATCH 06/10] vm: debugging new card marking --- vm/code_block.cpp | 3 +-- vm/copying_collector.hpp | 19 +++++++++++++------ vm/full_collector.cpp | 2 +- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/vm/code_block.cpp b/vm/code_block.cpp index f693f8e72c..69622f48a1 100755 --- a/vm/code_block.cpp +++ b/vm/code_block.cpp @@ -170,7 +170,7 @@ cell factor_vm::compute_relocation(relocation_entry rel, cell index, code_block case RT_HERE: { fixnum arg = untag_fixnum(ARG); - return (arg >= 0 ? offset + arg : (cell)(compiled +1) - arg); + return (arg >= 0 ? offset + arg : (cell)(compiled + 1) - arg); } case RT_THIS: return (cell)(compiled + 1); @@ -372,7 +372,6 @@ struct code_block_relocator { { myvm->relocate_code_block_step(rel,index,compiled); } - }; /* Perform all fixups on a code block */ diff --git a/vm/copying_collector.hpp b/vm/copying_collector.hpp index 0d11ba04bc..dfa08868a8 100644 --- a/vm/copying_collector.hpp +++ b/vm/copying_collector.hpp @@ -102,6 +102,8 @@ struct copying_collector : collector { { if(decks[deck_index] & mask) { + this->myvm->gc_stats.decks_scanned++; + cell first_card = first_card_in_deck(deck_index); cell last_card = last_card_in_deck(deck_index); @@ -111,7 +113,9 @@ struct copying_collector : collector { { if(cards[card_index] & mask) { - if(card_start_address(card_index) >= end) + this->myvm->gc_stats.cards_scanned++; + + if(end < card_start_address(card_index)) { start = gen->find_object_containing_card(card_index - gen_start_card); binary_start = start + this->myvm->binary_payload_start((object *)start); @@ -134,17 +138,20 @@ scan_next_object: { if(end < card_end_address(card_index)) { start = gen->next_object_after(this->myvm,start); - if(!start) goto end; - binary_start = start + this->myvm->binary_payload_start((object *)start); - end = start + this->myvm->untagged_object_size((object *)start); - goto scan_next_object; + if(start) + { + binary_start = start + this->myvm->binary_payload_start((object *)start); + end = start + this->myvm->untagged_object_size((object *)start); + goto scan_next_object; + } } } unmarker(card_dirty,&cards[card_index]); - this->myvm->gc_stats.cards_scanned++; deck_dirty |= card_dirty; + + if(!start) goto end; } } diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index 56fd34bba0..2496b963e4 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -155,7 +155,7 @@ void factor_vm::collect_growing_heap(cell requested_bytes, bool trace_contexts_p { data_heap *old = data; set_data_heap(data->grow(requested_bytes)); - collect_full(trace_contexts_p); + collect_full_impl(trace_contexts_p); free_unmarked_code_blocks(true); delete old; } From 0d970ff89d02fcbc0e49d2e884db21b8495147fc Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Wed, 14 Oct 2009 04:59:48 -0500 Subject: [PATCH 07/10] benchmark: update gc1 and add new gc3 benchmark --- extra/benchmark/gc1/gc1.factor | 2 +- extra/benchmark/gc3/authors.txt | 1 + extra/benchmark/gc3/gc3.factor | 11 +++++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 extra/benchmark/gc3/authors.txt create mode 100644 extra/benchmark/gc3/gc3.factor diff --git a/extra/benchmark/gc1/gc1.factor b/extra/benchmark/gc1/gc1.factor index da3b6bab66..4909496d12 100644 --- a/extra/benchmark/gc1/gc1.factor +++ b/extra/benchmark/gc1/gc1.factor @@ -3,6 +3,6 @@ USING: math sequences kernel ; IN: benchmark.gc1 -: gc1 ( -- ) 10 [ 600000 [ >bignum 1 + ] map drop ] times ; +: gc1 ( -- ) 600000 [ >bignum 1 + ] map drop ; MAIN: gc1 diff --git a/extra/benchmark/gc3/authors.txt b/extra/benchmark/gc3/authors.txt new file mode 100644 index 0000000000..d4f5d6b3ae --- /dev/null +++ b/extra/benchmark/gc3/authors.txt @@ -0,0 +1 @@ +Slava Pestov \ No newline at end of file diff --git a/extra/benchmark/gc3/gc3.factor b/extra/benchmark/gc3/gc3.factor new file mode 100644 index 0000000000..cb8e3ee72a --- /dev/null +++ b/extra/benchmark/gc3/gc3.factor @@ -0,0 +1,11 @@ +! Copyright (C) 2009 Slava Pestov. +! See http://factorcode.org/license.txt for BSD license. +USING: math.parser fry sequences kernel assocs hashtables ; +IN: benchmark.gc3 + +: gc3 ( -- ) + 1000000 iota + 1000000 + '[ [ number>string ] keep _ set-at ] each ; + +MAIN: gc3 From 1796688acdfaa6c3ee8d45de4aebd0f8d197b675 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Wed, 14 Oct 2009 06:03:52 -0500 Subject: [PATCH 08/10] vm: during an aging collection, promote objects referenced from tenured directly to tenured --- vm/aging_collector.cpp | 32 ++++++++++++++++++------------ vm/collector.hpp | 16 +++++---------- vm/copying_collector.hpp | 43 +++++++++------------------------------- 3 files changed, 33 insertions(+), 58 deletions(-) diff --git a/vm/aging_collector.cpp b/vm/aging_collector.cpp index 6867ea3083..6d67753b51 100644 --- a/vm/aging_collector.cpp +++ b/vm/aging_collector.cpp @@ -9,22 +9,28 @@ aging_collector::aging_collector(factor_vm *myvm_) : void factor_vm::collect_aging() { - std::swap(data->aging,data->aging_semispace); - reset_generation(data->aging); + { + to_tenured_collector collector(this); + collector.trace_cards(data->tenured, + card_points_to_aging, + simple_unmarker(card_mark_mask)); + collector.cheneys_algorithm(); + } + { + std::swap(data->aging,data->aging_semispace); + reset_generation(data->aging); - aging_collector collector(this); + aging_collector collector(this); - collector.trace_roots(); - collector.trace_contexts(); - collector.trace_cards(data->tenured, - card_points_to_aging, - complex_unmarker(card_mark_mask,card_points_to_nursery)); - collector.trace_code_heap_roots(&code->points_to_aging); - collector.cheneys_algorithm(); - update_dirty_code_blocks(&code->points_to_aging); + collector.trace_roots(); + collector.trace_contexts(); + collector.trace_code_heap_roots(&code->points_to_aging); + collector.cheneys_algorithm(); + update_dirty_code_blocks(&code->points_to_aging); - nursery.here = nursery.start; - code->points_to_nursery.clear(); + nursery.here = nursery.start; + code->points_to_nursery.clear(); + } } } diff --git a/vm/collector.hpp b/vm/collector.hpp index bb6c4042b9..8f9d4f26ac 100644 --- a/vm/collector.hpp +++ b/vm/collector.hpp @@ -30,15 +30,15 @@ template struct collector { return untagged; } - bool trace_handle(cell *handle) + void trace_handle(cell *handle) { cell pointer = *handle; - if(immediate_p(pointer)) return false; + if(immediate_p(pointer)) return; object *untagged = myvm->untag(pointer); if(!policy.should_copy_p(untagged)) - return false; + return; object *forwarding = resolve_forwarding(untagged); @@ -50,24 +50,18 @@ template struct collector { untagged = forwarding; *handle = RETAG(untagged,TAG(pointer)); - - return true; } - bool trace_slots(object *ptr) + void trace_slots(object *ptr) { cell *slot = (cell *)ptr; cell *end = (cell *)((cell)ptr + myvm->binary_payload_start(ptr)); - bool copied = false; - if(slot != end) { slot++; - for(; slot < end; slot++) copied |= trace_handle(slot); + for(; slot < end; slot++) trace_handle(slot); } - - return copied; } object *promote_object(object *untagged) diff --git a/vm/copying_collector.hpp b/vm/copying_collector.hpp index dfa08868a8..1e338899b8 100644 --- a/vm/copying_collector.hpp +++ b/vm/copying_collector.hpp @@ -2,24 +2,13 @@ namespace factor { struct dummy_unmarker { - void operator()(bool result, card *ptr) {} + void operator()(card *ptr) {} }; struct simple_unmarker { card unmask; simple_unmarker(card unmask_) : unmask(unmask_) {} - void operator()(bool result, card *ptr) { *ptr &= ~unmask; } -}; - -struct complex_unmarker { - card unmask_none, unmask_some; - complex_unmarker(card unmask_none_, card unmask_some_) : - unmask_none(unmask_none_), unmask_some(unmask_some_) {} - - void operator()(bool result, card *ptr) - { - *ptr &= (result ? ~unmask_some : ~unmask_none); - } + void operator()(card *ptr) { *ptr &= ~unmask; } }; template @@ -59,10 +48,8 @@ struct copying_collector : collector { return ((card + 1) << card_bits) + this->data->start; } - bool trace_partial_objects(cell start, cell end, cell card_start, cell card_end) + void trace_partial_objects(cell start, cell end, cell card_start, cell card_end) { - bool copied = false; - if(card_start < end) { start += sizeof(cell); @@ -76,11 +63,9 @@ struct copying_collector : collector { if(slot_ptr != end_ptr) { for(; slot_ptr < end_ptr; slot_ptr++) - copied |= this->trace_handle(slot_ptr); + this->trace_handle(slot_ptr); } } - - return copied; } template @@ -107,8 +92,6 @@ struct copying_collector : collector { cell first_card = first_card_in_deck(deck_index); cell last_card = last_card_in_deck(deck_index); - bool deck_dirty = false; - for(cell card_index = first_card; card_index < last_card; card_index++) { if(cards[card_index] & mask) @@ -122,15 +105,13 @@ struct copying_collector : collector { end = start + this->myvm->untagged_object_size((object *)start); } - bool card_dirty = false; - #ifdef FACTOR_DEBUG assert(addr_to_card(start - this->data->start) <= card_index); assert(start < card_end_address(card_index)); #endif scan_next_object: { - card_dirty |= trace_partial_objects( + trace_partial_objects( start, binary_start, card_start_address(card_index), @@ -147,15 +128,13 @@ scan_next_object: { } } - unmarker(card_dirty,&cards[card_index]); + unmarker(&cards[card_index]); - deck_dirty |= card_dirty; - if(!start) goto end; } } - unmarker(deck_dirty,&decks[deck_index]); + unmarker(&decks[deck_index]); } } @@ -180,17 +159,13 @@ end: this->myvm->gc_stats.card_scan_time += (current_micros() - start_time); } template - bool trace_objects_between(SourceGeneration *gen, cell scan, cell *end) + void trace_objects_between(SourceGeneration *gen, cell scan, cell *end) { - bool copied = false; - while(scan && scan < *end) { - copied |= this->trace_slots((object *)scan); + this->trace_slots((object *)scan); scan = gen->next_object_after(this->myvm,scan); } - - return copied; } void cheneys_algorithm() From ae90ebe3be9bd4835b238f8bdcade8025a80f404 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Wed, 14 Oct 2009 06:13:51 -0500 Subject: [PATCH 09/10] vm: faster write barrier for non-optimized code --- vm/vm.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm/vm.hpp b/vm/vm.hpp index ae8e882b6b..b6a3e30af3 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -227,8 +227,8 @@ struct factor_vm pointer from an older generation to a younger one */ inline void write_barrier(cell *slot_ptr) { - data->cards[addr_to_card((cell)slot_ptr - data->start)] = card_mark_mask; - data->decks[addr_to_deck((cell)slot_ptr - data->start)] = card_mark_mask; + *(char *)(cards_offset + ((cell)slot_ptr >> card_bits)) = card_mark_mask; + *(char *)(decks_offset + ((cell)slot_ptr >> deck_bits)) = card_mark_mask; } // gc From 38d120f92eebe80319199954cce98bfa3d5fe598 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Wed, 14 Oct 2009 19:24:23 -0500 Subject: [PATCH 10/10] vm: rt-vm relocation now supports accessing a field directly --- basis/bootstrap/image/image.factor | 13 ++++++++++--- basis/compiler/codegen/codegen.factor | 2 +- basis/compiler/codegen/fixup/fixup.factor | 4 ++-- basis/cpu/x86/32/32.factor | 2 +- basis/cpu/x86/64/bootstrap.factor | 4 ++-- basis/cpu/x86/bootstrap.factor | 3 ++- basis/cpu/x86/x86.factor | 4 ++-- core/quotations/quotations.factor | 4 ++-- vm/arrays.cpp | 19 +++++++++++++++++-- vm/arrays.hpp | 1 + vm/code_block.cpp | 4 ++-- vm/jit.hpp | 6 +++--- vm/layouts.hpp | 3 --- vm/quotations.cpp | 4 +++- 14 files changed, 48 insertions(+), 25 deletions(-) diff --git a/basis/bootstrap/image/image.factor b/basis/bootstrap/image/image.factor index eee65c1eba..d90cd71bc4 100644 --- a/basis/bootstrap/image/image.factor +++ b/basis/bootstrap/image/image.factor @@ -111,18 +111,25 @@ SYMBOL: jit-relocations : jit-rel ( rc rt -- ) over compute-offset 3array jit-relocations get push-all ; -: make-jit ( quot -- jit-data ) +SYMBOL: jit-literals + +: jit-literal ( literal -- ) + jit-literals get push ; + +: make-jit ( quot -- jit-literals jit-data ) [ + V{ } clone jit-literals set V{ } clone jit-relocations set call( -- ) + jit-literals get >array jit-relocations get >array ] B{ } make prefix ; : jit-define ( quot name -- ) - [ make-jit ] dip set ; + [ make-jit nip ] dip set ; : define-sub-primitive ( quot word -- ) - [ make-jit ] dip sub-primitives get set-at ; + [ make-jit 2array ] dip sub-primitives get set-at ; ! The image being constructed; a vector of word-size integers SYMBOL: image diff --git a/basis/compiler/codegen/codegen.factor b/basis/compiler/codegen/codegen.factor index 467e8d9b8d..31918658c4 100755 --- a/basis/compiler/codegen/codegen.factor +++ b/basis/compiler/codegen/codegen.factor @@ -63,7 +63,7 @@ M: ##no-tco generate-insn drop ; M: ##call generate-insn word>> dup sub-primitive>> - [ first % ] [ [ add-call ] [ %call ] bi ] ?if ; + [ second first % ] [ [ add-call ] [ %call ] bi ] ?if ; M: ##jump generate-insn word>> [ add-call ] [ %jump ] bi ; diff --git a/basis/compiler/codegen/fixup/fixup.factor b/basis/compiler/codegen/fixup/fixup.factor index f56d654c77..21a8db807b 100755 --- a/basis/compiler/codegen/fixup/fixup.factor +++ b/basis/compiler/codegen/fixup/fixup.factor @@ -77,8 +77,8 @@ SYMBOL: relocation-table : rel-here ( offset class -- ) [ add-literal ] dip rt-here rel-fixup ; -: rel-vm ( class -- ) - rt-vm rel-fixup ; +: rel-vm ( offset class -- ) + [ add-literal ] dip rt-vm rel-fixup ; : rel-cards-offset ( class -- ) rt-cards-offset rel-fixup ; diff --git a/basis/cpu/x86/32/32.factor b/basis/cpu/x86/32/32.factor index 55e9448c7c..c095f5493c 100755 --- a/basis/cpu/x86/32/32.factor +++ b/basis/cpu/x86/32/32.factor @@ -50,7 +50,7 @@ M: x86.32 reserved-area-size 0 ; M: x86.32 %alien-invoke 0 CALL rc-relative rel-dlsym ; : push-vm-ptr ( -- ) - 0 PUSH rc-absolute-cell rel-vm ; ! push the vm ptr as an argument + 0 PUSH 0 rc-absolute-cell rel-vm ; ! push the vm ptr as an argument M: x86.32 return-struct-in-registers? ( c-type -- ? ) c-type diff --git a/basis/cpu/x86/64/bootstrap.factor b/basis/cpu/x86/64/bootstrap.factor index bffe056656..e26e804232 100644 --- a/basis/cpu/x86/64/bootstrap.factor +++ b/basis/cpu/x86/64/bootstrap.factor @@ -26,10 +26,10 @@ IN: bootstrap.x86 temp0 temp0 [] MOV ! save stack pointer temp0 [] stack-reg MOV - ! load XT - temp1 0 MOV rc-absolute-cell rt-primitive jit-rel ! load vm ptr arg1 0 MOV rc-absolute-cell rt-vm jit-rel + ! load XT + temp1 0 MOV rc-absolute-cell rt-primitive jit-rel ! go temp1 JMP ] jit-primitive jit-define diff --git a/basis/cpu/x86/bootstrap.factor b/basis/cpu/x86/bootstrap.factor index 3cc71d22f7..eb4da296dc 100644 --- a/basis/cpu/x86/bootstrap.factor +++ b/basis/cpu/x86/bootstrap.factor @@ -252,7 +252,7 @@ big-endian off ! pop stack ds-reg bootstrap-cell SUB ! pass vm pointer - arg2 0 MOV rc-absolute-cell rt-vm jit-rel + arg2 0 MOV 0 jit-literal rc-absolute-cell rt-vm jit-rel ! call quotation arg1 quot-xt-offset [+] JMP ] \ (call) define-sub-primitive @@ -402,6 +402,7 @@ big-endian off ! Comparisons : jit-compare ( insn -- ) ! load t + t jit-literal temp3 0 MOV rc-absolute-cell rt-immediate jit-rel ! load f temp1 \ f tag-number MOV diff --git a/basis/cpu/x86/x86.factor b/basis/cpu/x86/x86.factor index 9b063d59e8..4a3545a5ba 100644 --- a/basis/cpu/x86/x86.factor +++ b/basis/cpu/x86/x86.factor @@ -370,10 +370,10 @@ M: x86 %shr int-rep two-operand [ SHR ] emit-shift ; M: x86 %sar int-rep two-operand [ SAR ] emit-shift ; : %mov-vm-ptr ( reg -- ) - 0 MOV rc-absolute-cell rel-vm ; + 0 MOV 0 rc-absolute-cell rel-vm ; M: x86 %vm-field-ptr ( dst field -- ) - [ drop %mov-vm-ptr ] [ vm-field-offset ADD ] 2bi ; + [ 0 MOV ] dip vm-field-offset rc-absolute-cell rel-vm ; : load-allot-ptr ( nursery-ptr allot-ptr -- ) [ drop "nursery" %vm-field-ptr ] [ swap [] MOV ] 2bi ; diff --git a/core/quotations/quotations.factor b/core/quotations/quotations.factor index af3c110d61..f2b17b3f9d 100644 --- a/core/quotations/quotations.factor +++ b/core/quotations/quotations.factor @@ -8,10 +8,10 @@ IN: quotations diff --git a/vm/arrays.cpp b/vm/arrays.cpp index af8e0c82d5..b09ff5c5af 100644 --- a/vm/arrays.cpp +++ b/vm/arrays.cpp @@ -64,14 +64,14 @@ cell factor_vm::allot_array_4(cell v1_, cell v2_, cell v3_, cell v4_) void factor_vm::primitive_resize_array() { - array* a = untag_check(dpop()); + array *a = untag_check(dpop()); cell capacity = unbox_array_size(); dpush(tag(reallot_array(a,capacity))); } void growable_array::add(cell elt_) { - factor_vm* parent_vm = elements.parent_vm; + factor_vm *parent_vm = elements.parent_vm; gc_root elt(elt_,parent_vm); if(count == array_capacity(elements.untagged())) elements = parent_vm->reallot_array(elements.untagged(),count * 2); @@ -79,6 +79,21 @@ void growable_array::add(cell elt_) parent_vm->set_array_nth(elements.untagged(),count++,elt.value()); } +void growable_array::append(array *elts_) +{ + factor_vm *parent_vm = elements.parent_vm; + gc_root elts(elts_,parent_vm); + cell capacity = array_capacity(elts.untagged()); + if(count + capacity > array_capacity(elements.untagged())) + { + elements = parent_vm->reallot_array(elements.untagged(), + (count + capacity) * 2); + } + + for(cell index = 0; index < capacity; index++) + parent_vm->set_array_nth(elements.untagged(),count++,array_nth(elts.untagged(),index)); +} + void growable_array::trim() { factor_vm *parent_vm = elements.parent_vm; diff --git a/vm/arrays.hpp b/vm/arrays.hpp index 79f31573f4..accf8b3b04 100755 --- a/vm/arrays.hpp +++ b/vm/arrays.hpp @@ -29,6 +29,7 @@ struct growable_array { explicit growable_array(factor_vm *myvm, cell capacity = 10) : count(0), elements(myvm->allot_array(capacity,F),myvm) {} void add(cell elt); + void append(array *elts); void trim(); }; diff --git a/vm/code_block.cpp b/vm/code_block.cpp index 69622f48a1..7214aa235e 100755 --- a/vm/code_block.cpp +++ b/vm/code_block.cpp @@ -34,13 +34,13 @@ int factor_vm::number_of_parameters(relocation_type type) case RT_IMMEDIATE: case RT_HERE: case RT_UNTAGGED: + case RT_VM: return 1; case RT_DLSYM: return 2; case RT_THIS: case RT_STACK_CHAIN: case RT_MEGAMORPHIC_CACHE_HITS: - case RT_VM: case RT_CARDS_OFFSET: case RT_DECKS_OFFSET: return 0; @@ -181,7 +181,7 @@ cell factor_vm::compute_relocation(relocation_entry rel, cell index, code_block case RT_MEGAMORPHIC_CACHE_HITS: return (cell)&megamorphic_cache_hits; case RT_VM: - return (cell)this; + return (cell)this + untag_fixnum(ARG); case RT_CARDS_OFFSET: return cards_offset; case RT_DECKS_OFFSET: diff --git a/vm/jit.hpp b/vm/jit.hpp index 789f68e22b..63b4454514 100644 --- a/vm/jit.hpp +++ b/vm/jit.hpp @@ -41,9 +41,9 @@ struct jit { void emit_subprimitive(cell word_) { gc_root word(word_,parent_vm); - gc_root code_template(word->subprimitive,parent_vm); - if(array_capacity(code_template.untagged()) > 1) literal(parent_vm->T); - emit(code_template.value()); + gc_root code_pair(word->subprimitive,parent_vm); + literals.append(parent_vm->untag(array_nth(code_pair.untagged(),0))); + emit(array_nth(code_pair.untagged(),1)); } void emit_class_lookup(fixnum index, cell type); diff --git a/vm/layouts.hpp b/vm/layouts.hpp index 988fc99ec5..2fba97d747 100644 --- a/vm/layouts.hpp +++ b/vm/layouts.hpp @@ -199,9 +199,6 @@ struct string : public object { /* The compiled code heap is structured into blocks. */ struct heap_block { - /* Bit 0: mark - Bit 1-7: type - Bit 8-...: size */ cell header; bool marked_p() { return header & 1; } diff --git a/vm/quotations.cpp b/vm/quotations.cpp index afb02bb485..2823d5cb78 100755 --- a/vm/quotations.cpp +++ b/vm/quotations.cpp @@ -199,7 +199,9 @@ void quotation_jit::iterate_quotation() /* Primitive calls */ if(primitive_call_p(i,length)) { - emit_with(parent_vm->userenv[JIT_PRIMITIVE],obj.value()); + literal(tag_fixnum(0)); + literal(obj.value()); + emit(parent_vm->userenv[JIT_PRIMITIVE]); i++;