vm: add a remembered set for code blocks which may reference young literals. Improves loading time

vm/code_block.cpp

@@ -341,7 +341,7 @@ void factor_vm::update_word_references(code_block *compiled)
 	   the code heap with dead PICs that will be freed on the next
 	   GC, we add them to the free list immediately. */
 	else if(compiled->type == PIC_TYPE)
-		code->heap_free(compiled);
+		code->code_heap_free(compiled);
 	else
 	{
 		word_references_updater updater(this);
@@ -372,18 +372,12 @@ struct code_block_relocator {
 /* Perform all fixups on a code block */
 void factor_vm::relocate_code_block(code_block *compiled)
 {
-	compiled->last_scan = data->nursery();
 	compiled->needs_fixup = false;
 	code_block_relocator relocator(this);
 	iterate_relocations(compiled,relocator);
 	flush_icache_for(compiled);
 }
 
-void relocate_code_block(code_block *compiled, factor_vm *myvm)
-{
-	return myvm->relocate_code_block(compiled);
-}
-
 /* Fixup labels. This is done at compile time, not image load time */
 void factor_vm::fixup_labels(array *labels, code_block *compiled)
 {
@@ -443,7 +437,6 @@ code_block *factor_vm::add_code_block(cell type, cell code_, cell labels_, cell
 
 	/* compiled header */
 	compiled->type = type;
-	compiled->last_scan = data->nursery();
 	compiled->needs_fixup = true;
 
 	/* slight space optimization */
@@ -466,7 +459,7 @@ code_block *factor_vm::add_code_block(cell type, cell code_, cell labels_, cell
 
 	/* next time we do a minor GC, we have to scan the code heap for
 	literals */
-	this->code->last_code_heap_scan = data->nursery();
+	this->code->write_barrier(compiled);
 
 	return compiled;
 }

vm/code_heap.cpp

@@ -5,6 +5,18 @@ namespace factor
 
 code_heap::code_heap(factor_vm *myvm, cell size) : heap(myvm,size) {}
 
+void code_heap::write_barrier(code_block *compiled)
+{
+	remembered_set[compiled] = myvm->data->nursery();
+	youngest_referenced_generation = myvm->data->nursery();
+}
+
+void code_heap::code_heap_free(code_block *compiled)
+{
+	remembered_set.erase(compiled);
+	heap_free(compiled);
+}
+
 /* Allocate a code heap during startup */
 void factor_vm::init_code_heap(cell size)
 {

vm/code_heap.hpp

@@ -2,13 +2,17 @@ namespace factor
 {
 
 struct code_heap : heap {
-	/* What generation was being collected when trace_code_heap_roots() was last
+	/* Maps code blocks to the youngest generation containing
-	   called? Until the next call to add_code_block(), future
+	one of their literals. If this is tenured (0), the code block
-	   collections of younger generations don't have to touch the code
+	is not part of the remembered set. */
-	   heap. */
+	unordered_map<code_block *, cell> remembered_set;
-	cell last_code_heap_scan;
 	
+	/* Minimum value in the above map. */
+	cell youngest_referenced_generation;
+
 	explicit code_heap(factor_vm *myvm, cell size);
+	void write_barrier(code_block *compiled);
+	void code_heap_free(code_block *compiled);
 };
 
 }

vm/data_gc.cpp

@@ -5,7 +5,7 @@ namespace factor
 
 void factor_vm::init_data_gc()
 {
-	code->last_code_heap_scan = data->nursery();
+	code->youngest_referenced_generation = data->nursery();
 }
 
 gc_state::gc_state(data_heap *data_, bool growing_data_heap_, cell collecting_gen_) :
@@ -287,47 +287,23 @@ template<typename Strategy> void factor_vm::trace_contexts(Strategy &strategy)
 /* Trace all literals referenced from a code block. Only for aging and nursery collections */
 template<typename Strategy> void factor_vm::trace_literal_references(code_block *compiled, Strategy &strategy)
 {
-	if(current_gc->collecting_gen >= compiled->last_scan)
+	trace_handle(&compiled->literals,strategy);
-	{
+	trace_handle(&compiled->relocation,strategy);
-		if(current_gc->collecting_accumulation_gen_p())
-			compiled->last_scan = current_gc->collecting_gen;
-		else
-			compiled->last_scan = current_gc->collecting_gen + 1;
-
-		trace_handle(&compiled->literals,strategy);
-		trace_handle(&compiled->relocation,strategy);
-
-		/* once we finish tracing, re-visit this code block and update
-		literals */
-		current_gc->dirty_code_blocks.insert(compiled);
-	}
 }
 
-template<typename Strategy> struct literal_reference_tracer {
-	factor_vm *myvm;
-	Strategy strategy;
-
-	explicit literal_reference_tracer(factor_vm *myvm_, Strategy &strategy_) :
-		myvm(myvm_), strategy(strategy_) {}
-
-	void operator()(code_block *compiled)
-	{
-		myvm->trace_literal_references(compiled,strategy);
-	}
-};
-
 /* Trace literals referenced from all code blocks. Only for aging and nursery collections */
 template<typename Strategy> void factor_vm::trace_code_heap_roots(Strategy &strategy)
 {
-	if(current_gc->collecting_gen >= code->last_code_heap_scan)
+	if(current_gc->collecting_gen >= code->youngest_referenced_generation)
 	{
-		literal_reference_tracer<Strategy> tracer(this,strategy);
+		unordered_map<code_block *,cell>::const_iterator iter = code->remembered_set.begin();
-		iterate_code_heap(tracer);
+		unordered_map<code_block *,cell>::const_iterator end = code->remembered_set.end();
 
-		if(current_gc->collecting_accumulation_gen_p())
+		for(; iter != end; iter++)
-			code->last_code_heap_scan = current_gc->collecting_gen;
+		{
-		else
+			if(current_gc->collecting_gen >= iter->second)
-			code->last_code_heap_scan = current_gc->collecting_gen + 1;
+				trace_literal_references(iter->first,strategy);
+		}
 
 		code_heap_scans++;
 	}
@@ -340,9 +316,7 @@ template<typename Strategy> void factor_vm::mark_code_block(code_block *compiled
 	check_code_address((cell)compiled);
 
 	code->mark_block(compiled);
-
+	trace_literal_references(compiled,strategy);
-	trace_handle(&compiled->literals,strategy);
-	trace_handle(&compiled->relocation,strategy);
 }
 
 struct literal_and_word_reference_updater {
@@ -362,19 +336,34 @@ void factor_vm::free_unmarked_code_blocks()
 {
 	literal_and_word_reference_updater updater(this);
 	code->free_unmarked(updater);
-	code->last_code_heap_scan = current_gc->collecting_gen;
+	code->remembered_set.clear();
+	code->youngest_referenced_generation = data->tenured();
 }
 
 void factor_vm::update_dirty_code_blocks()
 {
-	std::set<code_block *> dirty_code_blocks = current_gc->dirty_code_blocks;
+	/* The youngest generation that any code block can now reference */
-	std::set<code_block *>::const_iterator iter = dirty_code_blocks.begin();
+	cell gen;
-	std::set<code_block *>::const_iterator end = dirty_code_blocks.end();
+
+	if(current_gc->collecting_accumulation_gen_p())
+		gen = current_gc->collecting_gen;
+	else
+		gen = current_gc->collecting_gen + 1;
+
+	unordered_map<code_block *,cell>::iterator iter = code->remembered_set.begin();
+	unordered_map<code_block *,cell>::iterator end = code->remembered_set.end();
 
 	for(; iter != end; iter++)
-		update_literal_references(*iter);
+	{
+		if(current_gc->collecting_gen >= iter->second)
+		{
+			check_code_address((cell)iter->first);
+			update_literal_references(iter->first);
+			iter->second = gen;
+		}
+	}
 
-	dirty_code_blocks.clear();
+	code->youngest_referenced_generation = gen;
 }
 
 template<typename Strategy>
@@ -635,11 +624,8 @@ If growing_data_heap_ is true, we must grow the data heap to such a size that
 an allocation of requested_bytes won't fail */
 void factor_vm::garbage_collection(cell collecting_gen_, bool growing_data_heap_, bool trace_contexts_, cell requested_bytes)
 {
-	if(gc_off)
+	assert(!gc_off);
-	{
+	assert(!current_gc);
-		critical_error("GC disabled",collecting_gen_);
-		return;
-	}
 
 	current_gc = new gc_state(data,growing_data_heap_,collecting_gen_);
 

vm/data_gc.hpp

@@ -25,9 +25,6 @@ struct gc_state {
 	   full, we go on to collect tenured */
 	bool collecting_aging_again;
 
-	/* A set of code blocks which need to have their literals updated */
-	std::set<code_block *> dirty_code_blocks;
-
 	/* GC start time, for benchmarking */
 	u64 start_time;
 

vm/errors.cpp

@@ -29,7 +29,7 @@ void factor_vm::throw_error(cell error, stack_frame *callstack_top)
 {
 	/* If the error handler is set, we rewind any C stack frames and
 	pass the error to user-space. */
-	if(userenv[BREAK_ENV] != F)
+	if(!current_gc && userenv[BREAK_ENV] != F)
 	{
 		/* If error was thrown during heap scan, we re-enable the GC */
 		gc_off = false;

vm/inline_cache.cpp

@@ -28,7 +28,7 @@ void factor_vm::deallocate_inline_cache(cell return_address)
 #endif
 
 	if(old_type == PIC_TYPE)
-		code->heap_free(old_block);
+		code->code_heap_free(old_block);
 }
 
 /* Figure out what kind of type check the PIC needs based on the methods

vm/layouts.hpp

@@ -207,7 +207,7 @@ struct heap_block
 {
 	unsigned char status; /* free or allocated? */
 	unsigned char type; /* this is WORD_TYPE or QUOTATION_TYPE */
-	unsigned char last_scan; /* the youngest generation in which this block's literals may live */
+	unsigned char unused;
 	unsigned char needs_fixup; /* is this a new block that needs full fixup? */
 
 	/* In bytes, includes this header */