vm: more sweep phase optimizations

vm/compaction.cpp

@@ -159,26 +159,29 @@ struct dummy_slot_forwarder {
 };
 
 /* Compact just the code heap */
-void factor_vm::collect_compact_code_impl()
+void factor_vm::collect_compact_code_impl(bool trace_contexts_p)
 {
-	mark_bits<code_block> *code_forwarding_map = &code->allocator->state;
-
 	/* Figure out where blocks are going to go */
+	mark_bits<code_block> *code_forwarding_map = &code->allocator->state;
 	code_forwarding_map->compute_forwarding();
-
-	/* Update root pointers */
 	code_block_visitor<forwarder<code_block> > code_forwarder(this,forwarder<code_block>(code_forwarding_map));
 
+	if(trace_contexts_p)
+	{
+		code_forwarder.visit_context_code_blocks();
+		code_forwarder.visit_callback_code_blocks();
+	}
+
+	/* Update code heap references in data heap */
+	object_code_block_updater updater(&code_forwarder);
+	each_object(updater);
+
 	/* Slide everything in the code heap up, and update code heap
 	pointers inside code blocks. */
 	dummy_slot_forwarder slot_forwarder;
 	code_block_compaction_updater<dummy_slot_forwarder> code_block_updater(this,slot_forwarder);
 	standard_sizer<code_block> code_block_sizer;
 	code->allocator->compact(code_block_updater,code_block_sizer);
-
-	/* Update code heap references in data heap */
-	object_code_block_updater updater(&code_forwarder);
-	each_object(updater);
 }
 
 }

vm/full_collector.cpp

@@ -76,9 +76,7 @@ void factor_vm::collect_mark_impl(bool trace_contexts_p)
 void factor_vm::collect_sweep_impl()
 {
 	current_gc->event->started_data_sweep();
-	data->tenured->starts.clear_object_start_offsets();
-	object_start_map_updater updater(&data->tenured->starts);
-	data->tenured->sweep(updater);
+	data->tenured->sweep();
 	current_gc->event->ended_data_sweep();
 }
 
@@ -88,7 +86,7 @@ void factor_vm::collect_growing_heap(cell requested_bytes, bool trace_contexts_p
 	data_heap *old = data;
 	set_data_heap(data->grow(requested_bytes));
 	collect_mark_impl(trace_contexts_p);
-	collect_compact_code_impl();
+	collect_compact_code_impl(trace_contexts_p);
 	delete old;
 }
 

vm/mark_bits.hpp

@@ -166,18 +166,7 @@ template<typename Block> struct mark_bits {
 		return n;
 	}
 
-	Block *next_unmarked_block_after_slow(Block *original)
-	{
-		char *scan = (char *)original;
-		char *end = (char *)(this->start + this->size);
-
-		while(scan != end && marked_p((Block *)scan))
-			scan += block_granularity;
-
-		return (Block *)scan;
-	}
-
-	Block *next_unmarked_block_after_fast(Block *original)
+	Block *next_unmarked_block_after(Block *original)
 	{
 		std::pair<cell,cell> position = bitmap_deref(original);
 		cell bit_index = position.second;
@@ -204,23 +193,42 @@ template<typename Block> struct mark_bits {
 		return (Block *)(this->start + this->size);
 	}
 
-	Block *next_unmarked_block_after(Block *original)
+	cell rightmost_set_bit(u64 x)
 	{
-		Block *first_result = next_unmarked_block_after_slow(original);
-		Block *second_result = next_unmarked_block_after_fast(original);
-		assert(first_result == second_result);
-		return second_result;
+		cell n = 0;
+		while(!(x & 1))
+		{
+			n++;
+			x >>= 1;
+		}
+		return n;
 	}
 
 	Block *next_marked_block_after(Block *original)
 	{
-		char *scan = (char *)original;
-		char *end = (char *)(this->start + this->size);
+		std::pair<cell,cell> position = bitmap_deref(original);
+		cell bit_index = position.second;
 
-		while(scan != end && !marked_p((Block *)scan))
-			scan += block_granularity;
+		for(cell index = position.first; index < bits_size; index++)
+		{
+			u64 mask = (marked[index] >> bit_index);
+			if(mask)
+			{
+				/* Found an marked block on this page.
+				Stop, it's hammer time */
+				cell set_bit = rightmost_set_bit(mask);
+				return line_block(index * 64 + bit_index + set_bit);
+			}
+			else
+			{
+				/* No marked blocks on this page.
+				Keep looking */
+				bit_index = 0;
+			}
+		}
 
-		return (Block *)scan;
+		/* No marked blocks were found */
+		return (Block *)(this->start + this->size);
 	}
 
 	cell unmarked_block_size(Block *original)

vm/vm.hpp

@@ -258,7 +258,7 @@ struct factor_vm
 	void collect_mark_impl(bool trace_contexts_p);
 	void collect_sweep_impl();
 	void collect_compact_impl(bool trace_contexts_p);
-	void collect_compact_code_impl();
+	void collect_compact_code_impl(bool trace_contexts_p);
 	void collect_growing_heap(cell requested_bytes, bool trace_contexts_p);
 	void gc(gc_op op, cell requested_bytes, bool trace_contexts_p);
 	void primitive_minor_gc();