vm: new card marking implementation supports marking partial objects

vm/arrays.cpp

@@ -16,8 +16,7 @@ array *factor_vm::allot_array(cell capacity, cell fill_)
 		/* No need for write barrier here. Either the object is in
 		the nursery, or it was allocated directly in tenured space
 		and the write barrier is already hit for us in that case. */
-		cell i;
+		for(cell i = 0; i < capacity; i++)
-		for(i = 0; i < capacity; i++)
 			new_array->data()[i] = fill.value();
 	}
 	return new_array.untagged();

vm/arrays.hpp

@@ -17,8 +17,9 @@ inline void factor_vm::set_array_nth(array *array, cell slot, cell value)
 	assert(array->h.hi_tag() == ARRAY_TYPE);
 	check_tagged_pointer(value);
 #endif
-	array->data()[slot] = value;
+	cell *slot_ptr = &array->data()[slot];
-	write_barrier(array);
+	*slot_ptr = value;
+	write_barrier(slot_ptr);
 }
 
 struct growable_array {

vm/copying_collector.hpp

@@ -29,28 +29,24 @@ struct copying_collector : collector<TargetGeneration,Policy> {
 	explicit copying_collector(factor_vm *myvm_, TargetGeneration *target_, Policy policy_) :
 		collector<TargetGeneration,Policy>(myvm_,target_,policy_), scan(target_->here) {}
 
-	template<typename SourceGeneration>
+	inline cell first_card_in_deck(cell deck)
-	bool trace_objects_between(SourceGeneration *gen, cell scan, cell *end)
-	{
-		bool copied = false;
-
-		while(scan && scan < *end)
-		{
-			copied |= this->trace_slots((object *)scan);
-			scan = gen->next_object_after(this->myvm,scan);
-		}
-
-		return copied;
-	}
-
-	inline cell card_index(cell deck)
 	{
 		return deck << (deck_bits - card_bits);
 	}
 
-	inline cell card_deck_index(cell a)
+	inline cell last_card_in_deck(cell deck)
 	{
-		return (a - this->data->start) >> deck_bits;
+		return first_card_in_deck(deck + 1);
+	}
+
+	inline cell card_to_addr(cell c)
+	{
+		return c << card_bits + this->data->start;
+	}
+
+	inline cell card_deck_for_address(cell a)
+	{
+		return addr_to_deck(a - this->data->start);
 	}
 
 	inline cell card_start_address(cell card)
@@ -58,36 +54,31 @@ struct copying_collector : collector<TargetGeneration,Policy> {
 		return (card << card_bits) + this->data->start;
 	}
 
-	template<typename SourceGeneration, typename Unmarker>
+	inline cell card_end_address(cell card)
-	bool trace_card(SourceGeneration *gen, card *cards, cell card_index, Unmarker unmarker)
 	{
-		cell card_start = card_start_address(card_index);
+		return ((card + 1) << card_bits) + this->data->start;
-		cell card_scan = card_start + gen->first_object_in_card(card_start);
-		cell card_end = card_start_address(card_index + 1);
-
-		bool result = this->trace_objects_between(gen,card_scan,&card_end);
-		unmarker(result,&cards[card_index]);
-
-		this->myvm->gc_stats.cards_scanned++;
-
-		return result;
 	}
 
-	template<typename SourceGeneration, typename Unmarker>
+	bool trace_partial_objects(cell start, cell end, cell card_start, cell card_end)
-	bool trace_card_deck(SourceGeneration *gen, cell deck_index, card mask, Unmarker unmarker)
 	{
-		cell first_card = card_index(deck_index);
-		cell last_card = card_index(deck_index + 1);
-
 		bool copied = false;
 
-		card *cards = this->data->cards;
+		if(card_start < end)
-		for(cell i = first_card; i < last_card; i++)
 		{
-			if(cards[i] & mask) copied |= trace_card(gen,cards,i,unmarker);
+			start += sizeof(cell);
-		}
 
-		this->myvm->gc_stats.decks_scanned++;
+			if(start < card_start) start = card_start;
+			if(end > card_end) end = card_end;
+
+			cell *slot_ptr = (cell *)start;
+			cell *end_ptr = (cell *)end;
+
+			if(slot_ptr != end_ptr)
+			{
+				for(; slot_ptr < end_ptr; slot_ptr++)
+					copied |= this->trace_handle(slot_ptr);
+			}
+		}
 
 		return copied;
 	}
@@ -95,19 +86,73 @@ struct copying_collector : collector<TargetGeneration,Policy> {
 	template<typename SourceGeneration, typename Unmarker>
 	void trace_cards(SourceGeneration *gen, card mask, Unmarker unmarker)
 	{
-		u64 start = current_micros();
+		u64 start_time = current_micros();
-
+	
-		cell first_deck = card_deck_index(gen->start);
-		cell last_deck = card_deck_index(gen->end);
-
 		card_deck *decks = this->data->decks;
-		for(cell i = first_deck; i < last_deck; i++)
+		card_deck *cards = this->data->cards;
+	
+		cell gen_start_card = addr_to_card(gen->start - this->data->start);
+
+		cell first_deck = card_deck_for_address(gen->start);
+		cell last_deck = card_deck_for_address(gen->end);
+	
+		cell start = 0, binary_start = 0, end = 0;
+	
+		for(cell deck_index = first_deck; deck_index < last_deck; deck_index++)
 		{
-			if(decks[i] & mask)
+			if(decks[deck_index] & mask)
-				unmarker(trace_card_deck(gen,i,mask,unmarker),&decks[i]);
+			{
+				cell first_card = first_card_in_deck(deck_index);
+				cell last_card = last_card_in_deck(deck_index);
+	
+				bool deck_dirty = false;
+	
+				for(cell card_index = first_card; card_index < last_card; card_index++)
+				{
+					if(cards[card_index] & mask)
+					{
+						if(card_start_address(card_index) >= end)
+						{
+							start = gen->find_object_containing_card(card_index - gen_start_card);
+							binary_start = start + this->myvm->binary_payload_start((object *)start);
+							end = start + this->myvm->untagged_object_size((object *)start);
+						}
+	
+						bool card_dirty = false;
+	
+#ifdef FACTOR_DEBUG
+						assert(addr_to_card(start - this->data->start) <= card_index);
+						assert(start < card_end_address(card_index));
+#endif
+
+scan_next_object:				{
+							card_dirty |= trace_partial_objects(
+								start,
+								binary_start,
+								card_start_address(card_index),
+								card_end_address(card_index));
+							if(end < card_end_address(card_index))
+							{
+								start = gen->next_object_after(this->myvm,start);
+								if(!start) goto end;
+								binary_start = start + this->myvm->binary_payload_start((object *)start);
+								end = start + this->myvm->untagged_object_size((object *)start);
+								goto scan_next_object;
+							}
+						}
+	
+						unmarker(card_dirty,&cards[card_index]);
+						this->myvm->gc_stats.cards_scanned++;
+	
+						deck_dirty |= card_dirty;
+					}
+				}
+	
+				unmarker(deck_dirty,&decks[deck_index]);
+			}
 		}
 
-		this->myvm->gc_stats.card_scan_time += (current_micros() - start);
+end:		this->myvm->gc_stats.card_scan_time += (current_micros() - start_time);
 	}
 
 	/* Trace all literals referenced from a code block. Only for aging and nursery collections */
@@ -127,6 +172,20 @@ struct copying_collector : collector<TargetGeneration,Policy> {
 		for(; iter != end; iter++) trace_literal_references(*iter);
 	}
 
+	template<typename SourceGeneration>
+	bool trace_objects_between(SourceGeneration *gen, cell scan, cell *end)
+	{
+		bool copied = false;
+
+		while(scan && scan < *end)
+		{
+			copied |= this->trace_slots((object *)scan);
+			scan = gen->next_object_after(this->myvm,scan);
+		}
+
+		return copied;
+	}
+
 	void cheneys_algorithm()
 	{
 		trace_objects_between(this->target,scan,&this->target->here);

vm/data_heap.cpp

@@ -5,12 +5,11 @@ namespace factor
 
 void factor_vm::init_card_decks()
 {
-	cell start = align(data->seg->start,deck_size);
+	cards_offset = (cell)data->cards - addr_to_card(data->start);
-	cards_offset = (cell)data->cards - (start >> card_bits);
+	decks_offset = (cell)data->decks - addr_to_deck(data->start);
-	decks_offset = (cell)data->decks - (start >> deck_bits);
 }
 
-data_heap::data_heap(factor_vm *myvm, cell young_size_, cell aging_size_, cell tenured_size_)
+data_heap::data_heap(cell young_size_, cell aging_size_, cell tenured_size_)
 {
 	young_size_ = align(young_size_,deck_size);
 	aging_size_ = align(aging_size_,deck_size);
@@ -26,12 +25,12 @@ data_heap::data_heap(factor_vm *myvm, cell young_size_, cell aging_size_, cell t
 
 	seg = new segment(total_size);
 
-	cell cards_size = total_size >> card_bits;
+	cell cards_size = addr_to_card(total_size);
 
 	cards = new card[cards_size];
 	cards_end = cards + cards_size;
 
-	cell decks_size = total_size >> deck_bits;
+	cell decks_size = addr_to_deck(total_size);
 	decks = new card_deck[decks_size];
 	decks_end = decks + decks_size;
 
@@ -60,30 +59,24 @@ data_heap::~data_heap()
 	delete[] decks;
 }
 
-data_heap *factor_vm::grow_data_heap(data_heap *data, cell requested_bytes)
+data_heap *data_heap::grow(cell requested_bytes)
 {
-	cell new_tenured_size = (data->tenured_size * 2) + requested_bytes;
+	cell new_tenured_size = (tenured_size * 2) + requested_bytes;
-
+	return new data_heap(young_size,aging_size,new_tenured_size);
-	return new data_heap(this,
-		data->young_size,
-		data->aging_size,
-		new_tenured_size);
 }
 
 void factor_vm::clear_cards(old_space *gen)
 {
-	/* NOTE: reverse order due to heap layout. */
+	cell first_card = addr_to_card(gen->start - data->start);
-	card *first_card = addr_to_card(gen->start);
+	cell last_card = addr_to_card(gen->end - data->start);
-	card *last_card = addr_to_card(gen->end);
+	memset(&data->cards[first_card],0,last_card - first_card);
-	memset(first_card,0,last_card - first_card);
 }
 
 void factor_vm::clear_decks(old_space *gen)
 {
-	/* NOTE: reverse order due to heap layout. */
+	cell first_deck = addr_to_deck(gen->start - data->start);
-	card_deck *first_deck = addr_to_deck(gen->start);
+	cell last_deck = addr_to_deck(gen->end - data->start);
-	card_deck *last_deck = addr_to_deck(gen->end);
+	memset(&data->decks[first_deck],0,last_deck - first_deck);
-	memset(first_deck,0,last_deck - first_deck);
 }
 
 /* After garbage collection, any generations which are now empty need to have
@@ -110,7 +103,7 @@ void factor_vm::set_data_heap(data_heap *data_)
 
 void factor_vm::init_data_heap(cell young_size, cell aging_size, cell tenured_size, bool secure_gc_)
 {
-	set_data_heap(new data_heap(this,young_size,aging_size,tenured_size));
+	set_data_heap(new data_heap(young_size,aging_size,tenured_size));
 	secure_gc = secure_gc_;
 }
 

vm/data_heap.hpp

@@ -22,8 +22,9 @@ struct data_heap {
 	card_deck *decks;
 	card_deck *decks_end;
 	
-	explicit data_heap(factor_vm *myvm, cell young_size, cell aging_size, cell tenured_size);
+	explicit data_heap(cell young_size, cell aging_size, cell tenured_size);
 	~data_heap();
+	data_heap *grow(cell requested_size);
 };
 
 static const cell nursery_gen = 0;

vm/full_collector.cpp

@@ -99,7 +99,7 @@ void factor_vm::collect_full(cell requested_bytes, bool trace_contexts_p)
 	if(current_gc->growing_data_heap)
 	{
 		old = data;
-		set_data_heap(grow_data_heap(data,requested_bytes));
+		set_data_heap(data->grow(requested_bytes));
 	}
 	else
 	{

vm/gc.cpp

@@ -249,7 +249,9 @@ object *factor_vm::allot_object(header header, cell size)
 		/* Allows initialization code to store old->new pointers
 		without hitting the write barrier in the common case of
 		a nursery allocation */
-		write_barrier(obj);
+		char *start = (char *)obj;
+		for(cell offset = 0; offset < size; offset += card_size)
+			write_barrier((cell *)(start + offset));
 	}
 
 	obj->h = header;

vm/generic_arrays.hpp

@@ -45,13 +45,13 @@ template<typename Array> Array *factor_vm::reallot_array(Array *array_, cell cap
 		cell to_copy = array_capacity(array.untagged());
 		if(capacity < to_copy)
 			to_copy = capacity;
-			
+
 		Array *new_array = allot_array_internal<Array>(capacity);
-		
+
 		memcpy(new_array + 1,array.untagged() + 1,to_copy * Array::element_size);
 		memset((char *)(new_array + 1) + to_copy * Array::element_size,
 		       0,(capacity - to_copy) * Array::element_size);
-		
+
 		return new_array;
 	}
 }

vm/old_space.cpp

@@ -5,9 +5,8 @@ namespace factor
 
 old_space::old_space(cell size_, cell start_) : zone(size_,start_)
 {
-	cell cards_size = size_ >> card_bits;
+	object_start_offsets = new card[addr_to_card(size_)];
-	object_start_offsets = new card[cards_size];
+	object_start_offsets_end = object_start_offsets + addr_to_card(size_);
-	object_start_offsets_end = object_start_offsets + cards_size;
 }
 
 old_space::~old_space()
@@ -15,12 +14,38 @@ old_space::~old_space()
 	delete[] object_start_offsets;
 }
 
+cell old_space::first_object_in_card(cell card_index)
+{
+	return object_start_offsets[card_index];
+}
+
+cell old_space::find_object_containing_card(cell card_index)
+{
+	if(card_index == 0)
+		return start;
+	else
+	{
+		card_index--;
+
+		while(first_object_in_card(card_index) == card_starts_inside_object)
+		{
+#ifdef FACTOR_DEBUG
+			/* First card should start with an object */
+			assert(card_index > 0);
+#endif
+			card_index--;
+		}
+
+		return start + (card_index << card_bits) + first_object_in_card(card_index);
+	}
+}
+
 /* we need to remember the first object allocated in the card */
 void old_space::record_object_start_offset(object *obj)
 {
-	card *ptr = (card *)((((cell)obj - start) >> card_bits) + (cell)object_start_offsets);
+	cell idx = addr_to_card((cell)obj - start);
-	if(*ptr == card_starts_inside_object)
+	if(object_start_offsets[idx] == card_starts_inside_object)
-		*ptr = ((cell)obj & addr_card_mask);
+		object_start_offsets[idx] = ((cell)obj & addr_card_mask);
 }
 
 object *old_space::allot(cell size)
@@ -34,7 +59,7 @@ object *old_space::allot(cell size)
 
 void old_space::clear_object_start_offsets()
 {
-	memset(object_start_offsets,card_starts_inside_object,size >> card_bits);
+	memset(object_start_offsets,card_starts_inside_object,addr_to_card(size));
 }
 
 cell old_space::next_object_after(factor_vm *myvm, cell scan)

vm/old_space.hpp

@@ -10,26 +10,8 @@ struct old_space : zone {
 	old_space(cell size_, cell start_);
 	~old_space();
 
-	cell object_start_map_index(cell address)
+	cell old_space::first_object_in_card(cell card_index);
-	{
+	cell find_object_containing_card(cell card_index);
-		return (address - start) >> card_bits;
-	}
-
-	/* Find the first object starting on or after the given address */
-	cell first_object_in_card(cell address)
-	{
-		return object_start_offsets[object_start_map_index(address)];
-	}
-
-	/* Find the card which contains the header of the object which contains
-	the given address */
-	cell find_card_containing_header(cell address)
-	{
-		cell i = object_start_map_index(address);
-		while(i >= 0 && object_start_offsets[i] == card_starts_inside_object) i--;
-		return i;
-	}
-
 	void record_object_start_offset(object *obj);
 	object *allot(cell size);
 	void clear_object_start_offsets();

vm/run.cpp

@@ -37,8 +37,9 @@ void factor_vm::primitive_set_slot()
 	object *obj = untag<object>(dpop());
 	cell value = dpop();
 
-	obj->slots()[slot] = value;
+	cell *slot_ptr = &obj->slots()[slot];
-	write_barrier(obj);
+	*slot_ptr = value;
+	write_barrier(slot_ptr);
 }
 
 void factor_vm::primitive_load_locals()

vm/strings.cpp

@@ -45,8 +45,8 @@ void factor_vm::set_string_nth_slow(string *str_, cell index, cell ch)
 		the bits are clear. */
 		aux = allot_array_internal<byte_array>(untag_fixnum(str->length) * sizeof(u16));
 
-		write_barrier(str.untagged());
 		str->aux = tag<byte_array>(aux);
+		write_barrier(&str->aux);
 	}
 	else
 		aux = untag<byte_array>(str->aux);
@@ -143,8 +143,8 @@ string* factor_vm::reallot_string(string *str_, cell capacity)
 		{
 			byte_array *new_aux = allot_byte_array(capacity * sizeof(u16));
 
-			write_barrier(new_str.untagged());
 			new_str->aux = tag<byte_array>(new_aux);
+			write_barrier(&new_str->aux);
 
 			byte_array *aux = untag<byte_array>(str->aux);
 			memcpy(new_aux->data<u16>(),aux->data<u16>(),to_copy * sizeof(u16));

vm/vm.hpp

@@ -203,7 +203,6 @@ struct factor_vm
 
 	//data heap
 	void init_card_decks();
-	data_heap *grow_data_heap(data_heap *data, cell requested_bytes);
 	void clear_cards(old_space *gen);
 	void clear_decks(old_space *gen);
 	void reset_generation(old_space *gen);
@@ -224,38 +223,12 @@ struct factor_vm
 	cell find_all_words();
 	cell object_size(cell tagged);
 
-	//write barrier
-	inline card *addr_to_card(cell a)
-	{
-		return (card*)(((cell)(a) >> card_bits) + cards_offset);
-	}
-
-	inline cell card_to_addr(card *c)
-	{
-		return ((cell)c - cards_offset) << card_bits;
-	}
-
-	inline card_deck *addr_to_deck(cell a)
-	{
-		return (card_deck *)(((cell)a >> deck_bits) + decks_offset);
-	}
-
-	inline cell deck_to_addr(card_deck *c)
-	{
-		return ((cell)c - decks_offset) << deck_bits;
-	}
-
-	inline card *deck_to_card(card_deck *d)
-	{
-		return (card *)((((cell)d - decks_offset) << (deck_bits - card_bits)) + cards_offset);
-	}
-
 	/* the write barrier must be called any time we are potentially storing a
 	   pointer from an older generation to a younger one */
-	inline void write_barrier(object *obj)
+	inline void write_barrier(cell *slot_ptr)
 	{
-		*addr_to_card((cell)obj) = card_mark_mask;
+		data->cards[addr_to_card((cell)slot_ptr - data->start)] = card_mark_mask;
-		*addr_to_deck((cell)obj) = card_mark_mask;
+		data->decks[addr_to_deck((cell)slot_ptr - data->start)] = card_mark_mask;
 	}
 
 	// gc

vm/write_barrier.hpp

@@ -25,4 +25,13 @@ static const cell deck_bits = (card_bits + 10);
 static const cell deck_size = (1<<deck_bits);
 static const cell addr_deck_mask = (deck_size-1);
 
+inline cell addr_to_card(cell a)
+{
+	return a >> card_bits;
+}
+
+inline cell addr_to_deck(cell a)
+{
+	return a >> deck_bits;
+}
 }