vm: working on new compaction implementation

vm/code_heap.cpp

@@ -139,7 +139,11 @@ void factor_vm::primitive_code_room()
 
 code_block *code_heap::forward_code_block(code_block *compiled)
 {
-	return (code_block *)forwarding[compiled];
+	code_block *block1 = (code_block *)state->forward_block(compiled);
+	code_block *block2 = (code_block *)forwarding[compiled];
+	printf("%lx %lx\n",block1,block2);
+	assert(block1 == block2);
+	return block2;
 }
 
 struct callframe_forwarder {

vm/debug.cpp

@@ -290,8 +290,9 @@ void factor_vm::dump_code_heap()
 	cell reloc_size = 0, literal_size = 0;
 
 	heap_block *scan = code->first_block();
+	heap_block *end = code->last_block();
 
-	while(scan)
+	while(scan != end)
 	{
 		const char *status;
 		if(scan->type() == FREE_BLOCK_TYPE)
@@ -313,7 +314,7 @@ void factor_vm::dump_code_heap()
 		print_cell_hex(scan->size()); print_string(" ");
 		print_string(status); print_string("\n");
 
-		scan = code->next_block(scan);
+		scan = scan->next();
 	}
 	
 	print_cell(reloc_size); print_string(" bytes of relocation data\n");

vm/heap.cpp

@@ -141,7 +141,7 @@ void heap::heap_free(heap_block *block)
 
 void heap::mark_block(heap_block *block)
 {
-	state->set_marked_p(block,true);
+	state->set_marked_p(block);
 }
 
 /* Compute total sum of sizes of free blocks, and size of largest free block */
@@ -152,8 +152,9 @@ void heap::heap_usage(cell *used, cell *total_free, cell *max_free)
 	*max_free = 0;
 
 	heap_block *scan = first_block();
+	heap_block *end = last_block();
 
-	while(scan)
+	while(scan != end)
 	{
 		cell size = scan->size();
 
@@ -166,7 +167,7 @@ void heap::heap_usage(cell *used, cell *total_free, cell *max_free)
 		else
 			*used += size;
 
-		scan = next_block(scan);
+		scan = scan->next();
 	}
 }
 
@@ -174,11 +175,12 @@ void heap::heap_usage(cell *used, cell *total_free, cell *max_free)
 cell heap::heap_size()
 {
 	heap_block *scan = first_block();
+	heap_block *end = last_block();
 	
-	while(scan)
+	while(scan != end)
 	{
 		if(scan->type() == FREE_BLOCK_TYPE) break;
-		else scan = next_block(scan);
+		else scan = scan->next();
 	}
 
 	assert(scan->type() == FREE_BLOCK_TYPE);
@@ -190,14 +192,17 @@ cell heap::heap_size()
 void heap::compact_heap()
 {
 	forwarding.clear();
+	state->compute_forwarding();
 
 	heap_block *scan = first_block();
+	heap_block *end = last_block();
+
 	char *address = (char *)scan;
 
 	/* Slide blocks up while building the forwarding hashtable. */
-	while(scan)
+	while(scan != end)
 	{
-		heap_block *next = next_block(scan);
+		heap_block *next = scan->next();
  
 		if(state->is_marked_p(scan))
 		{

vm/heap.hpp

@@ -18,15 +18,6 @@ struct heap {
 
 	explicit heap(bool secure_gc_, cell size, bool executable_p);
 	~heap();
-
-	inline heap_block *next_block(heap_block *block)
-	{
-		cell next = ((cell)block + block->size());
-		if(next == seg->end)
-			return NULL;
-		else
-			return (heap_block *)next;
-	}
 	
 	inline heap_block *first_block()
 	{
@@ -62,8 +53,9 @@ struct heap {
 	
 		heap_block *prev = NULL;
 		heap_block *scan = first_block();
+		heap_block *end = last_block();
 	
-		while(scan)
+		while(scan != end)
 		{
 			if(scan->type() == FREE_BLOCK_TYPE)
 			{
@@ -82,7 +74,7 @@ struct heap {
 			else
 				prev = free_allocated(prev,scan);
 
-			scan = next_block(scan);
+			scan = scan->next();
 		}
 
 		if(prev && prev->type() == FREE_BLOCK_TYPE)

vm/layouts.hpp

@@ -212,6 +212,11 @@ struct heap_block
 	{
 		header = (header & 0x2f) | (size << 6);
 	}
+
+	inline heap_block *next()
+	{
+		return (heap_block *)((cell)this + size());
+	}
 };
 
 struct free_heap_block : public heap_block

vm/mark_bits.hpp

@@ -1,43 +1,41 @@
 namespace factor
 {
 
-const int forwarding_granularity = 128;
+const int forwarding_granularity = 64;
 
 template<typename Block, int Granularity> struct mark_bits {
 	cell start;
 	cell size;
 	cell bits_size;
-	unsigned int *marked;
+	u64 *marked;
-	unsigned int *freed;
+	u64 *allocated;
-	cell forwarding_size;
 	cell *forwarding;
 
 	void clear_mark_bits()
 	{
-		memset(marked,0,bits_size * sizeof(unsigned int));
+		memset(marked,0,bits_size * sizeof(u64));
 	}
 
-	void clear_free_bits()
+	void clear_allocated_bits()
 	{
-		memset(freed,0,bits_size * sizeof(unsigned int));
+		memset(allocated,0,bits_size * sizeof(u64));
 	}
 
 	void clear_forwarding()
 	{
-		memset(forwarding,0,forwarding_size * sizeof(cell));
+		memset(forwarding,0,bits_size * sizeof(cell));
 	}
 
 	explicit mark_bits(cell start_, cell size_) :
 		start(start_),
 		size(size_),
-		bits_size(size / Granularity / 32),
+		bits_size(size / Granularity / forwarding_granularity),
-		marked(new unsigned int[bits_size]),
+		marked(new u64[bits_size]),
-		freed(new unsigned int[bits_size]),
+		allocated(new u64[bits_size]),
-		forwarding_size(size / Granularity / forwarding_granularity),
+		forwarding(new cell[bits_size])
-		forwarding(new cell[forwarding_size])
 	{
 		clear_mark_bits();
-		clear_free_bits();
+		clear_allocated_bits();
 		clear_forwarding();
 	}
 
@@ -45,17 +43,27 @@ template<typename Block, int Granularity> struct mark_bits {
 	{
 		delete[] marked;
 		marked = NULL;
-		delete[] freed;
+		delete[] allocated;
-		freed = NULL;
+		allocated = NULL;
 		delete[] forwarding;
 		forwarding = NULL;
 	}
 
+	cell block_line(Block *address)
+	{
+		return (((cell)address - start) / Granularity);
+	}
+
+	Block *line_block(cell line)
+	{
+		return (Block *)(line * Granularity + start);
+	}
+
 	std::pair<cell,cell> bitmap_deref(Block *address)
 	{
-		cell word_number = (((cell)address - start) / Granularity);
+		cell line_number = block_line(address);
-		cell word_index = (word_number >> 5);
+		cell word_index = (line_number >> 6);
-		cell word_shift = (word_number & 31);
+		cell word_shift = (line_number & 63);
 
 #ifdef FACTOR_DEBUG
 		assert(word_index < bits_size);
@@ -64,19 +72,34 @@ template<typename Block, int Granularity> struct mark_bits {
 		return std::make_pair(word_index,word_shift);
 	}
 
-	bool bitmap_elt(unsigned int *bits, Block *address)
+	bool bitmap_elt(u64 *bits, Block *address)
 	{
 		std::pair<cell,cell> pair = bitmap_deref(address);
-		return (bits[pair.first] & (1 << pair.second)) != 0;
+		return (bits[pair.first] & ((u64)1 << pair.second)) != 0;
 	}
 
-	void set_bitmap_elt(unsigned int *bits, Block *address, bool flag)
+	void set_bitmap_range(u64 *bits, Block *address)
 	{
-		std::pair<cell,cell> pair = bitmap_deref(address);
+		std::pair<cell,cell> start = bitmap_deref(address);
-		if(flag)
+		std::pair<cell,cell> end = bitmap_deref(address->next());
-			bits[pair.first] |= (1 << pair.second);
+
+		u64 start_mask = ((u64)1 << start.second) - 1;
+		u64 end_mask = ((u64)1 << end.second) - 1;
+
+		if(start.first == end.first)
+			bits[start.first] |= start_mask ^ end_mask;
 		else
-			bits[pair.first] &= ~(1 << pair.second);
+		{
+			bits[start.first] |= ~start_mask;
+
+			if(end.first != 0)
+			{
+				for(cell index = start.first + 1; index < end.first - 1; index++)
+					bits[index] = (u64)-1;
+			}
+
+			bits[end.first] |= end_mask;
+		}
 	}
 
 	bool is_marked_p(Block *address)
@@ -84,19 +107,71 @@ template<typename Block, int Granularity> struct mark_bits {
 		return bitmap_elt(marked,address);
 	}
 
-	void set_marked_p(Block *address, bool marked_p)
+	void set_marked_p(Block *address)
 	{
-		set_bitmap_elt(marked,address,marked_p);
+		set_bitmap_range(marked,address);
 	}
 
-	bool is_free_p(Block *address)
+	bool is_allocated_p(Block *address)
 	{
-		return bitmap_elt(freed,address);
+		return bitmap_elt(allocated,address);
 	}
 
-	void set_free_p(Block *address, bool free_p)
+	void set_allocated_p(Block *address)
 	{
-		set_bitmap_elt(freed,address,free_p);
+		set_bitmap_range(allocated,address);
+	}
+
+	cell popcount1(u64 x)
+	{
+		cell accum = 0;
+		while(x > 0)
+		{
+			accum += (x & 1);
+			x >>= 1;
+		}
+		return accum;
+	}
+
+	/* From http://chessprogramming.wikispaces.com/Population+Count */
+	cell popcount(u64 x)
+	{
+		cell old = x;
+		u64 k1 = 0x5555555555555555ll;
+		u64 k2 = 0x3333333333333333ll;
+		u64 k4 = 0x0f0f0f0f0f0f0f0fll;
+		u64 kf = 0x0101010101010101ll;
+		x =  x       - ((x >> 1)  & k1); // put count of each 2 bits into those 2 bits
+		x = (x & k2) + ((x >> 2)  & k2); // put count of each 4 bits into those 4 bits
+		x = (x       +  (x >> 4)) & k4 ; // put count of each 8 bits into those 8 bits
+		x = (x * kf) >> 56; // returns 8 most significant bits of x + (x<<8) + (x<<16) + (x<<24) + ...
+		
+		assert(x == popcount1(old));
+		return (cell)x;
+	}
+
+	/* The eventual destination of a block after compaction is just the number
+	of marked blocks before it. */
+	void compute_forwarding()
+	{
+		cell accum = 0;
+		for(cell index = 0; index < bits_size; index++)
+		{
+			forwarding[index] = accum;
+			accum += popcount(marked[index]);
+		}
+	}
+
+	/* We have the popcount for every 64 entries; look up and compute the rest */
+	Block *forward_block(Block *original)
+	{
+		std::pair<cell,cell> pair = bitmap_deref(original);
+
+		cell approx_popcount = forwarding[pair.first];
+		u64 mask = ((u64)1 << pair.second) - 1;
+
+		cell new_line_number = approx_popcount + popcount(marked[pair.first] & mask);
+		return line_block(new_line_number);
 	}
 };
 

vm/vm.hpp

@@ -525,12 +525,13 @@ struct factor_vm
 	template<typename Iterator> void iterate_code_heap(Iterator &iter)
 	{
 		heap_block *scan = code->first_block();
+		heap_block *end = code->last_block();
 
-		while(scan)
+		while(scan != end)
 		{
 			if(scan->type() != FREE_BLOCK_TYPE)
 				iter((code_block *)scan);
-			scan = code->next_block(scan);
+			scan = scan->next();
 		}
 	}