vm: change code heap layout somewhat, remove unused allocation bitmap from mark_bits

vm/code_block.cpp

@@ -346,7 +346,7 @@ void factor_vm::update_word_references(code_block *compiled)
 	   are referenced after this is done. So instead of polluting
 	   the code heap with dead PICs that will be freed on the next
 	   GC, we add them to the free list immediately. */
-	else if(compiled->type() == PIC_TYPE)
+	else if(compiled->pic_p())
 		code->code_heap_free(compiled);
 	else
 	{
@@ -437,9 +437,9 @@ void factor_vm::fixup_labels(array *labels, code_block *compiled)
 }
 
 /* Might GC */
-code_block *factor_vm::allot_code_block(cell size, cell type)
+code_block *factor_vm::allot_code_block(cell size, code_block_type type)
 {
-	heap_block *block = code->heap_allot(size + sizeof(code_block),type);
+	heap_block *block = code->heap_allot(size + sizeof(code_block));
 
 	/* If allocation failed, do a full GC and compact the code heap.
 	A full GC that occurs as a result of the data heap filling up does not
@@ -449,7 +449,7 @@ code_block *factor_vm::allot_code_block(cell size, cell type)
 	if(block == NULL)
 	{
 		primitive_compact_gc();
-		block = code->heap_allot(size + sizeof(code_block),type);
+		block = code->heap_allot(size + sizeof(code_block));
 
 		/* Insufficient room even after code GC, give up */
 		if(block == NULL)
@@ -465,11 +465,13 @@ code_block *factor_vm::allot_code_block(cell size, cell type)
 		}
 	}
 
-	return (code_block *)block;
+	code_block *compiled = (code_block *)block;
+	compiled->set_type(type);
+	return compiled;
 }
 
 /* Might GC */
-code_block *factor_vm::add_code_block(cell type, cell code_, cell labels_, cell owner_, cell relocation_, cell literals_)
+code_block *factor_vm::add_code_block(code_block_type type, cell code_, cell labels_, cell owner_, cell relocation_, cell literals_)
 {
 	gc_root<byte_array> code(code_,this);
 	gc_root<object> labels(labels_,this);

vm/code_heap.cpp

@@ -144,7 +144,7 @@ void factor_vm::primitive_modify_code_heap()
 				cell code = array_nth(compiled_data,4);
 
 				code_block *compiled = add_code_block(
-					WORD_TYPE,
+					code_block_optimized,
 					code,
 					labels,
 					owner,

vm/debug.cpp

@@ -295,7 +295,7 @@ void factor_vm::dump_code_heap()
 	while(scan != end)
 	{
 		const char *status;
-		if(scan->type() == FREE_BLOCK_TYPE)
+		if(scan->free_p())
 			status = "free";
 		else if(code->state->is_marked_p(scan))
 		{

vm/heap.cpp

@@ -49,15 +49,16 @@ void heap::build_free_list(cell size)
 {
 	clear_free_list();
 	free_heap_block *end = (free_heap_block *)(seg->start + size);
-	end->set_type(FREE_BLOCK_TYPE);
+	end->set_free();
 	end->set_size(seg->end - (cell)end);
 	add_to_free_list(end);
 }
 
 void heap::assert_free_block(free_heap_block *block)
 {
-	if(block->type() != FREE_BLOCK_TYPE)
-		critical_error("Invalid block in free list",(cell)block);
+#ifdef FACTOR_DEBUG
+	assert(block->free_p());
+#endif
 }
 
 free_heap_block *heap::find_free_block(cell size)
@@ -102,11 +103,11 @@ free_heap_block *heap::find_free_block(cell size)
 
 free_heap_block *heap::split_free_block(free_heap_block *block, cell size)
 {
-	if(block->size() != size )
+	if(block->size() != size)
 	{
 		/* split the block in two */
 		free_heap_block *split = (free_heap_block *)((cell)block + size);
-		split->set_type(FREE_BLOCK_TYPE);
+		split->set_free();
 		split->set_size(block->size() - size);
 		split->next_free = block->next_free;
 		block->set_size(size);
@@ -116,27 +117,25 @@ free_heap_block *heap::split_free_block(free_heap_block *block, cell size)
 	return block;
 }
 
-/* Allocate a block of memory from the mark and sweep GC heap */
-heap_block *heap::heap_allot(cell size, cell type)
+heap_block *heap::heap_allot(cell size)
 {
-	size = (size + block_size_increment - 1) & ~(block_size_increment - 1);
+	size = align(size,block_size_increment);
 
 	free_heap_block *block = find_free_block(size);
 	if(block)
 	{
 		block = split_free_block(block,size);
-		block->set_type(type);
 		return block;
 	}
 	else
 		return NULL;
 }
 
-/* Deallocates a block manually */
 void heap::heap_free(heap_block *block)
 {
-	block->set_type(FREE_BLOCK_TYPE);
-	add_to_free_list((free_heap_block *)block);
+	free_heap_block *free_block = (free_heap_block *)block;
+	free_block->set_free();
+	add_to_free_list(free_block);
 }
 
 void heap::mark_block(heap_block *block)
@@ -158,7 +157,7 @@ void heap::heap_usage(cell *used, cell *total_free, cell *max_free)
 	{
 		cell size = scan->size();
 
-		if(scan->type() == FREE_BLOCK_TYPE)
+		if(scan->free_p())
 		{
 			*total_free += size;
 			if(size > *max_free)
@@ -179,31 +178,19 @@ cell heap::heap_size()
 	
 	while(scan != end)
 	{
-		if(scan->type() == FREE_BLOCK_TYPE) break;
+		if(scan->free_p()) break;
 		else scan = scan->next();
 	}
 
-	assert(scan->type() == FREE_BLOCK_TYPE);
-	assert((cell)scan + scan->size() == seg->end);
-
-	return (cell)scan - (cell)first_block();
-}
-
-heap_block *heap::free_allocated(heap_block *prev, heap_block *scan)
-{
-	if(secure_gc)
-		memset(scan + 1,0,scan->size() - sizeof(heap_block));
-
-	if(prev && prev->type() == FREE_BLOCK_TYPE)
+	if(scan != end)
 	{
-		prev->set_size(prev->size() + scan->size());
-		return prev;
+		assert(scan->free_p());
+		assert((cell)scan + scan->size() == seg->end);
+
+		return (cell)scan - (cell)first_block();
 	}
 	else
-	{
-		scan->set_type(FREE_BLOCK_TYPE);
-		return scan;
-	}
+		return seg->size;
 }
 
 }

vm/heap.hpp

@@ -34,15 +34,13 @@ struct heap {
 	void assert_free_block(free_heap_block *block);
 	free_heap_block *find_free_block(cell size);
 	free_heap_block *split_free_block(free_heap_block *block, cell size);
-	heap_block *heap_allot(cell size, cell type);
+	heap_block *heap_allot(cell size);
 	void heap_free(heap_block *block);
 	void mark_block(heap_block *block);
 	void heap_usage(cell *used, cell *total_free, cell *max_free);
 	cell heap_size();
 	void compact_heap();
 
-	heap_block *free_allocated(heap_block *prev, heap_block *scan);
-
 	template<typename Iterator> void sweep_heap(Iterator &iter);
 	template<typename Iterator> void compact_heap(Iterator &iter);
 
@@ -54,7 +52,7 @@ struct heap {
 		while(scan != end)
 		{
 			heap_block *next = scan->next();
-			if(scan->type() != FREE_BLOCK_TYPE) iter(scan,scan->size());
+			if(!scan->free_p()) iter(scan,scan->size());
 			scan = next;
 		}
 	}
@@ -72,27 +70,41 @@ template<typename Iterator> void heap::sweep_heap(Iterator &iter)
 
 	while(scan != end)
 	{
-		if(scan->type() == FREE_BLOCK_TYPE)
+		if(scan->free_p())
 		{
-			if(prev && prev->type() == FREE_BLOCK_TYPE)
+			if(prev && prev->free_p())
 				prev->set_size(prev->size() + scan->size());
 			else
 				prev = scan;
 		}
 		else if(this->state->is_marked_p(scan))
 		{
-			if(prev && prev->type() == FREE_BLOCK_TYPE)
+			if(prev && prev->free_p())
 				this->add_to_free_list((free_heap_block *)prev);
 			prev = scan;
 			iter(scan,scan->size());
 		}
 		else
-			prev = this->free_allocated(prev,scan);
+		{
+			if(secure_gc)
+				memset(scan + 1,0,scan->size() - sizeof(heap_block));
+
+			if(prev && prev->free_p())
+			{
+				free_heap_block *free_prev = (free_heap_block *)prev;
+				free_prev->set_size(free_prev->size() + scan->size());
+			}
+			else
+			{
+				scan->set_free();
+				prev = scan;
+			}
+		}
 
 		scan = scan->next();
 	}
 
-	if(prev && prev->type() == FREE_BLOCK_TYPE)
+	if(prev && prev->free_p())
 		this->add_to_free_list((free_heap_block *)prev);
 }
 

vm/inline_cache.cpp

@@ -19,15 +19,9 @@ void factor_vm::deallocate_inline_cache(cell return_address)
 	check_code_pointer((cell)old_xt);
 
 	code_block *old_block = (code_block *)old_xt - 1;
-	cell old_type = old_block->type();
 
-#ifdef FACTOR_DEBUG
-	/* The call target was either another PIC,
-	   or a compiled quotation (megamorphic stub) */
-	assert(old_type == PIC_TYPE || old_type == QUOTATION_TYPE);
-#endif
-
-	if(old_type == PIC_TYPE)
+	/* Free the old PIC since we know its unreachable */
+	if(old_block->pic_p())
 		code->code_heap_free(old_block);
 }
 
@@ -78,7 +72,7 @@ void factor_vm::update_pic_count(cell type)
 struct inline_cache_jit : public jit {
 	fixnum index;
 
-	explicit inline_cache_jit(cell generic_word_,factor_vm *vm) : jit(PIC_TYPE,generic_word_,vm) {};
+	explicit inline_cache_jit(cell generic_word_,factor_vm *vm) : jit(code_block_pic,generic_word_,vm) {};
 
 	void emit_check(cell klass);
 	void compile_inline_cache(fixnum index,

vm/jit.cpp

@@ -10,7 +10,7 @@ namespace factor
 - polymorphic inline caches (inline_cache.cpp) */
 
 /* Allocates memory */
-jit::jit(cell type_, cell owner_, factor_vm *vm)
+jit::jit(code_block_type type_, cell owner_, factor_vm *vm)
 	: type(type_),
 	  owner(owner_,vm),
 	  code(vm),

vm/jit.hpp

@@ -2,7 +2,7 @@ namespace factor
 {
 
 struct jit {
-	cell type;
+	code_block_type type;
 	gc_root<object> owner;
 	growable_byte_array code;
 	growable_byte_array relocation;
@@ -12,7 +12,7 @@ struct jit {
 	cell offset;
 	factor_vm *parent;
 
-	explicit jit(cell jit_type, cell owner, factor_vm *vm);
+	explicit jit(code_block_type type, cell owner, factor_vm *parent);
 	void compute_position(cell offset);
 
 	void emit_relocation(cell code_template);

vm/layouts.hpp

@@ -62,8 +62,14 @@ inline static cell align8(cell a)
 #define TYPE_COUNT 15
 
 /* Not real types, but code_block's type can be set to this */
-#define PIC_TYPE 16
-#define FREE_BLOCK_TYPE 17
+
+enum code_block_type
+{
+	code_block_unoptimized,
+	code_block_optimized,
+	code_block_profiling,
+	code_block_pic
+};
 
 /* Constants used when floating-point trap exceptions are thrown */
 enum
@@ -201,16 +207,29 @@ struct heap_block
 {
 	cell header;
 
-	cell type() { return (header >> 1) & 0x1f; }
-	void set_type(cell type)
+	bool free_p()
 	{
-		header = ((header & ~(0x1f << 1)) | (type << 1));
+		return header & 1 == 1;
+	}
+
+	void set_free()
+	{
+		header |= 1;
+	}
+
+	void clear_free()
+	{
+		header &= ~1;
+	}
+
+	cell size()
+	{
+		return header >> 3;
 	}
 
-	cell size() { return (header >> 6); }
 	void set_size(cell size)
 	{
-		header = (header & 0x2f) | (size << 6);
+		header = (header & 0x7) | (size << 3);
 	}
 
 	inline heap_block *next()
@@ -230,7 +249,30 @@ struct code_block : public heap_block
 	cell literals; /* tagged pointer to array or f */
 	cell relocation; /* tagged pointer to byte-array or f */
 
-	void *xt() { return (void *)(this + 1); }
+	void *xt()
+	{
+		return (void *)(this + 1);
+	}
+
+	cell type()
+	{
+		return (header >> 1) & 0x3;
+	}
+
+	void set_type(code_block_type type)
+	{
+		header = ((header & ~0x7) | (type << 1));
+	}
+
+	bool pic_p()
+	{
+		return type() == code_block_pic;
+	}
+
+	bool optimized_p()
+	{
+		return type() == code_block_optimized;
+	}
 };
 
 /* Assembly code makes assumptions about the layout of this struct */

vm/mark_bits.hpp

@@ -8,7 +8,6 @@ template<typename Block, int Granularity> struct mark_bits {
 	cell size;
 	cell bits_size;
 	u64 *marked;
-	u64 *allocated;
 	cell *forwarding;
 
 	void clear_mark_bits()
@@ -16,11 +15,6 @@ template<typename Block, int Granularity> struct mark_bits {
 		memset(marked,0,bits_size * sizeof(u64));
 	}
 
-	void clear_allocated_bits()
-	{
-		memset(allocated,0,bits_size * sizeof(u64));
-	}
-
 	void clear_forwarding()
 	{
 		memset(forwarding,0,bits_size * sizeof(cell));
@@ -31,11 +25,9 @@ template<typename Block, int Granularity> struct mark_bits {
 		size(size_),
 		bits_size(size / Granularity / forwarding_granularity),
 		marked(new u64[bits_size]),
-		allocated(new u64[bits_size]),
 		forwarding(new cell[bits_size])
 	{
 		clear_mark_bits();
-		clear_allocated_bits();
 		clear_forwarding();
 	}
 
@@ -43,8 +35,6 @@ template<typename Block, int Granularity> struct mark_bits {
 	{
 		delete[] marked;
 		marked = NULL;
-		delete[] allocated;
-		allocated = NULL;
 		delete[] forwarding;
 		forwarding = NULL;
 	}
@@ -109,16 +99,6 @@ template<typename Block, int Granularity> struct mark_bits {
 		set_bitmap_range(marked,address);
 	}
 
-	bool is_allocated_p(Block *address)
-	{
-		return bitmap_elt(allocated,address);
-	}
-
-	void set_allocated_p(Block *address)
-	{
-		set_bitmap_range(allocated,address);
-	}
-
 	/* From http://chessprogramming.wikispaces.com/Population+Count */
 	cell popcount(u64 x)
 	{

vm/profiler.cpp

@@ -13,7 +13,7 @@ code_block *factor_vm::compile_profiling_stub(cell word_)
 {
 	gc_root<word> word(word_,this);
 
-	jit jit(WORD_TYPE,word.value(),this);
+	jit jit(code_block_profiling,word.value(),this);
 	jit.emit_with(userenv[JIT_PROFILING],word.value());
 
 	return jit.to_code_block();

vm/quotations.cpp

@@ -335,7 +335,7 @@ void factor_vm::compile_all_words()
 	{
 		gc_root<word> word(array_nth(words.untagged(),i),this);
 
-		if(!word->code || !word_optimized_p(word.untagged()))
+		if(!word->code || !word->code->optimized_p())
 			jit_compile_word(word.value(),word->def,false);
 
 		update_word_xt(word.value());

vm/quotations.hpp

@@ -6,7 +6,7 @@ struct quotation_jit : public jit {
 	bool compiling, relocate;
 
 	explicit quotation_jit(cell quot, bool compiling_, bool relocate_, factor_vm *vm)
-		: jit(QUOTATION_TYPE,quot,vm),
+		: jit(code_block_unoptimized,quot,vm),
 		  elements(owner.as<quotation>().untagged()->array,vm),
 		  compiling(compiling_),
 		  relocate(relocate_){};

vm/tagged.hpp

@@ -27,23 +27,34 @@ struct tagged
 			return tag;
 	}
 
-	bool type_p(cell type_) const { return type() == type_; }
+	bool type_p(cell type_) const
+	{
+		return type() == type_;
+	}
+
+	bool type_p() const
+	{
+		if(Type::type_number == TYPE_COUNT)
+			return true;
+		else
+			return type_p(Type::type_number);
+	}
 
 	Type *untag_check(factor_vm *parent) const {
-		if(Type::type_number != TYPE_COUNT && !type_p(Type::type_number))
+		if(!type_p())
 			parent->type_error(Type::type_number,value_);
 		return untagged();
 	}
 
 	explicit tagged(cell tagged) : value_(tagged) {
 #ifdef FACTOR_DEBUG
-		untag_check(tls_vm());
+		assert(type_p());
 #endif
 	}
 
 	explicit tagged(Type *untagged) : value_(factor::tag(untagged)) {
 #ifdef FACTOR_DEBUG
-		untag_check(tls_vm()); 
+		assert(type_p());
 #endif
 	}
 

vm/vm.hpp

@@ -498,8 +498,8 @@ struct factor_vm
 	void check_code_address(cell address);
 	void relocate_code_block(code_block *compiled);
 	void fixup_labels(array *labels, code_block *compiled);
-	code_block *allot_code_block(cell size, cell type);
-	code_block *add_code_block(cell type, cell code_, cell labels_, cell owner_, cell relocation_, cell literals_);
+	code_block *allot_code_block(cell size, code_block_type type);
+	code_block *add_code_block(code_block_type type, cell code_, cell labels_, cell owner_, cell relocation_, cell literals_);
 
 	//code heap
 	inline void check_code_pointer(cell ptr)

vm/words.cpp

@@ -82,7 +82,8 @@ void factor_vm::update_word_xt(cell w_)
 
 void factor_vm::primitive_optimized_p()
 {
-	drepl(tag_boolean(word_optimized_p(untag_check<word>(dpeek()))));
+	word *w = untag_check<word>(dpeek());
+	drepl(tag_boolean(w->code->optimized_p()));
 }
 
 void factor_vm::primitive_wrapper()

vm/words.hpp

@@ -1,9 +1,4 @@
 namespace factor
 {
 
-inline bool word_optimized_p(word *word)
-{
-	return word->code->type() == WORD_TYPE;
-}
-
 }