namespace factor { // Size sans alignment. template cell object::base_size(Fixup fixup) const { switch (type()) { case ARRAY_TYPE: return array_size((array*)this); case BIGNUM_TYPE: return array_size((bignum*)this); case BYTE_ARRAY_TYPE: return array_size((byte_array*)this); case STRING_TYPE: return string_size(string_capacity((string*)this)); case TUPLE_TYPE: { tuple_layout* layout = (tuple_layout*)fixup.translate_data( untag(((tuple*)this)->layout)); return tuple_size(layout); } case QUOTATION_TYPE: return sizeof(quotation); case WORD_TYPE: return sizeof(word); case FLOAT_TYPE: return sizeof(boxed_float); case DLL_TYPE: return sizeof(dll); case ALIEN_TYPE: return sizeof(alien); case WRAPPER_TYPE: return sizeof(wrapper); case CALLSTACK_TYPE: { cell callstack_length = untag_fixnum(((callstack*)this)->length); return callstack_object_size(callstack_length); } default: critical_error("Invalid header in base_size", (cell)this); return 0; } } // Size of the object pointed to by an untagged pointer template cell object::size(Fixup fixup) const { if (free_p()) return ((free_heap_block*)this)->size(); return align(base_size(fixup), data_alignment); } inline cell object::size() const { return size(no_fixup()); } // The number of slots (cells) in an object which should be scanned by // the GC. The number can vary in arrays and tuples, in all other // types the number is a constant. template inline cell object::slot_count(Fixup fixup) const { if (free_p()) return 0; cell t = type(); if (t == ARRAY_TYPE) { // capacity + n slots return 1 + array_capacity((array*)this); } else if (t == TUPLE_TYPE) { tuple_layout* layout = (tuple_layout*)fixup.translate_data( untag(((tuple*)this)->layout)); // layout + n slots return 1 + tuple_capacity(layout); } else { switch (t) { // these objects do not refer to other objects at all case FLOAT_TYPE: case BYTE_ARRAY_TYPE: case BIGNUM_TYPE: case CALLSTACK_TYPE: return 0; case WORD_TYPE: return 8; case ALIEN_TYPE: return 2; case DLL_TYPE: return 1; case QUOTATION_TYPE: return 3; case STRING_TYPE: return 3; case WRAPPER_TYPE: return 1; default: critical_error("Invalid header in slot_count", (cell)this); return 0; // can't happen } } } inline cell object::slot_count() const { return slot_count(no_fixup()); } // Slot visitors iterate over the slots of an object, applying a functor to // each one that is a non-immediate slot. The pointer is untagged first. // The functor returns a new untagged object pointer. The return value may // or may not equal the old one, however the new pointer receives the same // tag before being stored back to the original location. // Slots storing immediate values are left unchanged and the visitor does // inspect them. // This is used by GC's copying, sweep and compact phases, and the // implementation of the become primitive. // Iteration is driven by visit_*() methods. Only one of them define GC // roots: // - visit_all_roots() // Code block visitors iterate over sets of code blocks, applying a functor // to each one. The functor returns a new code_block pointer, which may or // may not equal the old one. This is stored back to the original location. // This is used by GC's sweep and compact phases, and the implementation of // the modify-code-heap primitive. // Iteration is driven by visit_*() methods. Some of them define GC roots: // - visit_context_code_blocks() // - visit_callback_code_blocks() template struct slot_visitor { factor_vm* parent; Fixup fixup; cell cards_scanned; cell decks_scanned; slot_visitor(factor_vm* parent, Fixup fixup) : parent(parent), fixup(fixup), cards_scanned(0), decks_scanned(0) {} cell visit_pointer(cell pointer); void visit_handle(cell* handle); void visit_object_array(cell* start, cell* end); void visit_partial_objects(cell start, cell card_start, cell card_end); void visit_slots(object* ptr); void visit_stack_elements(segment* region, cell* top); void visit_all_roots(); void visit_callstack_object(callstack* stack); void visit_callstack(context* ctx); void visit_context(context *ctx); void visit_code_block_objects(code_block* compiled); void visit_embedded_literals(code_block* compiled); void visit_object_code_block(object* obj); void visit_context_code_blocks(); void visit_uninitialized_code_blocks(); void visit_embedded_code_pointers(code_block* compiled); void visit_object(object* obj); void visit_mark_stack(std::vector* mark_stack); void visit_instruction_operands(code_block* block, cell rel_base); template cell visit_card(SourceGeneration* gen, cell index, cell start); template void visit_cards(SourceGeneration* gen, card mask, card unmask); void visit_code_heap_roots(std::set* remembered_set); template void cheneys_algorithm(TargetGeneration* gen, cell scan); }; template cell slot_visitor::visit_pointer(cell pointer) { object* untagged = fixup.fixup_data(untag(pointer)); return RETAG(untagged, TAG(pointer)); } template void slot_visitor::visit_handle(cell* handle) { if (!immediate_p(*handle)) { *handle = visit_pointer(*handle); } } template void slot_visitor::visit_object_array(cell* start, cell* end) { while (start < end) visit_handle(start++); } template void slot_visitor::visit_slots(object* obj) { if (obj->type() == CALLSTACK_TYPE) visit_callstack_object((callstack*)obj); else { cell* start = (cell*)obj + 1; cell* end = start + obj->slot_count(fixup); visit_object_array(start, end); } } template void slot_visitor::visit_stack_elements(segment* region, cell* top) { visit_object_array((cell*)region->start, top + 1); } template void slot_visitor::visit_all_roots() { FACTOR_FOR_EACH(parent->data_roots) { visit_handle(*iter); } auto callback_slot_visitor = [&](code_block* stub, cell size) { visit_handle(&stub->owner); }; parent->callbacks->allocator->iterate(callback_slot_visitor); FACTOR_FOR_EACH(parent->code->uninitialized_blocks) { iter->second = visit_pointer(iter->second); } FACTOR_FOR_EACH(parent->sample_callstacks) { visit_handle(&*iter); } FACTOR_FOR_EACH(parent->samples) { visit_handle(&iter->thread); } visit_object_array(parent->special_objects, parent->special_objects + special_object_count); FACTOR_FOR_EACH(parent->active_contexts) { visit_context(*iter); } } // primitive_minor_gc() is invoked by inline GC checks, and it needs to // visit spill slots which references objects in the heap. // So for each call frame: // - trace roots in spill slots template struct call_frame_slot_visitor { slot_visitor* visitor; call_frame_slot_visitor(slot_visitor* visitor) : visitor(visitor) {} // frame top -> [return address] // [spill area] // ... // [entry_point] // [size] void operator()(cell frame_top, cell size, code_block* owner, cell addr) { cell return_address = owner->offset(addr); code_block* compiled = Fixup::translated_code_block_map ? owner : visitor->fixup.translate_code(owner); gc_info* info = compiled->block_gc_info(); FACTOR_ASSERT(return_address < compiled->size()); cell callsite = info->return_address_index(return_address); if (callsite == (cell)-1) return; #ifdef DEBUG_GC_MAPS FACTOR_PRINT("call frame code block " << compiled << " with offset " << return_address); #endif cell* stack_pointer = (cell*)frame_top; uint8_t* bitmap = info->gc_info_bitmap(); // Subtract old value of base pointer from every derived pointer. for (cell spill_slot = 0; spill_slot < info->derived_root_count; spill_slot++) { uint32_t base_pointer = info->lookup_base_pointer(callsite, spill_slot); if (base_pointer != (uint32_t)-1) { #ifdef DEBUG_GC_MAPS FACTOR_PRINT("visiting derived root " << spill_slot << " with base pointer " << base_pointer); #endif stack_pointer[spill_slot] -= stack_pointer[base_pointer]; } } // Update all GC roots, including base pointers. cell callsite_gc_roots = info->callsite_gc_roots(callsite); for (cell spill_slot = 0; spill_slot < info->gc_root_count; spill_slot++) { if (bitmap_p(bitmap, callsite_gc_roots + spill_slot)) { #ifdef DEBUG_GC_MAPS FACTOR_PRINT("visiting GC root " << spill_slot); #endif visitor->visit_handle(stack_pointer + spill_slot); } } // Add the base pointers to obtain new derived pointer values. for (cell spill_slot = 0; spill_slot < info->derived_root_count; spill_slot++) { uint32_t base_pointer = info->lookup_base_pointer(callsite, spill_slot); if (base_pointer != (uint32_t)-1) stack_pointer[spill_slot] += stack_pointer[base_pointer]; } } }; template void slot_visitor::visit_callstack_object(callstack* stack) { call_frame_slot_visitor call_frame_visitor(this); parent->iterate_callstack_object(stack, call_frame_visitor, fixup); } template void slot_visitor::visit_callstack(context* ctx) { call_frame_slot_visitor call_frame_visitor(this); parent->iterate_callstack(ctx, call_frame_visitor, fixup); } template void slot_visitor::visit_context(context* ctx) { // Callstack is visited first because it scrubs the data and retain // stacks. visit_callstack(ctx); cell ds_ptr = ctx->datastack; cell rs_ptr = ctx->retainstack; segment* ds_seg = ctx->datastack_seg; segment* rs_seg = ctx->retainstack_seg; visit_stack_elements(ds_seg, (cell*)ds_ptr); visit_stack_elements(rs_seg, (cell*)rs_ptr); visit_object_array(ctx->context_objects, ctx->context_objects + context_object_count); // Clear out the space not visited with a known pattern. That makes // it easier to see if uninitialized reads are made. ctx->fill_stack_seg(ds_ptr, ds_seg, 0xbaadbadd); ctx->fill_stack_seg(rs_ptr, rs_seg, 0xdaabdaab); } template void slot_visitor::visit_code_block_objects(code_block* compiled) { visit_handle(&compiled->owner); visit_handle(&compiled->parameters); visit_handle(&compiled->relocation); } template void slot_visitor::visit_embedded_literals(code_block* compiled) { if (parent->code->uninitialized_p(compiled)) return; auto update_literal_refs = [&](instruction_operand op) { if (op.rel.type() == RT_LITERAL) { cell value = op.load_value(op.pointer); if (!immediate_p(value)) { op.store_value(visit_pointer(value)); } } }; compiled->each_instruction_operand(update_literal_refs); } template struct call_frame_code_block_visitor { Fixup fixup; call_frame_code_block_visitor(Fixup fixup) : fixup(fixup) {} void operator()(cell frame_top, cell size, code_block* owner, cell addr) { code_block* compiled = Fixup::translated_code_block_map ? owner : fixup.fixup_code(owner); cell fixed_addr = compiled->address_for_offset(owner->offset(addr)); *(cell*)frame_top = fixed_addr; } }; template void slot_visitor::visit_object_code_block(object* obj) { switch (obj->type()) { case WORD_TYPE: { word* w = (word*)obj; if (w->entry_point) w->entry_point = fixup.fixup_code(w->code())->entry_point(); break; } case QUOTATION_TYPE: { quotation* q = (quotation*)obj; if (q->entry_point) q->entry_point = fixup.fixup_code(q->code())->entry_point(); break; } case CALLSTACK_TYPE: { callstack* stack = (callstack*)obj; call_frame_code_block_visitor call_frame_visitor(fixup); parent->iterate_callstack_object(stack, call_frame_visitor, fixup); break; } } } template void slot_visitor::visit_context_code_blocks() { call_frame_code_block_visitor call_frame_visitor(fixup); FACTOR_FOR_EACH(parent->active_contexts) { parent->iterate_callstack(*iter, call_frame_visitor, fixup); } } template void slot_visitor::visit_uninitialized_code_blocks() { std::map new_uninitialized_blocks; FACTOR_FOR_EACH(parent->code->uninitialized_blocks) { new_uninitialized_blocks.insert( std::make_pair(fixup.fixup_code(iter->first), iter->second)); } parent->code->uninitialized_blocks = new_uninitialized_blocks; } template void slot_visitor::visit_embedded_code_pointers(code_block* compiled) { if (parent->code->uninitialized_p(compiled)) return; auto update_code_block_refs = [&](instruction_operand op){ relocation_type type = op.rel.type(); if (type == RT_ENTRY_POINT || type == RT_ENTRY_POINT_PIC || type == RT_ENTRY_POINT_PIC_TAIL) { code_block* block = fixup.fixup_code(op.load_code_block()); op.store_value(block->entry_point()); } }; compiled->each_instruction_operand(update_code_block_refs); } template void slot_visitor::visit_object(object *ptr) { visit_slots(ptr); if (ptr->type() == ALIEN_TYPE) ((alien*)ptr)->update_address(); } // Pops items from the mark stack and visits them until the stack is // empty. Used when doing a full collection and when collecting to // tenured space. template void slot_visitor::visit_mark_stack(std::vector* mark_stack) { while (!mark_stack->empty()) { cell ptr = mark_stack->back(); mark_stack->pop_back(); if (ptr & 1) { code_block* compiled = (code_block*)(ptr - 1); visit_code_block_objects(compiled); visit_embedded_literals(compiled); visit_embedded_code_pointers(compiled); } else { object* obj = (object*)ptr; visit_object(obj); visit_object_code_block(obj); } } } // Visits the instruction operands in a code block. If the operand is // a pointer to a code block or data object, then the fixup is applied // to it. Otherwise, if it is an external addess, that address is // recomputed. If it is an untagged number literal (RT_UNTAGGED) or an // immediate value, then nothing is done with it. template void slot_visitor::visit_instruction_operands(code_block* block, cell rel_base) { auto visit_func = [&](instruction_operand op){ cell old_offset = rel_base + op.rel.offset(); cell old_value = op.load_value(old_offset); switch (op.rel.type()) { case RT_LITERAL: { if (!immediate_p(old_value)) { op.store_value(visit_pointer(old_value)); } break; } case RT_ENTRY_POINT: case RT_ENTRY_POINT_PIC: case RT_ENTRY_POINT_PIC_TAIL: case RT_HERE: { cell offset = TAG(old_value); code_block* compiled = (code_block*)UNTAG(old_value); op.store_value(RETAG(fixup.fixup_code(compiled), offset)); break; } case RT_UNTAGGED: break; default: op.store_value(parent->compute_external_address(op)); break; } }; block->each_instruction_operand(visit_func); } template void slot_visitor::visit_partial_objects(cell start, cell card_start, cell card_end) { cell *scan_start = (cell*)start + 1; cell *scan_end = scan_start + ((object*)start)->slot_count(); scan_start = std::max(scan_start, (cell*)card_start); scan_end = std::min(scan_end, (cell*)card_end); visit_object_array(scan_start, scan_end); } template template cell slot_visitor::visit_card(SourceGeneration* gen, cell index, cell start) { cell heap_base = parent->data->start; cell start_addr = heap_base + index * card_size; cell end_addr = start_addr + card_size; // Forward to the next object whose address is in the card. if (!start || (start + ((object*)start)->size()) < start_addr) { // Optimization because finding the objects in a memory range is // expensive. It helps a lot when tracing consecutive cards. cell gen_start_card = (gen->start - heap_base) / card_size; start = gen->starts .find_object_containing_card(index - gen_start_card); } while (start && start < end_addr) { visit_partial_objects(start, start_addr, end_addr); if ((start + ((object*)start)->size()) >= end_addr) { // The object can overlap the card boundary, then the // remainder of it will be handled in the next card // tracing if that card is marked. break; } start = gen->next_object_after(start); } return start; } template template void slot_visitor::visit_cards(SourceGeneration* gen, card mask, card unmask) { card_deck* decks = parent->data->decks; card_deck* cards = parent->data->cards; cell heap_base = parent->data->start; cell first_deck = (gen->start - heap_base) / deck_size; cell last_deck = (gen->end - heap_base) / deck_size; // Address of last traced object. cell start = 0; for (cell di = first_deck; di < last_deck; di++) { if (decks[di] & mask) { decks[di] &= ~unmask; decks_scanned++; cell first_card = cards_per_deck * di; cell last_card = first_card + cards_per_deck; for (cell ci = first_card; ci < last_card; ci++) { if (cards[ci] & mask) { cards[ci] &= ~unmask; cards_scanned++; start = visit_card(gen, ci, start); if (!start) { // At end of generation, no need to scan more cards. return; } } } } } } template void slot_visitor::visit_code_heap_roots(std::set* remembered_set) { FACTOR_FOR_EACH(*remembered_set) { code_block* compiled = *iter; visit_code_block_objects(compiled); visit_embedded_literals(compiled); compiled->flush_icache(); } } template template void slot_visitor::cheneys_algorithm(TargetGeneration* gen, cell scan) { while (scan && scan < gen->here) { visit_object((object*)scan); scan = gen->next_object_after(scan); } } }