factor/vm/compaction.cpp

#include "master.hpp"

namespace factor {

struct compaction_fixup {
  static const bool translated_code_block_map = false;

  mark_bits* data_forwarding_map;
  mark_bits* code_forwarding_map;
  const object** data_finger;
  const code_block** code_finger;

  compaction_fixup(mark_bits* data_forwarding_map,
                   mark_bits* code_forwarding_map,
                   const object** data_finger,
                   const code_block** code_finger)
      : data_forwarding_map(data_forwarding_map),
        code_forwarding_map(code_forwarding_map),
        data_finger(data_finger),
        code_finger(code_finger) {}

  object* fixup_data(object* obj) {
    return (object*)data_forwarding_map->forward_block((cell)obj);
  }

  code_block* fixup_code(code_block* compiled) {
    return (code_block*)code_forwarding_map->forward_block((cell)compiled);
  }

  object* translate_data(const object* obj) {
    if (obj < *data_finger)
      return fixup_data((object*)obj);
    return (object*)obj;
  }

  code_block* translate_code(const code_block* compiled) {
    if (compiled < *code_finger)
      return fixup_code((code_block*)compiled);
    return (code_block*)compiled;
  }

  cell size(object* obj) {
    if (data_forwarding_map->marked_p((cell)obj))
      return obj->size(*this);
    return data_forwarding_map->unmarked_block_size((cell)obj);
  }

  cell size(code_block* compiled) {
    if (code_forwarding_map->marked_p((cell)compiled))
      return compiled->size(*this);
    return code_forwarding_map->unmarked_block_size((cell)compiled);
  }
};

/* After a compaction, invalidate any code heap roots which are not
marked, and also slide the valid roots up so that call sites can be updated
correctly in case an inline cache compilation triggered compaction. */
void factor_vm::update_code_roots_for_compaction() {

  mark_bits* state = &code->allocator->state;

  FACTOR_FOR_EACH(code_roots) {
    code_root* root = *iter;
    cell block = root->value & (~data_alignment + 1);

    /* Offset of return address within 16-byte allocation line */
    cell offset = root->value - block;

    if (root->valid && state->marked_p(block)) {
      block = state->forward_block(block);
      root->value = block + offset;
    } else
      root->valid = false;
  }
}

/* Compact data and code heaps */
void factor_vm::collect_compact_impl() {
  gc_event* event = current_gc->event;

#ifdef FACTOR_DEBUG
  code->verify_all_blocks_set();
#endif

  if (event)
    event->started_compaction();

  tenured_space* tenured = data->tenured;
  mark_bits* data_forwarding_map = &tenured->state;
  mark_bits* code_forwarding_map = &code->allocator->state;

  /* Figure out where blocks are going to go */
  data_forwarding_map->compute_forwarding();
  code_forwarding_map->compute_forwarding();

  const object* data_finger = (object*)tenured->start;
  const code_block* code_finger = (code_block*)code->allocator->start;

  {
    compaction_fixup fixup(data_forwarding_map, code_forwarding_map, &data_finger,
                           &code_finger);
    slot_visitor<compaction_fixup> forwarder(this, fixup);

    forwarder.visit_uninitialized_code_blocks();

    /* Object start offsets get recomputed by the object_compaction_updater */
    data->tenured->starts.clear_object_start_offsets();

    /* Slide everything in tenured space up, and update data and code heap
       pointers inside objects. */
    auto compact_object_func = [&](object* old_addr, object* new_addr, cell size) {
      forwarder.visit_slots(new_addr);
      forwarder.visit_object_code_block(new_addr);
      tenured->starts.record_object_start_offset(new_addr);
    };
    tenured->compact(compact_object_func, fixup, &data_finger);

    /* Slide everything in the code heap up, and update data and code heap
       pointers inside code blocks. */
    auto compact_code_func = [&](code_block* old_addr,
                                 code_block* new_addr,
                                 cell size) {
      forwarder.visit_code_block_objects(new_addr);
      cell old_entry_point = old_addr->entry_point();
      forwarder.visit_instruction_operands(new_addr, old_entry_point);
    };
    code->allocator->compact(compact_code_func, fixup, &code_finger);

    forwarder.visit_all_roots();
    forwarder.visit_context_code_blocks();
  }

  update_code_roots_for_compaction();

  /* Each callback has a relocation with a pointer to a code block in
     the code heap. Since the code heap has now been compacted, those
     pointers are invalid and we need to update them. */
  auto callback_updater = [&](code_block* stub, cell size) {
    callbacks->update(stub);
  };
  callbacks->allocator->iterate(callback_updater);

  code->initialize_all_blocks_set();

  if (event)
    event->ended_compaction();
}

void factor_vm::collect_compact() {
  collect_mark_impl();
  collect_compact_impl();

  if (data->high_fragmentation_p()) {
    /* Compaction did not free up enough memory. Grow the heap. */
    set_current_gc_op(collect_growing_heap_op);
    collect_growing_heap(0);
  }

  code->flush_icache();
}

void factor_vm::collect_growing_heap(cell requested_size) {
  /* Grow the data heap and copy all live objects to the new heap. */
  data_heap* old = data;
  set_data_heap(data->grow(&nursery, requested_size));
  collect_mark_impl();
  collect_compact_impl();
  code->flush_icache();
  delete old;
}

}