diff --git a/vm/atomic-cl-32.hpp b/vm/atomic-cl-32.hpp new file mode 100644 index 0000000000..a27516884d --- /dev/null +++ b/vm/atomic-cl-32.hpp @@ -0,0 +1,46 @@ +namespace factor { + namespace atomic { + __forceinline static bool cas(volatile cell *ptr, cell old_val, cell new_val) + { + return InterlockedCompareExchange( + reinterpret_cast(ptr), + (LONG)old_val, + (LONG)new_val) == (LONG)old_val; + } + __forceinline static bool cas(volatile fixnum *ptr, fixnum old_val, fixnum new_val) + { + return InterlockedCompareExchange( + reinterpret_cast(ptr), + (LONG)old_val, + (LONG)new_val) == (LONG)old_val; + } + + __forceinline static cell add(volatile cell *ptr, cell val) + { + return (cell)InterlockedAdd( + reinterpret_cast(ptr), (LONG)val); + } + __forceinline static fixnum add(volatile fixnum *ptr, fixnum val) + { + return (fixnum)InterlockedAdd( + reinterpret_cast(ptr), (LONG)val); + } + + __forceinline static cell subtract(volatile cell *ptr, cell val) + { + return (cell)InterlockedAdd( + reinterpret_cast(ptr), -(LONG)val); + } + __forceinline static fixnum subtract(volatile fixnum *ptr, fixnum val) + { + return (fixnum)InterlockedAdd( + reinterpret_cast(ptr), -(LONG)val); + } + + __forceinline static void fence() + { + MemoryBarrier(); + } + } +} + diff --git a/vm/atomic-cl-64.hpp b/vm/atomic-cl-64.hpp new file mode 100644 index 0000000000..595bf5f30d --- /dev/null +++ b/vm/atomic-cl-64.hpp @@ -0,0 +1,46 @@ +namespace factor { + namespace atomic { + __forceinline static bool cas(volatile cell *ptr, cell old_val, cell new_val) + { + return InterlockedCompareExchange64( + reinterpret_cast(ptr), + (LONG64)old_val, + (LONG64)new_val) == (LONG64)old_val; + } + __forceinline static bool cas(volatile fixnum *ptr, fixnum old_val, fixnum new_val) + { + return InterlockedCompareExchange64( + reinterpret_cast(ptr), + (LONG64)old_val, + (LONG64)new_val) == (LONG64)old_val; + } + + __forceinline static cell add(volatile cell *ptr, cell val) + { + return (cell)InterlockedAdd64( + reinterpret_cast(ptr), (LONG64)val); + } + __forceinline static fixnum add(volatile fixnum *ptr, fixnum val) + { + return (fixnum)InterlockedAdd64( + reinterpret_cast(ptr), (LONG64)val); + } + + __forceinline static cell subtract(volatile cell *ptr, cell val) + { + return (cell)InterlockedAdd64( + reinterpret_cast(ptr), -(LONG64)val); + } + __forceinline static fixnum subtract(volatile fixnum *ptr, fixnum val) + { + return (fixnum)InterlockedAdd64( + reinterpret_cast(ptr), -(LONG64)val); + } + + __forceinline static void fence() + { + MemoryBarrier(); + } + } +} + diff --git a/vm/atomic-gcc.hpp b/vm/atomic-gcc.hpp new file mode 100644 index 0000000000..d6fd5a2b77 --- /dev/null +++ b/vm/atomic-gcc.hpp @@ -0,0 +1,42 @@ +namespace factor { + namespace atomic { + __attribute__((always_inline)) + inline static bool cas(volatile cell *ptr, cell old_val, cell new_val) + { + return __sync_bool_compare_and_swap(ptr, old_val, new_val); + } + __attribute__((always_inline)) + inline static bool cas(volatile fixnum *ptr, fixnum old_val, fixnum new_val) + { + return __sync_bool_compare_and_swap(ptr, old_val, new_val); + } + + __attribute__((always_inline)) + inline static cell add(volatile cell *ptr, cell val) + { + return __sync_add_and_fetch(ptr, val); + } + __attribute__((always_inline)) + inline static fixnum add(volatile fixnum *ptr, fixnum val) + { + return __sync_add_and_fetch(ptr, val); + } + + __attribute__((always_inline)) + inline static cell subtract(volatile cell *ptr, cell val) + { + return __sync_sub_and_fetch(ptr, val); + } + __attribute__((always_inline)) + inline static fixnum subtract(volatile fixnum *ptr, fixnum val) + { + return __sync_sub_and_fetch(ptr, val); + } + + __attribute__((always_inline)) + inline static void fence() + { + __sync_synchronize(); + } + } +} diff --git a/vm/errors.cpp b/vm/errors.cpp index e5a2f385a5..bbf02c2db2 100755 --- a/vm/errors.cpp +++ b/vm/errors.cpp @@ -170,23 +170,6 @@ void factor_vm::enqueue_safepoint_fep() code->guard_safepoint(); } -void factor_vm::enqueue_safepoint_sample(cell samples, cell pc, bool foreign_thread_p) -{ - if (FACTOR_MEMORY_BARRIER(), sampling_profiler_p) - { - FACTOR_ATOMIC_ADD(&safepoint_sample_counts.sample_count, samples); - if (foreign_thread_p) - FACTOR_ATOMIC_ADD(&safepoint_sample_counts.foreign_thread_sample_count, samples); - else { - if (FACTOR_MEMORY_BARRIER(), current_gc) - FACTOR_ATOMIC_ADD(&safepoint_sample_counts.gc_sample_count, samples); - if (pc != 0 && !code->seg->in_segment_p(pc)) - FACTOR_ATOMIC_ADD(&safepoint_sample_counts.foreign_sample_count, samples); - } - code->guard_safepoint(); - } -} - void factor_vm::handle_safepoint() { code->unguard_safepoint(); @@ -199,9 +182,7 @@ void factor_vm::handle_safepoint() safepoint_fep = false; } else if (sampling_profiler_p) - { record_sample(); - } } } diff --git a/vm/os-unix.hpp b/vm/os-unix.hpp index 2c7dde9c61..a64b8f0e08 100644 --- a/vm/os-unix.hpp +++ b/vm/os-unix.hpp @@ -10,6 +10,8 @@ #include #include +#include "atomic-gcc.hpp" + namespace factor { diff --git a/vm/os-windows.32.hpp b/vm/os-windows.32.hpp index 748272ff38..c6e4c8b7b3 100644 --- a/vm/os-windows.32.hpp +++ b/vm/os-windows.32.hpp @@ -1,3 +1,5 @@ +#include "atomic-cl-32.hpp" + namespace factor { diff --git a/vm/os-windows.64.hpp b/vm/os-windows.64.hpp index aff662a489..ff9cf4a0bb 100644 --- a/vm/os-windows.64.hpp +++ b/vm/os-windows.64.hpp @@ -1,3 +1,5 @@ +#include "atomic-cl-64.hpp" + namespace factor { diff --git a/vm/os-windows.cpp b/vm/os-windows.cpp index 978cbc7bed..cfbbf666df 100755 --- a/vm/os-windows.cpp +++ b/vm/os-windows.cpp @@ -309,7 +309,7 @@ void factor_vm::sampler_thread_loop() assert(QueryPerformanceCounter(&counter)); counter.QuadPart *= FACTOR_PROFILE_SAMPLES_PER_SECOND; - while (FACTOR_MEMORY_BARRIER(), sampling_profiler_p) + while (atomic::fence(), sampling_profiler_p) { SwitchToThread(); assert(QueryPerformanceCounter(&new_counter)); @@ -346,7 +346,7 @@ void factor_vm::start_sampling_profiler_timer() void factor_vm::end_sampling_profiler_timer() { sampling_profiler_p = false; - FACTOR_MEMORY_BARRIER(); + atomic::fence(); DWORD wait_result = WaitForSingleObject(sampler_thread, 3000/FACTOR_PROFILE_SAMPLES_PER_SECOND); if (wait_result != WAIT_OBJECT_0) diff --git a/vm/sampling_profiler.cpp b/vm/sampling_profiler.cpp index 07c61afbfb..738616dc3c 100644 --- a/vm/sampling_profiler.cpp +++ b/vm/sampling_profiler.cpp @@ -5,16 +5,16 @@ namespace factor profiling_sample_count profiling_sample_count::record_counts() volatile { - FACTOR_MEMORY_BARRIER(); + atomic::fence(); profiling_sample_count returned( sample_count, gc_sample_count, foreign_sample_count, foreign_thread_sample_count); - FACTOR_ATOMIC_SUB(&sample_count, returned.sample_count); - FACTOR_ATOMIC_SUB(&gc_sample_count, returned.gc_sample_count); - FACTOR_ATOMIC_SUB(&foreign_sample_count, returned.foreign_sample_count); - FACTOR_ATOMIC_SUB(&foreign_thread_sample_count, returned.foreign_thread_sample_count); + atomic::subtract(&sample_count, returned.sample_count); + atomic::subtract(&gc_sample_count, returned.gc_sample_count); + atomic::subtract(&foreign_sample_count, returned.foreign_sample_count); + atomic::subtract(&foreign_thread_sample_count, returned.foreign_thread_sample_count); return returned; } @@ -24,7 +24,7 @@ void profiling_sample_count::clear() volatile gc_sample_count = 0; foreign_sample_count = 0; foreign_thread_sample_count = 0; - FACTOR_MEMORY_BARRIER(); + atomic::fence(); } profiling_sample::profiling_sample(factor_vm *vm, @@ -91,7 +91,7 @@ void factor_vm::start_sampling_profiler() void factor_vm::end_sampling_profiler() { sampling_profiler_p = false; - FACTOR_MEMORY_BARRIER(); + atomic::fence(); end_sampling_profiler_timer(); record_sample(); } @@ -145,4 +145,21 @@ void factor_vm::primitive_clear_samples() clear_samples(); } +void factor_vm::enqueue_safepoint_sample(cell samples, cell pc, bool foreign_thread_p) +{ + if (atomic::fence(), sampling_profiler_p) + { + atomic::add(&safepoint_sample_counts.sample_count, samples); + if (foreign_thread_p) + atomic::add(&safepoint_sample_counts.foreign_thread_sample_count, samples); + else { + if (atomic::fence(), current_gc) + atomic::add(&safepoint_sample_counts.gc_sample_count, samples); + if (pc != 0 && !code->seg->in_segment_p(pc)) + atomic::add(&safepoint_sample_counts.foreign_sample_count, samples); + } + code->guard_safepoint(); + } +} + } diff --git a/vm/utilities.hpp b/vm/utilities.hpp index da515d80ce..5f37644213 100755 --- a/vm/utilities.hpp +++ b/vm/utilities.hpp @@ -53,53 +53,4 @@ vm_char *safe_strdup(const vm_char *str); cell read_cell_hex(); VM_C_API void *factor_memcpy(void *dst, void *src, size_t len); -#if defined(WINDOWS) - - #if defined(FACTOR_64) - - #define FACTOR_ATOMIC_CAS(ptr, old_val, new_val) \ - (InterlockedCompareExchange64( \ - reinterpret_cast(ptr), new_val, old_val) == old_val) - - #define FACTOR_ATOMIC_ADD(ptr, val) \ - InterlockedAdd64(reinterpret_cast(ptr), val) - - #define FACTOR_ATOMIC_SUB(ptr, val) \ - InterlockedAdd64(reinterpret_cast(ptr), -val) - - #else - - #define FACTOR_ATOMIC_CAS(ptr, old_val, new_val) \ - (InterlockedCompareExchange( \ - reinterpret_cast(ptr), new_val, old_val) == old_val) - - #define FACTOR_ATOMIC_ADD(ptr, val) \ - InterlockedAdd(reinterpret_cast(ptr), val) - - #define FACTOR_ATOMIC_SUB(ptr, val) \ - InterlockedAdd(reinterpret_cast(ptr), -val) - - #endif - - #define FACTOR_MEMORY_BARRIER() \ - MemoryBarrier() - -#elif defined(__GNUC__) || defined(__clang__) - - #define FACTOR_ATOMIC_CAS(ptr, old_val, new_val) \ - __sync_bool_compare_and_swap(ptr, old_val, new_val) - - #define FACTOR_ATOMIC_ADD(ptr, val) \ - __sync_add_and_fetch(ptr, val) - - #define FACTOR_ATOMIC_SUB(ptr, val) \ - __sync_sub_and_fetch(ptr, val) - - #define FACTOR_MEMORY_BARRIER() \ - __sync_synchronize() - -#else - #error "Unsupported compiler" -#endif - } diff --git a/vm/vm.hpp b/vm/vm.hpp index 1e53b04e23..6d08ca2041 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -197,6 +197,7 @@ struct factor_vm void start_sampling_profiler(); void end_sampling_profiler(); void set_sampling_profiler(bool sampling); + void enqueue_safepoint_sample(cell samples, cell pc, bool foreign_thread_p); void primitive_sampling_profiler(); void primitive_get_samples(); void primitive_clear_samples(); @@ -214,7 +215,6 @@ struct factor_vm void synchronous_signal_handler_impl(); void fp_signal_handler_impl(); void enqueue_safepoint_fep(); - void enqueue_safepoint_sample(cell samples, cell pc, bool foreign_thread_p); void handle_safepoint(); // bignum