diff --git a/Makefile b/Makefile index 35cf7a05c4..2ea43706f4 100755 --- a/Makefile +++ b/Makefile @@ -41,22 +41,23 @@ DLL_OBJS = $(PLAF_DLL_OBJS) \ vm/callstack.o \ vm/code_block.o \ vm/code_heap.o \ + vm/compaction.o \ vm/contexts.o \ vm/data_heap.o \ vm/debug.o \ vm/dispatch.o \ vm/errors.o \ vm/factor.o \ + vm/free_list.o \ vm/full_collector.o \ vm/gc.o \ - vm/heap.o \ vm/image.o \ vm/inline_cache.o \ vm/io.o \ vm/jit.o \ vm/math.o \ vm/nursery_collector.o \ - vm/old_space.o \ + vm/object_start_map.o \ vm/primitives.o \ vm/profiler.o \ vm/quotations.o \ diff --git a/basis/alien/c-types/c-types.factor b/basis/alien/c-types/c-types.factor index 119e437734..cfbed5378d 100755 --- a/basis/alien/c-types/c-types.factor +++ b/basis/alien/c-types/c-types.factor @@ -230,6 +230,10 @@ M: byte-array byte-length length ; inline M: f byte-length drop 0 ; inline +: >c-bool ( ? -- int ) 1 0 ? ; inline + +: c-bool> ( int -- ? ) 0 = not ; inline + MIXIN: value-type : c-getter ( name -- quot ) @@ -256,6 +260,7 @@ PREDICATE: typedef-word < c-type-word "c-type" word-prop c-type-name? ; M: string typedef ( old new -- ) c-types get set-at ; + M: word typedef ( old new -- ) { [ nip define-symbol ] @@ -292,7 +297,7 @@ M: long-long-type box-return ( c-type -- ) : define-out ( name -- ) [ "alien.c-types" constructor-word ] - [ dup c-setter '[ _ heap-size [ 0 @ ] keep ] ] bi + [ dup c-setter '[ _ heap-size (byte-array) [ 0 @ ] keep ] ] bi (( value -- c-ptr )) define-inline ; : define-primitive-type ( c-type name -- ) @@ -338,7 +343,7 @@ SYMBOLS: [ alien-signed-8 ] >>getter [ set-alien-signed-8 ] >>setter 8 >>size - 8 >>align + cpu x86.32? os windows? not and 4 8 ? >>align "box_signed_8" >>boxer "to_signed_8" >>unboxer \ longlong define-primitive-type @@ -349,7 +354,7 @@ SYMBOLS: [ alien-unsigned-8 ] >>getter [ set-alien-unsigned-8 ] >>setter 8 >>size - 8 >>align + cpu x86.32? os windows? not and 4 8 ? >>align "box_unsigned_8" >>boxer "to_unsigned_8" >>unboxer \ ulonglong define-primitive-type @@ -442,14 +447,24 @@ SYMBOLS: "to_cell" >>unboxer \ uchar define-primitive-type - - [ alien-unsigned-1 0 = not ] >>getter - [ [ 1 0 ? ] 2dip set-alien-unsigned-1 ] >>setter - 1 >>size - 1 >>align - "box_boolean" >>boxer - "to_boolean" >>unboxer - \ bool define-primitive-type + cpu ppc? [ + + [ alien-unsigned-4 c-bool> ] >>getter + [ [ >c-bool ] 2dip set-alien-unsigned-4 ] >>setter + 4 >>size + 4 >>align + "box_boolean" >>boxer + "to_boolean" >>unboxer + ] [ + + [ alien-unsigned-1 c-bool> ] >>getter + [ [ >c-bool ] 2dip set-alien-unsigned-1 ] >>setter + 1 >>size + 1 >>align + "box_boolean" >>boxer + "to_boolean" >>unboxer + \ bool define-primitive-type + ] if math:float >>class @@ -470,17 +485,24 @@ SYMBOLS: [ alien-double ] >>getter [ [ >float ] 2dip set-alien-double ] >>setter 8 >>size - 8 >>align + cpu x86.32? os windows? not and 4 8 ? >>align "box_double" >>boxer "to_double" >>unboxer double-rep >>rep [ >float ] >>unboxer-quot \ double define-primitive-type - \ long c-type \ ptrdiff_t typedef - \ long c-type \ intptr_t typedef - \ ulong c-type \ uintptr_t typedef - \ ulong c-type \ size_t typedef + cpu x86.64? os windows? and [ + \ longlong c-type \ ptrdiff_t typedef + \ longlong c-type \ intptr_t typedef + \ ulonglong c-type \ uintptr_t typedef + \ ulonglong c-type \ size_t typedef + ] [ + \ long c-type \ ptrdiff_t typedef + \ long c-type \ intptr_t typedef + \ ulong c-type \ uintptr_t typedef + \ ulong c-type \ size_t typedef + ] if ] with-compilation-unit M: char-16-rep rep-component-type drop char ; diff --git a/basis/alien/data/data.factor b/basis/alien/data/data.factor index fc18921ef1..93b1afd436 100644 --- a/basis/alien/data/data.factor +++ b/basis/alien/data/data.factor @@ -65,10 +65,6 @@ M: memory-stream stream-read : byte-array>memory ( byte-array base -- ) swap dup byte-length memcpy ; inline -: >c-bool ( ? -- int ) 1 0 ? ; inline - -: c-bool> ( int -- ? ) 0 = not ; inline - M: value-type c-type-rep drop int-rep ; M: value-type c-type-getter @@ -77,5 +73,3 @@ M: value-type c-type-getter M: value-type c-type-setter ( type -- quot ) [ c-type-getter ] [ c-type-unboxer-quot ] [ heap-size ] tri '[ @ swap @ _ memcpy ] ; - - diff --git a/basis/bootstrap/image/image.factor b/basis/bootstrap/image/image.factor index 421a7d2ecd..2178b5d4cb 100644 --- a/basis/bootstrap/image/image.factor +++ b/basis/bootstrap/image/image.factor @@ -176,14 +176,12 @@ USERENV: callback-stub 45 ! PIC stubs USERENV: pic-load 47 USERENV: pic-tag 48 -USERENV: pic-hi-tag 49 -USERENV: pic-tuple 50 -USERENV: pic-hi-tag-tuple 51 -USERENV: pic-check-tag 52 -USERENV: pic-check 53 -USERENV: pic-hit 54 -USERENV: pic-miss-word 55 -USERENV: pic-miss-tail-word 56 +USERENV: pic-tuple 49 +USERENV: pic-check-tag 50 +USERENV: pic-check-tuple 51 +USERENV: pic-hit 52 +USERENV: pic-miss-word 53 +USERENV: pic-miss-tail-word 54 ! Megamorphic dispatch USERENV: mega-lookup 57 @@ -217,13 +215,18 @@ USERENV: undefined-quot 60 : here-as ( tag -- pointer ) here bitor ; +: (align-here) ( alignment -- ) + [ here neg ] dip rem + [ bootstrap-cell /i [ 0 emit ] times ] unless-zero ; + : align-here ( -- ) - here 8 mod 4 = [ 0 emit ] when ; + data-alignment get (align-here) ; : emit-fixnum ( n -- ) tag-fixnum emit ; : emit-object ( class quot -- addr ) - over tag-number here-as [ swap type-number tag-fixnum emit call align-here ] dip ; + [ type-number ] dip over here-as + [ swap tag-fixnum emit call align-here ] dip ; inline ! Write an object to the image. @@ -292,7 +295,7 @@ M: fake-bignum ' n>> tag-fixnum ; M: float ' [ float [ - align-here double>bits emit-64 + 8 (align-here) double>bits emit-64 ] emit-object ] cache-eql-object ; @@ -304,7 +307,7 @@ M: float ' M: f ' #! f is #define F RETAG(0,F_TYPE) - drop \ f tag-number ; + drop \ f type-number ; : 0, ( -- ) 0 >bignum ' 0-offset fixup ; : 1, ( -- ) 1 >bignum ' 1-offset fixup ; @@ -410,6 +413,7 @@ M: byte-array ' [ byte-array [ dup length emit-fixnum + bootstrap-cell 4 = [ 0 emit 0 emit ] when pad-bytes emit-bytes ] emit-object ] cache-eq-object ; diff --git a/basis/bootstrap/tools/tools.factor b/basis/bootstrap/tools/tools.factor index 848e310d63..51f44025c9 100644 --- a/basis/bootstrap/tools/tools.factor +++ b/basis/bootstrap/tools/tools.factor @@ -12,6 +12,7 @@ IN: bootstrap.tools "tools.deploy" "tools.destructors" "tools.disassembler" + "tools.dispatch" "tools.memory" "tools.profiler" "tools.test" diff --git a/basis/compiler/cfg/alias-analysis/alias-analysis.factor b/basis/compiler/cfg/alias-analysis/alias-analysis.factor index 2303b98aed..9fffa0eed2 100644 --- a/basis/compiler/cfg/alias-analysis/alias-analysis.factor +++ b/basis/compiler/cfg/alias-analysis/alias-analysis.factor @@ -284,7 +284,7 @@ M: ##copy analyze-aliases* M: ##compare analyze-aliases* call-next-method dup useless-compare? [ - dst>> \ f tag-number \ ##load-immediate new-insn + dst>> \ f type-number \ ##load-immediate new-insn analyze-aliases* ] when ; diff --git a/basis/compiler/cfg/builder/builder-tests.factor b/basis/compiler/cfg/builder/builder-tests.factor index 9d1502d3f0..7f1b6aa6f2 100644 --- a/basis/compiler/cfg/builder/builder-tests.factor +++ b/basis/compiler/cfg/builder/builder-tests.factor @@ -119,7 +119,6 @@ IN: compiler.cfg.builder.tests { byte-array - simple-alien alien POSTPONE: f } [| class | @@ -192,7 +191,7 @@ IN: compiler.cfg.builder.tests ] unit-test [ f t ] [ - [ { fixnum simple-alien } declare 0 alien-cell ] + [ { fixnum alien } declare 0 alien-cell ] [ [ ##unbox-any-c-ptr? ] contains-insn? ] [ [ ##unbox-alien? ] contains-insn? ] bi ] unit-test @@ -205,7 +204,7 @@ IN: compiler.cfg.builder.tests ] unit-test [ f t ] [ - [ { byte-array fixnum } declare alien-cell { simple-alien } declare 4 alien-float ] + [ { byte-array fixnum } declare alien-cell { alien } declare 4 alien-float ] [ [ ##box-alien? ] contains-insn? ] [ [ ##allot? ] contains-insn? ] bi ] unit-test diff --git a/basis/compiler/cfg/builder/builder.factor b/basis/compiler/cfg/builder/builder.factor index 11aae28bf3..cf6215c5cd 100755 --- a/basis/compiler/cfg/builder/builder.factor +++ b/basis/compiler/cfg/builder/builder.factor @@ -117,7 +117,7 @@ M: #recursive emit-node and ; : emit-trivial-if ( -- ) - ds-pop \ f tag-number cc/= ^^compare-imm ds-push ; + ds-pop \ f type-number cc/= ^^compare-imm ds-push ; : trivial-not-if? ( #if -- ? ) children>> first2 @@ -126,12 +126,12 @@ M: #recursive emit-node and ; : emit-trivial-not-if ( -- ) - ds-pop \ f tag-number cc= ^^compare-imm ds-push ; + ds-pop \ f type-number cc= ^^compare-imm ds-push ; : emit-actual-if ( #if -- ) ! Inputs to the final instruction need to be copied because of ! loc>vreg sync - ds-pop any-rep ^^copy \ f tag-number cc/= ##compare-imm-branch emit-if ; + ds-pop any-rep ^^copy \ f type-number cc/= ##compare-imm-branch emit-if ; M: #if emit-node { diff --git a/basis/compiler/cfg/def-use/def-use.factor b/basis/compiler/cfg/def-use/def-use.factor index 825ff71b9b..54cff2ccaa 100644 --- a/basis/compiler/cfg/def-use/def-use.factor +++ b/basis/compiler/cfg/def-use/def-use.factor @@ -11,6 +11,10 @@ GENERIC: defs-vreg ( insn -- vreg/f ) GENERIC: temp-vregs ( insn -- seq ) GENERIC: uses-vregs ( insn -- seq ) +M: insn defs-vreg drop f ; +M: insn temp-vregs drop { } ; +M: insn uses-vregs drop { } ; + M: ##phi uses-vregs inputs>> values ; > values ; } case ; : define-defs-vreg-method ( insn -- ) - [ \ defs-vreg create-method ] - [ insn-def-slot [ name>> reader-word 1quotation ] [ [ drop f ] ] if* ] bi - define ; + dup insn-def-slot dup [ + [ \ defs-vreg create-method ] + [ name>> reader-word 1quotation ] bi* + define + ] [ 2drop ] if ; : define-uses-vregs-method ( insn -- ) - [ \ uses-vregs create-method ] - [ insn-use-slots [ name>> ] map slot-array-quot ] bi - define ; + dup insn-use-slots [ drop ] [ + [ \ uses-vregs create-method ] + [ [ name>> ] map slot-array-quot ] bi* + define + ] if-empty ; : define-temp-vregs-method ( insn -- ) - [ \ temp-vregs create-method ] - [ insn-temp-slots [ name>> ] map slot-array-quot ] bi - define ; + dup insn-temp-slots [ drop ] [ + [ \ temp-vregs create-method ] + [ [ name>> ] map slot-array-quot ] bi* + define + ] if-empty ; PRIVATE> diff --git a/basis/compiler/cfg/gc-checks/gc-checks.factor b/basis/compiler/cfg/gc-checks/gc-checks.factor index 7285685b48..6d192ec54a 100644 --- a/basis/compiler/cfg/gc-checks/gc-checks.factor +++ b/basis/compiler/cfg/gc-checks/gc-checks.factor @@ -1,7 +1,7 @@ ! Copyright (C) 2009 Slava Pestov. ! See http://factorcode.org/license.txt for BSD license. -USING: accessors kernel sequences assocs fry -cpu.architecture layouts +USING: accessors kernel sequences assocs fry math +cpu.architecture layouts namespaces compiler.cfg.rpo compiler.cfg.registers compiler.cfg.instructions @@ -21,12 +21,14 @@ GENERIC: allocation-size* ( insn -- n ) M: ##allot allocation-size* size>> ; -M: ##box-alien allocation-size* drop 4 cells ; +M: ##box-alien allocation-size* drop 5 cells ; -M: ##box-displaced-alien allocation-size* drop 4 cells ; +M: ##box-displaced-alien allocation-size* drop 5 cells ; : allocation-size ( bb -- n ) - instructions>> [ ##allocation? ] filter [ allocation-size* ] map-sum ; + instructions>> + [ ##allocation? ] filter + [ allocation-size* data-alignment get align ] map-sum ; : insert-gc-check ( bb -- ) dup dup '[ diff --git a/basis/compiler/cfg/hats/hats.factor b/basis/compiler/cfg/hats/hats.factor index 783df0678c..9d1945c525 100644 --- a/basis/compiler/cfg/hats/hats.factor +++ b/basis/compiler/cfg/hats/hats.factor @@ -43,14 +43,14 @@ insn-classes get [ : ^^load-literal ( obj -- dst ) [ next-vreg dup ] dip { - { [ dup not ] [ drop \ f tag-number ##load-immediate ] } + { [ dup not ] [ drop \ f type-number ##load-immediate ] } { [ dup fixnum? ] [ tag-fixnum ##load-immediate ] } { [ dup float? ] [ ##load-constant ] } [ ##load-reference ] } cond ; : ^^offset>slot ( slot -- vreg' ) - cell 4 = [ 1 ^^shr-imm ] [ any-rep ^^copy ] if ; + cell 4 = 2 1 ? ^^shr-imm ; : ^^tag-fixnum ( src -- dst ) tag-bits get ^^shl-imm ; diff --git a/basis/compiler/cfg/instructions/instructions.factor b/basis/compiler/cfg/instructions/instructions.factor index 30fe8b590e..00ded636ac 100644 --- a/basis/compiler/cfg/instructions/instructions.factor +++ b/basis/compiler/cfg/instructions/instructions.factor @@ -512,13 +512,12 @@ temp: temp/int-rep ; PURE-INSN: ##box-displaced-alien def: dst/int-rep use: displacement/int-rep base/int-rep -temp: temp1/int-rep temp2/int-rep +temp: temp/int-rep literal: base-class ; PURE-INSN: ##unbox-any-c-ptr def: dst/int-rep -use: src/int-rep -temp: temp/int-rep ; +use: src/int-rep ; : ##unbox-f ( dst src -- ) drop 0 ##load-immediate ; : ##unbox-byte-array ( dst src -- ) byte-array-offset ##add-imm ; @@ -527,12 +526,12 @@ PURE-INSN: ##unbox-alien def: dst/int-rep use: src/int-rep ; -: ##unbox-c-ptr ( dst src class temp -- ) +: ##unbox-c-ptr ( dst src class -- ) { - { [ over \ f class<= ] [ 2drop ##unbox-f ] } - { [ over simple-alien class<= ] [ 2drop ##unbox-alien ] } - { [ over byte-array class<= ] [ 2drop ##unbox-byte-array ] } - [ nip ##unbox-any-c-ptr ] + { [ dup \ f class<= ] [ drop ##unbox-f ] } + { [ dup alien class<= ] [ drop ##unbox-alien ] } + { [ dup byte-array class<= ] [ drop ##unbox-byte-array ] } + [ drop ##unbox-any-c-ptr ] } cond ; ! Alien accessors diff --git a/basis/compiler/cfg/intrinsics/alien/alien.factor b/basis/compiler/cfg/intrinsics/alien/alien.factor index fb993681e8..320a0a08f7 100644 --- a/basis/compiler/cfg/intrinsics/alien/alien.factor +++ b/basis/compiler/cfg/intrinsics/alien/alien.factor @@ -33,7 +33,7 @@ IN: compiler.cfg.intrinsics.alien bi and ; : ^^unbox-c-ptr ( src class -- dst ) - [ next-vreg dup ] 2dip next-vreg ##unbox-c-ptr ; + [ next-vreg dup ] 2dip ##unbox-c-ptr ; : prepare-alien-accessor ( info -- ptr-vreg offset ) class>> [ 2inputs ^^untag-fixnum swap ] dip ^^unbox-c-ptr ^^add 0 ; diff --git a/basis/compiler/cfg/intrinsics/allot/allot.factor b/basis/compiler/cfg/intrinsics/allot/allot.factor index 044b839f4d..9804244ecb 100644 --- a/basis/compiler/cfg/intrinsics/allot/allot.factor +++ b/basis/compiler/cfg/intrinsics/allot/allot.factor @@ -8,7 +8,7 @@ compiler.cfg.utilities compiler.cfg.builder.blocks ; IN: compiler.cfg.intrinsics.allot : ##set-slots ( regs obj class -- ) - '[ _ swap 1 + _ tag-number ##set-slot-imm ] each-index ; + '[ _ swap 1 + _ type-number ##set-slot-imm ] each-index ; : emit-simple-allot ( node -- ) [ in-d>> length ] [ node-output-infos first class>> ] bi @@ -31,10 +31,10 @@ IN: compiler.cfg.intrinsics.allot ] [ drop emit-primitive ] if ; : store-length ( len reg class -- ) - [ [ ^^load-literal ] dip 1 ] dip tag-number ##set-slot-imm ; + [ [ ^^load-literal ] dip 1 ] dip type-number ##set-slot-imm ; :: store-initial-element ( len reg elt class -- ) - len [ [ elt reg ] dip 2 + class tag-number ##set-slot-imm ] each ; + len [ [ elt reg ] dip 2 + class type-number ##set-slot-imm ] each ; : expand-? ( obj -- ? ) dup integer? [ 0 8 between? ] [ drop f ] if ; @@ -62,7 +62,7 @@ IN: compiler.cfg.intrinsics.allot : bytes>cells ( m -- n ) cell align cell /i ; : ^^allot-byte-array ( n -- dst ) - 2 cells + byte-array ^^allot ; + 16 + byte-array ^^allot ; : emit-allot-byte-array ( len -- dst ) ds-drop diff --git a/basis/compiler/cfg/intrinsics/fixnum/fixnum.factor b/basis/compiler/cfg/intrinsics/fixnum/fixnum.factor index 8ead484cf1..e4d1735eae 100644 --- a/basis/compiler/cfg/intrinsics/fixnum/fixnum.factor +++ b/basis/compiler/cfg/intrinsics/fixnum/fixnum.factor @@ -21,7 +21,7 @@ IN: compiler.cfg.intrinsics.fixnum ds-push ; : tag-literal ( n -- tagged ) - literal>> [ tag-fixnum ] [ \ f tag-number ] if* ; + literal>> [ tag-fixnum ] [ \ f type-number ] if* ; : emit-fixnum-op ( insn -- ) [ 2inputs ] dip call ds-push ; inline diff --git a/basis/compiler/cfg/intrinsics/slots/slots.factor b/basis/compiler/cfg/intrinsics/slots/slots.factor index 39151083e5..1424aba354 100644 --- a/basis/compiler/cfg/intrinsics/slots/slots.factor +++ b/basis/compiler/cfg/intrinsics/slots/slots.factor @@ -8,7 +8,7 @@ compiler.cfg.instructions compiler.cfg.utilities compiler.cfg.builder.blocks compiler.constants ; IN: compiler.cfg.intrinsics.slots -: value-tag ( info -- n ) class>> class-tag ; inline +: value-tag ( info -- n ) class>> class-type ; inline : ^^tag-offset>slot ( slot tag -- vreg' ) [ ^^offset>slot ] dip ^^sub-imm ; diff --git a/basis/compiler/cfg/renaming/functor/functor.factor b/basis/compiler/cfg/renaming/functor/functor.factor index 2af68e9175..261aab6c54 100644 --- a/basis/compiler/cfg/renaming/functor/functor.factor +++ b/basis/compiler/cfg/renaming/functor/functor.factor @@ -20,15 +20,19 @@ WHERE GENERIC: rename-insn-defs ( insn -- ) -insn-classes get [ +M: insn rename-insn-defs drop ; + +insn-classes get [ insn-def-slot ] filter [ [ \ rename-insn-defs create-method-in ] - [ insn-def-slot dup [ name>> 1array ] when DEF-QUOT slot-change-quot ] bi + [ insn-def-slot name>> 1array DEF-QUOT slot-change-quot ] bi define ] each GENERIC: rename-insn-uses ( insn -- ) -insn-classes get { ##phi } diff [ +M: insn rename-insn-uses drop ; + +insn-classes get { ##phi } diff [ insn-use-slots empty? not ] filter [ [ \ rename-insn-uses create-method-in ] [ insn-use-slots [ name>> ] map USE-QUOT slot-change-quot ] bi define @@ -39,7 +43,9 @@ M: ##phi rename-insn-uses GENERIC: rename-insn-temps ( insn -- ) -insn-classes get [ +M: insn rename-insn-temps drop ; + +insn-classes get [ insn-temp-slots empty? not ] filter [ [ \ rename-insn-temps create-method-in ] [ insn-temp-slots [ name>> ] map TEMP-QUOT slot-change-quot ] bi define diff --git a/basis/compiler/cfg/representations/preferred/preferred.factor b/basis/compiler/cfg/representations/preferred/preferred.factor index 1e07e56b35..726521cfe1 100644 --- a/basis/compiler/cfg/representations/preferred/preferred.factor +++ b/basis/compiler/cfg/representations/preferred/preferred.factor @@ -11,6 +11,10 @@ GENERIC: defs-vreg-rep ( insn -- rep/f ) GENERIC: temp-vreg-reps ( insn -- reps ) GENERIC: uses-vreg-reps ( insn -- reps ) +M: insn defs-vreg-rep drop f ; +M: insn temp-vreg-reps drop { } ; +M: insn uses-vreg-reps drop { } ; + > rep-getter-quot ] [ [ drop f ] ] if* ] - bi define ; + dup insn-def-slot dup [ + [ \ defs-vreg-rep create-method ] + [ rep>> rep-getter-quot ] + bi* define + ] [ 2drop ] if ; : reps-getter-quot ( reps -- quot ) dup [ rep>> { f scalar-rep } member-eq? not ] all? [ @@ -38,14 +44,18 @@ GENERIC: uses-vreg-reps ( insn -- reps ) ] if ; : define-uses-vreg-reps-method ( insn -- ) - [ \ uses-vreg-reps create-method ] - [ insn-use-slots reps-getter-quot ] - bi define ; + dup insn-use-slots [ drop ] [ + [ \ uses-vreg-reps create-method ] + [ reps-getter-quot ] + bi* define + ] if-empty ; : define-temp-vreg-reps-method ( insn -- ) - [ \ temp-vreg-reps create-method ] - [ insn-temp-slots reps-getter-quot ] - bi define ; + dup insn-temp-slots [ drop ] [ + [ \ temp-vreg-reps create-method ] + [ reps-getter-quot ] + bi* define + ] if-empty ; PRIVATE> diff --git a/basis/compiler/cfg/representations/representations.factor b/basis/compiler/cfg/representations/representations.factor index 9546721594..005fe8c90b 100644 --- a/basis/compiler/cfg/representations/representations.factor +++ b/basis/compiler/cfg/representations/representations.factor @@ -47,7 +47,7 @@ M:: vector-rep emit-box ( dst src rep -- ) int-rep next-vreg-rep :> temp dst 16 2 cells + byte-array int-rep next-vreg-rep ##allot temp 16 tag-fixnum ##load-immediate - temp dst 1 byte-array tag-number ##set-slot-imm + temp dst 1 byte-array type-number ##set-slot-imm dst byte-array-offset src rep ##set-alien-vector ; M: vector-rep emit-unbox diff --git a/basis/compiler/cfg/value-numbering/rewrite/rewrite.factor b/basis/compiler/cfg/value-numbering/rewrite/rewrite.factor index 5d4ff5efb9..4864a8bfb7 100755 --- a/basis/compiler/cfg/value-numbering/rewrite/rewrite.factor +++ b/basis/compiler/cfg/value-numbering/rewrite/rewrite.factor @@ -37,7 +37,7 @@ M: insn rewrite drop f ; dup ##compare-imm-branch? [ { [ cc>> cc/= eq? ] - [ src2>> \ f tag-number eq? ] + [ src2>> \ f type-number eq? ] } 1&& ] [ drop f ] if ; inline @@ -110,7 +110,7 @@ M: ##compare-imm rewrite-tagged-comparison : rewrite-redundant-comparison? ( insn -- ? ) { [ src1>> vreg>expr general-compare-expr? ] - [ src2>> \ f tag-number = ] + [ src2>> \ f type-number = ] [ cc>> { cc= cc/= } member-eq? ] } 1&& ; inline @@ -204,7 +204,7 @@ M: ##compare-branch rewrite [ dst>> ] dip { { t [ t \ ##load-constant new-insn ] } - { f [ \ f tag-number \ ##load-immediate new-insn ] } + { f [ \ f type-number \ ##load-immediate new-insn ] } } case ; : rewrite-self-compare ( insn -- insn' ) @@ -440,7 +440,7 @@ M: ##sar rewrite \ ##sar-imm rewrite-arithmetic ; :: rewrite-unbox-displaced-alien ( insn expr -- insns ) [ next-vreg :> temp - temp expr base>> vn>vreg expr base-class>> insn temp>> ##unbox-c-ptr + temp expr base>> vn>vreg expr base-class>> ##unbox-c-ptr insn dst>> temp expr displacement>> vn>vreg ##add ] { } make ; diff --git a/basis/compiler/cfg/value-numbering/value-numbering-tests.factor b/basis/compiler/cfg/value-numbering/value-numbering-tests.factor index 55ff39e9d2..b404c4d4a4 100644 --- a/basis/compiler/cfg/value-numbering/value-numbering-tests.factor +++ b/basis/compiler/cfg/value-numbering/value-numbering-tests.factor @@ -82,7 +82,7 @@ IN: compiler.cfg.value-numbering.tests T{ ##load-reference f 1 + } T{ ##peek f 2 D 0 } T{ ##compare f 4 2 1 cc> } - T{ ##compare-imm f 6 4 5 cc/= } + T{ ##compare-imm f 6 4 $[ \ f type-number ] cc/= } T{ ##replace f 6 D 0 } } value-numbering-step trim-temps ] unit-test @@ -100,7 +100,7 @@ IN: compiler.cfg.value-numbering.tests T{ ##load-reference f 1 + } T{ ##peek f 2 D 0 } T{ ##compare f 4 2 1 cc<= } - T{ ##compare-imm f 6 4 5 cc= } + T{ ##compare-imm f 6 4 $[ \ f type-number ] cc= } T{ ##replace f 6 D 0 } } value-numbering-step trim-temps ] unit-test @@ -118,7 +118,7 @@ IN: compiler.cfg.value-numbering.tests T{ ##peek f 8 D 0 } T{ ##peek f 9 D -1 } T{ ##compare-float-unordered f 12 8 9 cc< } - T{ ##compare-imm f 14 12 5 cc= } + T{ ##compare-imm f 14 12 $[ \ f type-number ] cc= } T{ ##replace f 14 D 0 } } value-numbering-step trim-temps ] unit-test @@ -135,7 +135,7 @@ IN: compiler.cfg.value-numbering.tests T{ ##peek f 29 D -1 } T{ ##peek f 30 D -2 } T{ ##compare f 33 29 30 cc<= } - T{ ##compare-imm-branch f 33 5 cc/= } + T{ ##compare-imm-branch f 33 $[ \ f type-number ] cc/= } } value-numbering-step trim-temps ] unit-test @@ -149,7 +149,7 @@ IN: compiler.cfg.value-numbering.tests { T{ ##peek f 1 D -1 } T{ ##test-vector f 2 1 f float-4-rep vcc-any } - T{ ##compare-imm-branch f 2 5 cc/= } + T{ ##compare-imm-branch f 2 $[ \ f type-number ] cc/= } } value-numbering-step trim-temps ] unit-test @@ -1071,14 +1071,14 @@ cell 8 = [ ! Branch folding [ { - T{ ##load-immediate f 1 1 } - T{ ##load-immediate f 2 2 } - T{ ##load-immediate f 3 5 } + T{ ##load-immediate f 1 10 } + T{ ##load-immediate f 2 20 } + T{ ##load-immediate f 3 $[ \ f type-number ] } } ] [ { - T{ ##load-immediate f 1 1 } - T{ ##load-immediate f 2 2 } + T{ ##load-immediate f 1 10 } + T{ ##load-immediate f 2 20 } T{ ##compare f 3 1 2 cc= } } value-numbering-step ] unit-test @@ -1113,14 +1113,14 @@ cell 8 = [ [ { - T{ ##load-immediate f 1 1 } - T{ ##load-immediate f 2 2 } - T{ ##load-immediate f 3 5 } + T{ ##load-immediate f 1 10 } + T{ ##load-immediate f 2 20 } + T{ ##load-immediate f 3 $[ \ f type-number ] } } ] [ { - T{ ##load-immediate f 1 1 } - T{ ##load-immediate f 2 2 } + T{ ##load-immediate f 1 10 } + T{ ##load-immediate f 2 20 } T{ ##compare f 3 2 1 cc< } } value-numbering-step ] unit-test @@ -1128,7 +1128,7 @@ cell 8 = [ [ { T{ ##peek f 0 D 0 } - T{ ##load-immediate f 1 5 } + T{ ##load-immediate f 1 $[ \ f type-number ] } } ] [ { @@ -1152,7 +1152,7 @@ cell 8 = [ [ { T{ ##peek f 0 D 0 } - T{ ##load-immediate f 1 5 } + T{ ##load-immediate f 1 $[ \ f type-number ] } } ] [ { @@ -1176,7 +1176,7 @@ cell 8 = [ [ { T{ ##peek f 0 D 0 } - T{ ##load-immediate f 1 5 } + T{ ##load-immediate f 1 $[ \ f type-number ] } } ] [ { @@ -1557,7 +1557,7 @@ cell 8 = [ { T{ ##peek f 0 D 0 } T{ ##compare f 1 0 0 cc<= } - T{ ##compare-imm-branch f 1 5 cc/= } + T{ ##compare-imm-branch f 1 $[ \ f type-number ] cc/= } } test-branch-folding ] unit-test @@ -1659,7 +1659,7 @@ V{ T{ ##copy { dst 21 } { src 20 } { rep any-rep } } T{ ##compare-imm-branch { src1 21 } - { src2 5 } + { src2 $[ \ f type-number ] } { cc cc/= } } } 1 test-bb diff --git a/basis/compiler/constants/constants.factor b/basis/compiler/constants/constants.factor index a22d522809..19cdb6eebd 100644 --- a/basis/compiler/constants/constants.factor +++ b/basis/compiler/constants/constants.factor @@ -12,19 +12,18 @@ CONSTANT: deck-bits 18 ! These constants must match vm/layouts.h : slot-offset ( slot tag -- n ) [ bootstrap-cells ] dip - ; inline -: header-offset ( -- n ) 0 object tag-number slot-offset ; inline -: float-offset ( -- n ) 8 float tag-number - ; inline -: string-offset ( -- n ) 4 string tag-number slot-offset ; inline -: string-aux-offset ( -- n ) 2 string tag-number slot-offset ; inline -: profile-count-offset ( -- n ) 8 \ word tag-number slot-offset ; inline -: byte-array-offset ( -- n ) 2 byte-array tag-number slot-offset ; inline -: alien-offset ( -- n ) 3 alien tag-number slot-offset ; inline -: underlying-alien-offset ( -- n ) 1 alien tag-number slot-offset ; inline -: tuple-class-offset ( -- n ) 1 tuple tag-number slot-offset ; inline -: word-xt-offset ( -- n ) 10 \ word tag-number slot-offset ; inline -: quot-xt-offset ( -- n ) 4 quotation tag-number slot-offset ; inline -: word-code-offset ( -- n ) 11 \ word tag-number slot-offset ; inline -: array-start-offset ( -- n ) 2 array tag-number slot-offset ; inline +: float-offset ( -- n ) 8 float type-number - ; inline +: string-offset ( -- n ) 4 string type-number slot-offset ; inline +: string-aux-offset ( -- n ) 2 string type-number slot-offset ; inline +: profile-count-offset ( -- n ) 8 \ word type-number slot-offset ; inline +: byte-array-offset ( -- n ) 16 byte-array type-number - ; inline +: alien-offset ( -- n ) 4 alien type-number slot-offset ; inline +: underlying-alien-offset ( -- n ) 1 alien type-number slot-offset ; inline +: tuple-class-offset ( -- n ) 1 tuple type-number slot-offset ; inline +: word-xt-offset ( -- n ) 10 \ word type-number slot-offset ; inline +: quot-xt-offset ( -- n ) 4 quotation type-number slot-offset ; inline +: word-code-offset ( -- n ) 11 \ word type-number slot-offset ; inline +: array-start-offset ( -- n ) 2 array type-number slot-offset ; inline : compiled-header-size ( -- n ) 4 bootstrap-cells ; inline ! Relocation classes diff --git a/basis/compiler/tests/codegen.factor b/basis/compiler/tests/codegen.factor index 18f3a618f6..eba6580574 100644 --- a/basis/compiler/tests/codegen.factor +++ b/basis/compiler/tests/codegen.factor @@ -175,20 +175,6 @@ TUPLE: my-tuple ; ] compile-call ] unit-test -[ 1 t ] [ - B{ 1 2 3 4 } [ - { c-ptr } declare - [ 0 alien-unsigned-1 ] keep hi-tag - ] compile-call byte-array type-number = -] unit-test - -[ t ] [ - B{ 1 2 3 4 } [ - { c-ptr } declare - 0 alien-cell hi-tag - ] compile-call alien type-number = -] unit-test - [ 2 1 ] [ 2 1 [ 2dup fixnum< [ [ die ] dip ] when ] compile-call diff --git a/basis/compiler/tests/intrinsics.factor b/basis/compiler/tests/intrinsics.factor old mode 100644 new mode 100755 index 75cfc1d67f..a26ba5a27a --- a/basis/compiler/tests/intrinsics.factor +++ b/basis/compiler/tests/intrinsics.factor @@ -244,20 +244,20 @@ IN: compiler.tests.intrinsics [ -4294967296 ] [ -1 [ 16 fixnum-shift 16 fixnum-shift ] compile-call ] unit-test [ HEX: 10000000 ] [ HEX: 1000000 HEX: 10 [ fixnum* ] compile-call ] unit-test -[ HEX: 10000000 ] [ HEX: -10000000 >fixnum [ 0 swap fixnum- ] compile-call ] unit-test -[ HEX: 10000000 ] [ HEX: -fffffff >fixnum [ 1 swap fixnum- ] compile-call ] unit-test +[ HEX: 8000000 ] [ HEX: -8000000 >fixnum [ 0 swap fixnum- ] compile-call ] unit-test +[ HEX: 8000000 ] [ HEX: -7ffffff >fixnum [ 1 swap fixnum- ] compile-call ] unit-test -[ t ] [ 1 27 fixnum-shift dup [ fixnum+ ] compile-call 1 28 fixnum-shift = ] unit-test -[ -268435457 ] [ 1 28 shift neg >fixnum [ -1 fixnum+ ] compile-call ] unit-test +[ t ] [ 1 26 fixnum-shift dup [ fixnum+ ] compile-call 1 27 fixnum-shift = ] unit-test +[ -134217729 ] [ 1 27 shift neg >fixnum [ -1 fixnum+ ] compile-call ] unit-test [ t ] [ 1 20 shift 1 20 shift [ fixnum* ] compile-call 1 40 shift = ] unit-test [ t ] [ 1 20 shift neg 1 20 shift [ fixnum* ] compile-call 1 40 shift neg = ] unit-test [ t ] [ 1 20 shift neg 1 20 shift neg [ fixnum* ] compile-call 1 40 shift = ] unit-test [ -351382792 ] [ -43922849 [ 3 fixnum-shift ] compile-call ] unit-test -[ 268435456 ] [ -268435456 >fixnum -1 [ fixnum/i ] compile-call ] unit-test +[ 134217728 ] [ -134217728 >fixnum -1 [ fixnum/i ] compile-call ] unit-test -[ 268435456 0 ] [ -268435456 >fixnum -1 [ fixnum/mod ] compile-call ] unit-test +[ 134217728 0 ] [ -134217728 >fixnum -1 [ fixnum/mod ] compile-call ] unit-test [ t ] [ f [ f eq? ] compile-call ] unit-test @@ -285,8 +285,8 @@ cell 8 = [ ! 64-bit overflow cell 8 = [ - [ t ] [ 1 59 fixnum-shift dup [ fixnum+ ] compile-call 1 60 fixnum-shift = ] unit-test - [ -1152921504606846977 ] [ 1 60 shift neg >fixnum [ -1 fixnum+ ] compile-call ] unit-test + [ t ] [ 1 58 fixnum-shift dup [ fixnum+ ] compile-call 1 59 fixnum-shift = ] unit-test + [ -576460752303423489 ] [ 1 59 shift neg >fixnum [ -1 fixnum+ ] compile-call ] unit-test [ t ] [ 1 40 shift 1 40 shift [ fixnum* ] compile-call 1 80 shift = ] unit-test [ t ] [ 1 40 shift neg 1 40 shift [ fixnum* ] compile-call 1 80 shift neg = ] unit-test @@ -301,9 +301,9 @@ cell 8 = [ [ -18446744073709551616 ] [ -1 [ 64 fixnum-shift ] compile-call ] unit-test [ -18446744073709551616 ] [ -1 [ 32 fixnum-shift 32 fixnum-shift ] compile-call ] unit-test - [ 1152921504606846976 ] [ -1152921504606846976 >fixnum -1 [ fixnum/i ] compile-call ] unit-test + [ 576460752303423488 ] [ -576460752303423488 >fixnum -1 [ fixnum/i ] compile-call ] unit-test - [ 1152921504606846976 0 ] [ -1152921504606846976 >fixnum -1 [ fixnum/mod ] compile-call ] unit-test + [ 576460752303423488 0 ] [ -576460752303423488 >fixnum -1 [ fixnum/mod ] compile-call ] unit-test [ -268435457 ] [ 28 2^ [ fixnum-bitnot ] compile-call ] unit-test ] when @@ -311,12 +311,14 @@ cell 8 = [ ! Some randomized tests : compiled-fixnum* ( a b -- c ) fixnum* ; +ERROR: bug-in-fixnum* x y a b ; + [ ] [ 10000 [ - 32 random-bits >fixnum 32 random-bits >fixnum - 2dup - [ fixnum* ] 2keep compiled-fixnum* = - [ 2drop ] [ "Oops" throw ] if + 32 random-bits >fixnum + 32 random-bits >fixnum + 2dup [ fixnum* ] [ compiled-fixnum* ] 2bi 2dup = + [ 2drop 2drop ] [ bug-in-fixnum* ] if ] times ] unit-test @@ -419,7 +421,7 @@ cell 8 = [ "b" get [ [ 3 ] [ "b" get 2 [ alien-unsigned-1 ] compile-call ] unit-test [ 3 ] [ "b" get [ { alien } declare 2 alien-unsigned-1 ] compile-call ] unit-test - [ 3 ] [ "b" get 2 [ { simple-alien fixnum } declare alien-unsigned-1 ] compile-call ] unit-test + [ 3 ] [ "b" get 2 [ { alien fixnum } declare alien-unsigned-1 ] compile-call ] unit-test [ 3 ] [ "b" get 2 [ { c-ptr fixnum } declare alien-unsigned-1 ] compile-call ] unit-test [ ] [ "b" get free ] unit-test diff --git a/basis/compiler/tests/low-level-ir.factor b/basis/compiler/tests/low-level-ir.factor index 14c470d63f..b6b8e1c031 100644 --- a/basis/compiler/tests/low-level-ir.factor +++ b/basis/compiler/tests/low-level-ir.factor @@ -36,7 +36,7 @@ IN: compiler.tests.low-level-ir ! loading immediates [ f ] [ V{ - T{ ##load-immediate f 0 5 } + T{ ##load-immediate f 0 $[ \ f type-number ] } } compile-test-bb ] unit-test @@ -50,7 +50,7 @@ IN: compiler.tests.low-level-ir ! one of the sources [ t ] [ V{ - T{ ##load-immediate f 1 $[ 2 cell log2 shift array tag-number - ] } + T{ ##load-immediate f 1 $[ 2 cell log2 shift array type-number - ] } T{ ##load-reference f 0 { t f t } } T{ ##slot f 0 0 1 } } compile-test-bb @@ -59,13 +59,13 @@ IN: compiler.tests.low-level-ir [ t ] [ V{ T{ ##load-reference f 0 { t f t } } - T{ ##slot-imm f 0 0 2 $[ array tag-number ] } + T{ ##slot-imm f 0 0 2 $[ array type-number ] } } compile-test-bb ] unit-test [ t ] [ V{ - T{ ##load-immediate f 1 $[ 2 cell log2 shift array tag-number - ] } + T{ ##load-immediate f 1 $[ 2 cell log2 shift array type-number - ] } T{ ##load-reference f 0 { t f t } } T{ ##set-slot f 0 0 1 } } compile-test-bb @@ -75,12 +75,12 @@ IN: compiler.tests.low-level-ir [ t ] [ V{ T{ ##load-reference f 0 { t f t } } - T{ ##set-slot-imm f 0 0 2 $[ array tag-number ] } + T{ ##set-slot-imm f 0 0 2 $[ array type-number ] } } compile-test-bb dup first eq? ] unit-test -[ 8 ] [ +[ 4 ] [ V{ T{ ##load-immediate f 0 4 } T{ ##shl f 0 0 0 } @@ -90,16 +90,16 @@ IN: compiler.tests.low-level-ir [ 4 ] [ V{ T{ ##load-immediate f 0 4 } - T{ ##shl-imm f 0 0 3 } + T{ ##shl-imm f 0 0 4 } } compile-test-bb ] unit-test [ 31 ] [ V{ T{ ##load-reference f 1 B{ 31 67 52 } } - T{ ##unbox-any-c-ptr f 0 1 2 } + T{ ##unbox-any-c-ptr f 0 1 } T{ ##alien-unsigned-1 f 0 0 0 } - T{ ##shl-imm f 0 0 3 } + T{ ##shl-imm f 0 0 4 } } compile-test-bb ] unit-test @@ -108,13 +108,13 @@ IN: compiler.tests.low-level-ir T{ ##load-reference f 0 "hello world" } T{ ##load-immediate f 1 3 } T{ ##string-nth f 0 0 1 2 } - T{ ##shl-imm f 0 0 3 } + T{ ##shl-imm f 0 0 4 } } compile-test-bb ] unit-test [ 1 ] [ V{ - T{ ##load-immediate f 0 16 } - T{ ##add-imm f 0 0 -8 } + T{ ##load-immediate f 0 32 } + T{ ##add-imm f 0 0 -16 } } compile-test-bb ] unit-test diff --git a/basis/compiler/tests/optimizer.factor b/basis/compiler/tests/optimizer.factor index 32f5750cd3..0831d6e8dd 100644 --- a/basis/compiler/tests/optimizer.factor +++ b/basis/compiler/tests/optimizer.factor @@ -202,7 +202,7 @@ USE: binary-search.private dup length 1 <= [ from>> ] [ - [ midpoint swap call ] 3keep roll dup zero? + [ midpoint swap call ] 3keep [ rot ] dip swap dup zero? [ drop dup from>> swap midpoint@ + ] [ drop dup midpoint@ head-slice old-binsearch ] if ] if ; inline recursive diff --git a/basis/compiler/tree/propagation/known-words/known-words.factor b/basis/compiler/tree/propagation/known-words/known-words.factor index aa2bc01f9e..6e906e685a 100644 --- a/basis/compiler/tree/propagation/known-words/known-words.factor +++ b/basis/compiler/tree/propagation/known-words/known-words.factor @@ -278,7 +278,7 @@ generic-comparison-ops [ ] each \ alien-cell [ - 2drop simple-alien \ f class-or + 2drop alien \ f class-or ] "outputs" set-word-prop { } [ diff --git a/basis/compiler/tree/propagation/propagation-tests.factor b/basis/compiler/tree/propagation/propagation-tests.factor index 0f04a5e3d5..3627757acd 100644 --- a/basis/compiler/tree/propagation/propagation-tests.factor +++ b/basis/compiler/tree/propagation/propagation-tests.factor @@ -890,10 +890,10 @@ M: tuple-with-read-only-slot clone [ { 1 2 3 } dup tuple-with-read-only-slot boa clone x>> eq? ] final-classes ] unit-test -! alien-cell outputs a simple-alien or f +! alien-cell outputs a alien or f [ t ] [ [ { byte-array fixnum } declare alien-cell dup [ "OOPS" throw ] unless ] final-classes - first simple-alien class= + first alien class= ] unit-test ! Don't crash if bad literal inputs are passed to unsafe words diff --git a/basis/cpu/architecture/architecture.factor b/basis/cpu/architecture/architecture.factor index 81aea67eb5..b5bb0157af 100644 --- a/basis/cpu/architecture/architecture.factor +++ b/basis/cpu/architecture/architecture.factor @@ -386,9 +386,9 @@ M: object %horizontal-shl-vector-imm-reps { } ; M: object %horizontal-shr-vector-imm-reps { } ; HOOK: %unbox-alien cpu ( dst src -- ) -HOOK: %unbox-any-c-ptr cpu ( dst src temp -- ) +HOOK: %unbox-any-c-ptr cpu ( dst src -- ) HOOK: %box-alien cpu ( dst src temp -- ) -HOOK: %box-displaced-alien cpu ( dst displacement base temp1 temp2 base-class -- ) +HOOK: %box-displaced-alien cpu ( dst displacement base temp base-class -- ) HOOK: %alien-unsigned-1 cpu ( dst src offset -- ) HOOK: %alien-unsigned-2 cpu ( dst src offset -- ) diff --git a/basis/cpu/ppc/bootstrap.factor b/basis/cpu/ppc/bootstrap.factor index cd877cfafe..c16d564e13 100644 --- a/basis/cpu/ppc/bootstrap.factor +++ b/basis/cpu/ppc/bootstrap.factor @@ -69,7 +69,7 @@ CONSTANT: rs-reg 14 [ 3 ds-reg 0 LWZ ds-reg dup 4 SUBI - 0 3 \ f tag-number CMPI + 0 3 \ f type-number CMPI 2 BEQ 0 B rc-relative-ppc-3 rt-xt jit-rel 0 B rc-relative-ppc-3 rt-xt jit-rel @@ -174,40 +174,15 @@ CONSTANT: rs-reg 14 [ load-tag ] pic-tag jit-define -! Hi-tag -[ - 3 4 MR - load-tag - 0 4 object tag-number tag-fixnum CMPI - 2 BNE - 4 3 object tag-number neg LWZ -] pic-hi-tag jit-define - ! Tuple [ 3 4 MR load-tag - 0 4 tuple tag-number tag-fixnum CMPI + 0 4 tuple type-number tag-fixnum CMPI 2 BNE - 4 3 tuple tag-number neg bootstrap-cell + LWZ + 4 3 tuple type-number neg bootstrap-cell + LWZ ] pic-tuple jit-define -! Hi-tag and tuple -[ - 3 4 MR - load-tag - ! If bits 2 and 3 are set, the tag is either 6 (object) or 7 (tuple) - 0 4 BIN: 110 tag-fixnum CMPI - 5 BLT - ! Untag r3 - 3 3 0 0 31 tag-bits get - RLWINM - ! Set r4 to 0 for objects, and bootstrap-cell for tuples - 4 4 1 tag-fixnum ANDI - 4 4 1 SRAWI - ! Load header cell or tuple layout cell - 4 4 3 LWZX -] pic-hi-tag-tuple jit-define - [ 0 4 0 CMPI rc-absolute-ppc-2 rt-immediate jit-rel ] pic-check-tag jit-define @@ -215,7 +190,7 @@ CONSTANT: rs-reg 14 [ 0 5 LOAD32 rc-absolute-ppc-2/2 rt-immediate jit-rel 4 0 5 CMP -] pic-check jit-define +] pic-check-tuple jit-define [ 2 BNE 0 B rc-relative-ppc-3 rt-xt jit-rel ] pic-hit jit-define @@ -224,8 +199,13 @@ CONSTANT: rs-reg 14 [ ! cache = ... 0 3 LOAD32 rc-absolute-ppc-2/2 rt-immediate jit-rel - ! key = class - 5 4 MR + ! key = hashcode(class) + 5 4 3 SRAWI + 6 4 8 SRAWI + 5 5 6 ADD + 6 4 13 SRAWI + 5 5 6 ADD + 5 5 3 SLWI ! key &= cache.length - 1 5 5 mega-cache-size get 1 - bootstrap-cell * ANDI ! cache += array-start-offset @@ -278,7 +258,7 @@ CONSTANT: rs-reg 14 [ 3 ds-reg 0 LWZ 4 ds-reg -4 LWZU - 3 3 1 SRAWI + 3 3 2 SRAWI 4 4 0 0 31 tag-bits get - RLWINM 4 3 3 LWZX 3 ds-reg 0 STW @@ -399,7 +379,7 @@ CONSTANT: rs-reg 14 5 ds-reg -4 LWZU 5 0 4 CMP 2 swap execute( offset -- ) ! magic number - \ f tag-number 3 LI + \ f type-number 3 LI 3 ds-reg 0 STW ; : define-jit-compare ( insn word -- ) @@ -418,7 +398,7 @@ CONSTANT: rs-reg 14 4 ds-reg 0 LWZ 3 3 4 OR 3 3 tag-mask get ANDI - \ f tag-number 4 LI + \ f type-number 4 LI 0 3 0 CMPI 2 BNE 1 tag-fixnum 4 LI diff --git a/basis/cpu/ppc/ppc.factor b/basis/cpu/ppc/ppc.factor index 8ddacaa0e1..0f33df8df7 100644 --- a/basis/cpu/ppc/ppc.factor +++ b/basis/cpu/ppc/ppc.factor @@ -266,7 +266,7 @@ M:: ppc %unbox-any-c-ptr ( dst src temp -- ) ! We come back here with displaced aliens "start" resolve-label ! Is the object f? - 0 scratch-reg \ f tag-number CMPI + 0 scratch-reg \ f type-number CMPI ! If so, done "end" get BEQ ! Is the object an alien? @@ -288,25 +288,20 @@ M:: ppc %unbox-any-c-ptr ( dst src temp -- ) "end" resolve-label ] with-scope ; -: alien@ ( n -- n' ) cells object tag-number - ; - -:: %allot-alien ( dst displacement base temp -- ) - dst 4 cells alien temp %allot - temp \ f tag-number %load-immediate - ! Store underlying-alien slot - base dst 1 alien@ STW - ! Store expired slot - temp dst 2 alien@ STW - ! Store offset - displacement dst 3 alien@ STW ; +: alien@ ( n -- n' ) cells alien type-number - ; M:: ppc %box-alien ( dst src temp -- ) [ "f" define-label - dst \ f tag-number %load-immediate + dst %load-immediate 0 src 0 CMPI "f" get BEQ - dst src temp temp %allot-alien + dst 5 cells alien temp %allot + temp \ f type-number %load-immediate + temp dst 1 alien@ STW + temp dst 2 alien@ STW + displacement dst 3 alien@ STW + displacement dst 4 alien@ STW "f" resolve-label ] with-scope ; @@ -323,7 +318,7 @@ M:: ppc %box-displaced-alien ( dst displacement base displacement' base' base-cl displacement' :> temp dst 4 cells alien temp %allot ! If base is already a displaced alien, unpack it - 0 base \ f tag-number CMPI + 0 base \ f type-number CMPI "simple-case" get BEQ temp base header-offset LWZ 0 temp alien type-number tag-fixnum CMPI @@ -343,7 +338,7 @@ M:: ppc %box-displaced-alien ( dst displacement base displacement' base' base-cl ! Store offset displacement' dst 3 alien@ STW ! Store expired slot (its ok to clobber displacement') - temp \ f tag-number %load-immediate + temp \ f type-number %load-immediate temp dst 2 alien@ STW "end" resolve-label ] with-scope ; @@ -374,7 +369,7 @@ M: ppc %set-alien-double -rot STFD ; [ drop load-zone-ptr ] [ swap 0 LWZ ] 2bi ; :: inc-allot-ptr ( nursery-ptr allot-ptr n -- ) - scratch-reg allot-ptr n 8 align ADDI + scratch-reg allot-ptr n data-alignment get align ADDI scratch-reg nursery-ptr 0 STW ; :: store-header ( dst class -- ) @@ -382,7 +377,7 @@ M: ppc %set-alien-double -rot STFD ; scratch-reg dst 0 STW ; : store-tagged ( dst tag -- ) - dupd tag-number ORI ; + dupd type-number ORI ; M:: ppc %allot ( dst size class nursery-ptr -- ) nursery-ptr dst load-allot-ptr @@ -460,7 +455,7 @@ M: ppc %epilogue ( n -- ) :: (%boolean) ( dst temp branch1 branch2 -- ) "end" define-label - dst \ f tag-number %load-immediate + dst \ f type-number %load-immediate "end" get branch1 execute( label -- ) branch2 [ "end" get branch2 execute( label -- ) ] when dst \ t %load-reference @@ -742,14 +737,3 @@ USE: vocabs.loader } cond "complex-double" c-type t >>return-in-registers? drop - -[ - - [ alien-unsigned-4 c-bool> ] >>getter - [ [ >c-bool ] 2dip set-alien-unsigned-4 ] >>setter - 4 >>size - 4 >>align - "box_boolean" >>boxer - "to_boolean" >>unboxer - bool define-primitive-type -] with-compilation-unit diff --git a/basis/cpu/x86/32/32.factor b/basis/cpu/x86/32/32.factor index cff5c561c8..8867ca6597 100755 --- a/basis/cpu/x86/32/32.factor +++ b/basis/cpu/x86/32/32.factor @@ -11,9 +11,6 @@ cpu.x86.assembler cpu.x86.assembler.operands cpu.x86 cpu.architecture ; IN: cpu.x86.32 -! We implement the FFI for Linux, OS X and Windows all at once. -! OS X requires that the stack be 16-byte aligned. - M: x86.32 machine-registers { { int-regs { EAX ECX EDX EBP EBX } } @@ -327,10 +324,4 @@ M: x86.32 dummy-fp-params? f ; ! Dreadful M: object flatten-value-type (flatten-int-type) ; -os windows? [ - cell longlong c-type (>>align) - cell ulonglong c-type (>>align) - 4 double c-type (>>align) -] unless - check-sse diff --git a/basis/cpu/x86/32/bootstrap.factor b/basis/cpu/x86/32/bootstrap.factor index e532d42dfe..f777040e86 100644 --- a/basis/cpu/x86/32/bootstrap.factor +++ b/basis/cpu/x86/32/bootstrap.factor @@ -21,7 +21,7 @@ IN: bootstrap.x86 : stack-reg ( -- reg ) ESP ; : ds-reg ( -- reg ) ESI ; : rs-reg ( -- reg ) EDI ; -: fixnum>slot@ ( -- ) temp0 1 SAR ; +: fixnum>slot@ ( -- ) temp0 2 SAR ; : rex-length ( -- n ) 0 ; [ diff --git a/basis/cpu/x86/64/bootstrap.factor b/basis/cpu/x86/64/bootstrap.factor index 662eaed3e0..0fc029fdfe 100644 --- a/basis/cpu/x86/64/bootstrap.factor +++ b/basis/cpu/x86/64/bootstrap.factor @@ -18,7 +18,7 @@ IN: bootstrap.x86 : stack-reg ( -- reg ) RSP ; : ds-reg ( -- reg ) R14 ; : rs-reg ( -- reg ) R15 ; -: fixnum>slot@ ( -- ) ; +: fixnum>slot@ ( -- ) temp0 1 SAR ; : rex-length ( -- n ) 1 ; [ diff --git a/basis/cpu/x86/64/winnt/winnt.factor b/basis/cpu/x86/64/winnt/winnt.factor index 3ecd56bdd1..a398c6565c 100644 --- a/basis/cpu/x86/64/winnt/winnt.factor +++ b/basis/cpu/x86/64/winnt/winnt.factor @@ -24,9 +24,3 @@ M: x86.64 dummy-fp-params? t ; M: x86.64 temp-reg RAX ; -<< -longlong ptrdiff_t typedef -longlong intptr_t typedef -int c-type long define-primitive-type -uint c-type ulong define-primitive-type ->> diff --git a/basis/cpu/x86/bootstrap.factor b/basis/cpu/x86/bootstrap.factor index 7930970193..98a5188962 100644 --- a/basis/cpu/x86/bootstrap.factor +++ b/basis/cpu/x86/bootstrap.factor @@ -60,7 +60,7 @@ big-endian off ! pop boolean ds-reg bootstrap-cell SUB ! compare boolean with f - temp0 \ f tag-number CMP + temp0 \ f type-number CMP ! jump to true branch if not equal 0 JNE rc-relative rt-xt jit-rel ! jump to false branch if equal @@ -154,7 +154,7 @@ big-endian off ! ! ! Polymorphic inline caches -! The PIC and megamorphic code stubs are not permitted to touch temp3. +! The PIC stubs are not permitted to touch temp3. ! Load a value from a stack position [ @@ -171,41 +171,15 @@ big-endian off ! The 'make' trick lets us compute the jump distance for the ! conditional branches there -! Hi-tag -[ - temp0 temp1 MOV - load-tag - temp1 object tag-number tag-fixnum CMP - [ temp1 temp0 object tag-number neg [+] MOV ] { } make - [ length JNE ] [ % ] bi -] pic-hi-tag jit-define - ! Tuple [ temp0 temp1 MOV load-tag - temp1 tuple tag-number tag-fixnum CMP - [ temp1 temp0 tuple tag-number neg bootstrap-cell + [+] MOV ] { } make + temp1 tuple type-number tag-fixnum CMP + [ temp1 temp0 tuple type-number neg bootstrap-cell + [+] MOV ] { } make [ length JNE ] [ % ] bi ] pic-tuple jit-define -! Hi-tag and tuple -[ - temp0 temp1 MOV - load-tag - ! If bits 2 and 3 are set, the tag is either 6 (object) or 7 (tuple) - temp1 BIN: 110 tag-fixnum CMP - [ - ! Untag temp0 - temp0 tag-mask get bitnot AND - ! Set temp1 to 0 for objects, and bootstrap-cell for tuples - temp1 1 tag-fixnum AND - bootstrap-cell 4 = [ temp1 1 SHR ] when - ! Load header cell or tuple layout cell - temp1 temp0 temp1 [+] MOV - ] [ ] make [ length JL ] [ % ] bi -] pic-hi-tag-tuple jit-define - [ temp1 HEX: ffffffff CMP rc-absolute rt-immediate jit-rel ] pic-check-tag jit-define @@ -213,7 +187,7 @@ big-endian off [ temp2 HEX: ffffffff MOV rc-absolute-cell rt-immediate jit-rel temp1 temp2 CMP -] pic-check jit-define +] pic-check-tuple jit-define [ 0 JE rc-relative rt-xt jit-rel ] pic-hit jit-define @@ -222,9 +196,9 @@ big-endian off [ ! cache = ... temp0 0 MOV rc-absolute-cell rt-immediate jit-rel - ! key = class + ! key = hashcode(class) temp2 temp1 MOV - bootstrap-cell 8 = [ temp2 1 SHL ] when + bootstrap-cell 4 = [ temp2 1 SHR ] when ! key &= cache.length - 1 temp2 mega-cache-size get 1 - bootstrap-cell * AND ! cache += array-start-offset @@ -410,7 +384,7 @@ big-endian off t jit-literal temp3 0 MOV rc-absolute-cell rt-immediate jit-rel ! load f - temp1 \ f tag-number MOV + temp1 \ f type-number MOV ! load first value temp0 ds-reg [] MOV ! adjust stack pointer @@ -540,7 +514,7 @@ big-endian off ds-reg bootstrap-cell SUB temp0 ds-reg [] OR temp0 tag-mask get AND - temp0 \ f tag-number MOV + temp0 \ f type-number MOV temp1 1 tag-fixnum MOV temp0 temp1 CMOVE ds-reg [] temp0 MOV diff --git a/basis/cpu/x86/x86.factor b/basis/cpu/x86/x86.factor index d78d8c852e..0de9e7d1e4 100644 --- a/basis/cpu/x86/x86.factor +++ b/basis/cpu/x86/x86.factor @@ -45,8 +45,7 @@ HOOK: extra-stack-space cpu ( stack-frame -- n ) : incr-stack-reg ( n -- ) dup 0 = [ drop ] [ stack-reg swap ADD ] if ; -: align-stack ( n -- n' ) - os macosx? cpu x86.64? or [ 16 align ] when ; +: align-stack ( n -- n' ) 16 align ; M: x86 stack-frame-size ( stack-frame -- i ) [ (stack-frame-size) ] @@ -141,8 +140,10 @@ M: x86 %not int-rep one-operand NOT ; M: x86 %neg int-rep one-operand NEG ; M: x86 %log2 BSR ; +! A bit of logic to avoid using MOVSS/MOVSD for reg-reg moves +! since this induces partial register stalls GENERIC: copy-register* ( dst src rep -- ) -GENERIC: copy-unaligned* ( dst src rep -- ) +GENERIC: copy-memory* ( dst src rep -- ) M: int-rep copy-register* drop MOV ; M: tagged-rep copy-register* drop MOV ; @@ -152,17 +153,14 @@ M: float-4-rep copy-register* drop MOVAPS ; M: double-2-rep copy-register* drop MOVAPS ; M: vector-rep copy-register* drop MOVDQA ; -M: object copy-unaligned* copy-register* ; -M: float-rep copy-unaligned* drop MOVSS ; -M: double-rep copy-unaligned* drop MOVSD ; -M: float-4-rep copy-unaligned* drop MOVUPS ; -M: double-2-rep copy-unaligned* drop MOVUPS ; -M: vector-rep copy-unaligned* drop MOVDQU ; +M: object copy-memory* copy-register* ; +M: float-rep copy-memory* drop MOVSS ; +M: double-rep copy-memory* drop MOVSD ; M: x86 %copy ( dst src rep -- ) 2over eq? [ 3drop ] [ [ [ dup spill-slot? [ n>> spill@ ] when ] bi@ ] dip - 2over [ register? ] both? [ copy-register* ] [ copy-unaligned* ] if + 2over [ register? ] both? [ copy-register* ] [ copy-memory* ] if ] if ; M: x86 %fixnum-add ( label dst src1 src2 -- ) @@ -177,76 +175,109 @@ M: x86 %fixnum-mul ( label dst src1 src2 -- ) M: x86 %unbox-alien ( dst src -- ) alien-offset [+] MOV ; -M:: x86 %unbox-any-c-ptr ( dst src temp -- ) +M:: x86 %unbox-any-c-ptr ( dst src -- ) [ - { "is-byte-array" "end" "start" } [ define-label ] each - dst 0 MOV - temp src MOV - ! We come back here with displaced aliens - "start" resolve-label + "end" define-label + dst dst XOR ! Is the object f? - temp \ f tag-number CMP + src \ f type-number CMP "end" get JE + ! Compute tag in dst register + dst src MOV + dst tag-mask get AND ! Is the object an alien? - temp header-offset [+] alien type-number tag-fixnum CMP - "is-byte-array" get JNE - ! If so, load the offset and add it to the address - dst temp alien-offset [+] ADD - ! Now recurse on the underlying alien - temp temp underlying-alien-offset [+] MOV - "start" get JMP - "is-byte-array" resolve-label - ! Add byte array address to address being computed - dst temp ADD + dst alien type-number CMP ! Add an offset to start of byte array's data - dst byte-array-offset ADD + dst src byte-array-offset [+] LEA + "end" get JNE + ! If so, load the offset and add it to the address + dst src alien-offset [+] MOV "end" resolve-label ] with-scope ; -: alien@ ( reg n -- op ) cells alien tag-number - [+] ; - -:: %allot-alien ( dst displacement base temp -- ) - dst 4 cells alien temp %allot - dst 1 alien@ base MOV ! alien - dst 2 alien@ \ f tag-number MOV ! expired - dst 3 alien@ displacement MOV ! displacement - ; +: alien@ ( reg n -- op ) cells alien type-number - [+] ; M:: x86 %box-alien ( dst src temp -- ) [ "end" define-label - dst \ f tag-number MOV - src 0 CMP + dst \ f type-number MOV + src src TEST "end" get JE - dst src \ f tag-number temp %allot-alien + dst 5 cells alien temp %allot + dst 1 alien@ \ f type-number MOV ! base + dst 2 alien@ \ f type-number MOV ! expired + dst 3 alien@ src MOV ! displacement + dst 4 alien@ src MOV ! address "end" resolve-label ] with-scope ; -M:: x86 %box-displaced-alien ( dst displacement base displacement' base' base-class -- ) +M:: x86 %box-displaced-alien ( dst displacement base temp base-class -- ) + ! This is ridiculous [ "end" define-label - "ok" define-label + "not-f" define-label + "not-alien" define-label + ! If displacement is zero, return the base dst base MOV - displacement 0 CMP + displacement displacement TEST "end" get JE - ! Quickly use displacement' before its needed for real, as allot temporary - dst 4 cells alien displacement' %allot - ! If base is already a displaced alien, unpack it - base' base MOV - displacement' displacement MOV - base \ f tag-number CMP - "ok" get JE - base header-offset [+] alien type-number tag-fixnum CMP - "ok" get JNE - ! displacement += base.displacement - displacement' base 3 alien@ ADD - ! base = base.base - base' base 1 alien@ MOV - "ok" resolve-label - dst 1 alien@ base' MOV ! alien - dst 2 alien@ \ f tag-number MOV ! expired - dst 3 alien@ displacement' MOV ! displacement + + ! Displacement is non-zero, we're going to be allocating a new + ! object + dst 5 cells alien temp %allot + + ! Set expired to f + dst 2 alien@ \ f type-number MOV + + ! Is base f? + base \ f type-number CMP + "not-f" get JNE + + ! Yes, it is f. Fill in new object + dst 1 alien@ base MOV + dst 3 alien@ displacement MOV + dst 4 alien@ displacement MOV + + "end" get JMP + + "not-f" resolve-label + + ! Check base type + temp base MOV + temp tag-mask get AND + + ! Is base an alien? + temp alien type-number CMP + "not-alien" get JNE + + ! Yes, it is an alien. Set new alien's base to base.base + temp base 1 alien@ MOV + dst 1 alien@ temp MOV + + ! Compute displacement + temp base 3 alien@ MOV + temp displacement ADD + dst 3 alien@ temp MOV + + ! Compute address + temp base 4 alien@ MOV + temp displacement ADD + dst 4 alien@ temp MOV + + ! We are done + "end" get JMP + + ! Is base a byte array? It has to be, by now... + "not-alien" resolve-label + + dst 1 alien@ base MOV + dst 3 alien@ displacement MOV + temp base MOV + temp byte-array-offset ADD + temp displacement ADD + dst 4 alien@ temp MOV + "end" resolve-label ] with-scope ; @@ -396,13 +427,13 @@ M: x86 %vm-field-ptr ( dst field -- ) [ drop "nursery" %vm-field-ptr ] [ swap [] MOV ] 2bi ; : inc-allot-ptr ( nursery-ptr n -- ) - [ [] ] dip 8 align ADD ; + [ [] ] dip data-alignment get align ADD ; : store-header ( temp class -- ) [ [] ] [ type-number tag-fixnum ] bi* MOV ; : store-tagged ( dst tag -- ) - tag-number OR ; + type-number OR ; M:: x86 %allot ( dst size class nursery-ptr -- ) nursery-ptr dst load-allot-ptr @@ -444,7 +475,7 @@ M: x86 %alien-global ( dst symbol library -- ) M: x86 %epilogue ( n -- ) cell - incr-stack-reg ; :: %boolean ( dst temp word -- ) - dst \ f tag-number MOV + dst \ f type-number MOV temp 0 MOV \ t rc-absolute-cell rel-immediate dst temp word execute ; inline diff --git a/basis/game/input/dinput/keys-array/keys-array.factor b/basis/game/input/dinput/keys-array/keys-array.factor index 3426b89141..b9f21f70a2 100755 --- a/basis/game/input/dinput/keys-array/keys-array.factor +++ b/basis/game/input/dinput/keys-array/keys-array.factor @@ -1,5 +1,5 @@ USING: sequences sequences.private math -accessors alien.data ; +accessors alien.c-types ; IN: game.input.dinput.keys-array TUPLE: keys-array diff --git a/extra/half-floats/authors.txt b/basis/half-floats/authors.txt similarity index 100% rename from extra/half-floats/authors.txt rename to basis/half-floats/authors.txt diff --git a/extra/half-floats/half-floats-tests.factor b/basis/half-floats/half-floats-tests.factor similarity index 100% rename from extra/half-floats/half-floats-tests.factor rename to basis/half-floats/half-floats-tests.factor diff --git a/extra/half-floats/half-floats.factor b/basis/half-floats/half-floats.factor similarity index 100% rename from extra/half-floats/half-floats.factor rename to basis/half-floats/half-floats.factor diff --git a/extra/half-floats/summary.txt b/basis/half-floats/summary.txt similarity index 100% rename from extra/half-floats/summary.txt rename to basis/half-floats/summary.txt diff --git a/extra/images/normalization/authors.txt b/basis/images/normalization/authors.txt similarity index 100% rename from extra/images/normalization/authors.txt rename to basis/images/normalization/authors.txt diff --git a/extra/images/normalization/normalization-docs.factor b/basis/images/normalization/normalization-docs.factor similarity index 100% rename from extra/images/normalization/normalization-docs.factor rename to basis/images/normalization/normalization-docs.factor diff --git a/extra/images/normalization/normalization-tests.factor b/basis/images/normalization/normalization-tests.factor similarity index 100% rename from extra/images/normalization/normalization-tests.factor rename to basis/images/normalization/normalization-tests.factor diff --git a/extra/images/normalization/normalization.factor b/basis/images/normalization/normalization.factor similarity index 100% rename from extra/images/normalization/normalization.factor rename to basis/images/normalization/normalization.factor diff --git a/basis/io/buffers/buffers.factor b/basis/io/buffers/buffers.factor index 91524dd6e1..f45d3bb062 100644 --- a/basis/io/buffers/buffers.factor +++ b/basis/io/buffers/buffers.factor @@ -8,7 +8,7 @@ IN: io.buffers TUPLE: buffer { size fixnum } -{ ptr simple-alien } +{ ptr alien } { fill fixnum } { pos fixnum } disposed ; diff --git a/basis/io/encodings/8-bit/8-bit.factor b/basis/io/encodings/8-bit/8-bit.factor index bba22268c6..c13bbccd43 100644 --- a/basis/io/encodings/8-bit/8-bit.factor +++ b/basis/io/encodings/8-bit/8-bit.factor @@ -27,6 +27,7 @@ CONSTANT: mappings { { "latin9" "ISO-8859-15" "8859-15" } { "latin10" "ISO-8859-16" "8859-16" } { "koi8-r" "KOI8-R" "KOI8-R" } + { "windows-1250" "windows-1250" "CP1250" } { "windows-1252" "windows-1252" "CP1252" } { "ebcdic" "IBM037" "CP037" } { "mac-roman" "macintosh" "ROMAN" } diff --git a/basis/io/encodings/8-bit/CP1250.TXT b/basis/io/encodings/8-bit/CP1250.TXT new file mode 100644 index 0000000000..6bfab9380d --- /dev/null +++ b/basis/io/encodings/8-bit/CP1250.TXT @@ -0,0 +1,274 @@ +# +# Name: cp1250 to Unicode table +# Unicode version: 2.0 +# Table version: 2.01 +# Table format: Format A +# Date: 04/15/98 +# +# Contact: Shawn.Steele@microsoft.com +# +# General notes: none +# +# Format: Three tab-separated columns +# Column #1 is the cp1250 code (in hex) +# Column #2 is the Unicode (in hex as 0xXXXX) +# Column #3 is the Unicode name (follows a comment sign, '#') +# +# The entries are in cp1250 order +# +0x00 0x0000 #NULL +0x01 0x0001 #START OF HEADING +0x02 0x0002 #START OF TEXT +0x03 0x0003 #END OF TEXT +0x04 0x0004 #END OF TRANSMISSION +0x05 0x0005 #ENQUIRY +0x06 0x0006 #ACKNOWLEDGE +0x07 0x0007 #BELL +0x08 0x0008 #BACKSPACE +0x09 0x0009 #HORIZONTAL TABULATION +0x0A 0x000A #LINE FEED +0x0B 0x000B #VERTICAL TABULATION +0x0C 0x000C #FORM FEED +0x0D 0x000D #CARRIAGE RETURN +0x0E 0x000E #SHIFT OUT +0x0F 0x000F #SHIFT IN +0x10 0x0010 #DATA LINK ESCAPE +0x11 0x0011 #DEVICE CONTROL ONE +0x12 0x0012 #DEVICE CONTROL TWO +0x13 0x0013 #DEVICE CONTROL THREE +0x14 0x0014 #DEVICE CONTROL FOUR +0x15 0x0015 #NEGATIVE ACKNOWLEDGE +0x16 0x0016 #SYNCHRONOUS IDLE +0x17 0x0017 #END OF TRANSMISSION BLOCK +0x18 0x0018 #CANCEL +0x19 0x0019 #END OF MEDIUM +0x1A 0x001A #SUBSTITUTE +0x1B 0x001B #ESCAPE +0x1C 0x001C #FILE SEPARATOR +0x1D 0x001D #GROUP SEPARATOR +0x1E 0x001E #RECORD SEPARATOR +0x1F 0x001F #UNIT SEPARATOR +0x20 0x0020 #SPACE +0x21 0x0021 #EXCLAMATION MARK +0x22 0x0022 #QUOTATION MARK +0x23 0x0023 #NUMBER SIGN +0x24 0x0024 #DOLLAR SIGN +0x25 0x0025 #PERCENT SIGN +0x26 0x0026 #AMPERSAND +0x27 0x0027 #APOSTROPHE +0x28 0x0028 #LEFT PARENTHESIS +0x29 0x0029 #RIGHT PARENTHESIS +0x2A 0x002A #ASTERISK +0x2B 0x002B #PLUS SIGN +0x2C 0x002C #COMMA +0x2D 0x002D #HYPHEN-MINUS +0x2E 0x002E #FULL STOP +0x2F 0x002F #SOLIDUS +0x30 0x0030 #DIGIT ZERO +0x31 0x0031 #DIGIT ONE +0x32 0x0032 #DIGIT TWO +0x33 0x0033 #DIGIT THREE +0x34 0x0034 #DIGIT FOUR +0x35 0x0035 #DIGIT FIVE +0x36 0x0036 #DIGIT SIX +0x37 0x0037 #DIGIT SEVEN +0x38 0x0038 #DIGIT EIGHT +0x39 0x0039 #DIGIT NINE +0x3A 0x003A #COLON +0x3B 0x003B #SEMICOLON +0x3C 0x003C #LESS-THAN SIGN +0x3D 0x003D #EQUALS SIGN +0x3E 0x003E #GREATER-THAN SIGN +0x3F 0x003F #QUESTION MARK +0x40 0x0040 #COMMERCIAL AT +0x41 0x0041 #LATIN CAPITAL LETTER A +0x42 0x0042 #LATIN CAPITAL LETTER B +0x43 0x0043 #LATIN CAPITAL LETTER C +0x44 0x0044 #LATIN CAPITAL LETTER D +0x45 0x0045 #LATIN CAPITAL LETTER E +0x46 0x0046 #LATIN CAPITAL LETTER F +0x47 0x0047 #LATIN CAPITAL LETTER G +0x48 0x0048 #LATIN CAPITAL LETTER H +0x49 0x0049 #LATIN CAPITAL LETTER I +0x4A 0x004A #LATIN CAPITAL LETTER J +0x4B 0x004B #LATIN CAPITAL LETTER K +0x4C 0x004C #LATIN CAPITAL LETTER L +0x4D 0x004D #LATIN CAPITAL LETTER M +0x4E 0x004E #LATIN CAPITAL LETTER N +0x4F 0x004F #LATIN CAPITAL LETTER O +0x50 0x0050 #LATIN CAPITAL LETTER P +0x51 0x0051 #LATIN CAPITAL LETTER Q +0x52 0x0052 #LATIN CAPITAL LETTER R +0x53 0x0053 #LATIN CAPITAL LETTER S +0x54 0x0054 #LATIN CAPITAL LETTER T +0x55 0x0055 #LATIN CAPITAL LETTER U +0x56 0x0056 #LATIN CAPITAL LETTER V +0x57 0x0057 #LATIN CAPITAL LETTER W +0x58 0x0058 #LATIN CAPITAL LETTER X +0x59 0x0059 #LATIN CAPITAL LETTER Y +0x5A 0x005A #LATIN CAPITAL LETTER Z +0x5B 0x005B #LEFT SQUARE BRACKET +0x5C 0x005C #REVERSE SOLIDUS +0x5D 0x005D #RIGHT SQUARE BRACKET +0x5E 0x005E #CIRCUMFLEX ACCENT +0x5F 0x005F #LOW LINE +0x60 0x0060 #GRAVE ACCENT +0x61 0x0061 #LATIN SMALL LETTER A +0x62 0x0062 #LATIN SMALL LETTER B +0x63 0x0063 #LATIN SMALL LETTER C +0x64 0x0064 #LATIN SMALL LETTER D +0x65 0x0065 #LATIN SMALL LETTER E +0x66 0x0066 #LATIN SMALL LETTER F +0x67 0x0067 #LATIN SMALL LETTER G +0x68 0x0068 #LATIN SMALL LETTER H +0x69 0x0069 #LATIN SMALL LETTER I +0x6A 0x006A #LATIN SMALL LETTER J +0x6B 0x006B #LATIN SMALL LETTER K +0x6C 0x006C #LATIN SMALL LETTER L +0x6D 0x006D #LATIN SMALL LETTER M +0x6E 0x006E #LATIN SMALL LETTER N +0x6F 0x006F #LATIN SMALL LETTER O +0x70 0x0070 #LATIN SMALL LETTER P +0x71 0x0071 #LATIN SMALL LETTER Q +0x72 0x0072 #LATIN SMALL LETTER R +0x73 0x0073 #LATIN SMALL LETTER S +0x74 0x0074 #LATIN SMALL LETTER T +0x75 0x0075 #LATIN SMALL LETTER U +0x76 0x0076 #LATIN SMALL LETTER V +0x77 0x0077 #LATIN SMALL LETTER W +0x78 0x0078 #LATIN SMALL LETTER X +0x79 0x0079 #LATIN SMALL LETTER Y +0x7A 0x007A #LATIN SMALL LETTER Z +0x7B 0x007B #LEFT CURLY BRACKET +0x7C 0x007C #VERTICAL LINE +0x7D 0x007D #RIGHT CURLY BRACKET +0x7E 0x007E #TILDE +0x7F 0x007F #DELETE +0x80 0x20AC #EURO SIGN +0x81 #UNDEFINED +0x82 0x201A #SINGLE LOW-9 QUOTATION MARK +0x83 #UNDEFINED +0x84 0x201E #DOUBLE LOW-9 QUOTATION MARK +0x85 0x2026 #HORIZONTAL ELLIPSIS +0x86 0x2020 #DAGGER +0x87 0x2021 #DOUBLE DAGGER +0x88 #UNDEFINED +0x89 0x2030 #PER MILLE SIGN +0x8A 0x0160 #LATIN CAPITAL LETTER S WITH CARON +0x8B 0x2039 #SINGLE LEFT-POINTING ANGLE QUOTATION MARK +0x8C 0x015A #LATIN CAPITAL LETTER S WITH ACUTE +0x8D 0x0164 #LATIN CAPITAL LETTER T WITH CARON +0x8E 0x017D #LATIN CAPITAL LETTER Z WITH CARON +0x8F 0x0179 #LATIN CAPITAL LETTER Z WITH ACUTE +0x90 #UNDEFINED +0x91 0x2018 #LEFT SINGLE QUOTATION MARK +0x92 0x2019 #RIGHT SINGLE QUOTATION MARK +0x93 0x201C #LEFT DOUBLE QUOTATION MARK +0x94 0x201D #RIGHT DOUBLE QUOTATION MARK +0x95 0x2022 #BULLET +0x96 0x2013 #EN DASH +0x97 0x2014 #EM DASH +0x98 #UNDEFINED +0x99 0x2122 #TRADE MARK SIGN +0x9A 0x0161 #LATIN SMALL LETTER S WITH CARON +0x9B 0x203A #SINGLE RIGHT-POINTING ANGLE QUOTATION MARK +0x9C 0x015B #LATIN SMALL LETTER S WITH ACUTE +0x9D 0x0165 #LATIN SMALL LETTER T WITH CARON +0x9E 0x017E #LATIN SMALL LETTER Z WITH CARON +0x9F 0x017A #LATIN SMALL LETTER Z WITH ACUTE +0xA0 0x00A0 #NO-BREAK SPACE +0xA1 0x02C7 #CARON +0xA2 0x02D8 #BREVE +0xA3 0x0141 #LATIN CAPITAL LETTER L WITH STROKE +0xA4 0x00A4 #CURRENCY SIGN +0xA5 0x0104 #LATIN CAPITAL LETTER A WITH OGONEK +0xA6 0x00A6 #BROKEN BAR +0xA7 0x00A7 #SECTION SIGN +0xA8 0x00A8 #DIAERESIS +0xA9 0x00A9 #COPYRIGHT SIGN +0xAA 0x015E #LATIN CAPITAL LETTER S WITH CEDILLA +0xAB 0x00AB #LEFT-POINTING DOUBLE ANGLE QUOTATION MARK +0xAC 0x00AC #NOT SIGN +0xAD 0x00AD #SOFT HYPHEN +0xAE 0x00AE #REGISTERED SIGN +0xAF 0x017B #LATIN CAPITAL LETTER Z WITH DOT ABOVE +0xB0 0x00B0 #DEGREE SIGN +0xB1 0x00B1 #PLUS-MINUS SIGN +0xB2 0x02DB #OGONEK +0xB3 0x0142 #LATIN SMALL LETTER L WITH STROKE +0xB4 0x00B4 #ACUTE ACCENT +0xB5 0x00B5 #MICRO SIGN +0xB6 0x00B6 #PILCROW SIGN +0xB7 0x00B7 #MIDDLE DOT +0xB8 0x00B8 #CEDILLA +0xB9 0x0105 #LATIN SMALL LETTER A WITH OGONEK +0xBA 0x015F #LATIN SMALL LETTER S WITH CEDILLA +0xBB 0x00BB #RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK +0xBC 0x013D #LATIN CAPITAL LETTER L WITH CARON +0xBD 0x02DD #DOUBLE ACUTE ACCENT +0xBE 0x013E #LATIN SMALL LETTER L WITH CARON +0xBF 0x017C #LATIN SMALL LETTER Z WITH DOT ABOVE +0xC0 0x0154 #LATIN CAPITAL LETTER R WITH ACUTE +0xC1 0x00C1 #LATIN CAPITAL LETTER A WITH ACUTE +0xC2 0x00C2 #LATIN CAPITAL LETTER A WITH CIRCUMFLEX +0xC3 0x0102 #LATIN CAPITAL LETTER A WITH BREVE +0xC4 0x00C4 #LATIN CAPITAL LETTER A WITH DIAERESIS +0xC5 0x0139 #LATIN CAPITAL LETTER L WITH ACUTE +0xC6 0x0106 #LATIN CAPITAL LETTER C WITH ACUTE +0xC7 0x00C7 #LATIN CAPITAL LETTER C WITH CEDILLA +0xC8 0x010C #LATIN CAPITAL LETTER C WITH CARON +0xC9 0x00C9 #LATIN CAPITAL LETTER E WITH ACUTE +0xCA 0x0118 #LATIN CAPITAL LETTER E WITH OGONEK +0xCB 0x00CB #LATIN CAPITAL LETTER E WITH DIAERESIS +0xCC 0x011A #LATIN CAPITAL LETTER E WITH CARON +0xCD 0x00CD #LATIN CAPITAL LETTER I WITH ACUTE +0xCE 0x00CE #LATIN CAPITAL LETTER I WITH CIRCUMFLEX +0xCF 0x010E #LATIN CAPITAL LETTER D WITH CARON +0xD0 0x0110 #LATIN CAPITAL LETTER D WITH STROKE +0xD1 0x0143 #LATIN CAPITAL LETTER N WITH ACUTE +0xD2 0x0147 #LATIN CAPITAL LETTER N WITH CARON +0xD3 0x00D3 #LATIN CAPITAL LETTER O WITH ACUTE +0xD4 0x00D4 #LATIN CAPITAL LETTER O WITH CIRCUMFLEX +0xD5 0x0150 #LATIN CAPITAL LETTER O WITH DOUBLE ACUTE +0xD6 0x00D6 #LATIN CAPITAL LETTER O WITH DIAERESIS +0xD7 0x00D7 #MULTIPLICATION SIGN +0xD8 0x0158 #LATIN CAPITAL LETTER R WITH CARON +0xD9 0x016E #LATIN CAPITAL LETTER U WITH RING ABOVE +0xDA 0x00DA #LATIN CAPITAL LETTER U WITH ACUTE +0xDB 0x0170 #LATIN CAPITAL LETTER U WITH DOUBLE ACUTE +0xDC 0x00DC #LATIN CAPITAL LETTER U WITH DIAERESIS +0xDD 0x00DD #LATIN CAPITAL LETTER Y WITH ACUTE +0xDE 0x0162 #LATIN CAPITAL LETTER T WITH CEDILLA +0xDF 0x00DF #LATIN SMALL LETTER SHARP S +0xE0 0x0155 #LATIN SMALL LETTER R WITH ACUTE +0xE1 0x00E1 #LATIN SMALL LETTER A WITH ACUTE +0xE2 0x00E2 #LATIN SMALL LETTER A WITH CIRCUMFLEX +0xE3 0x0103 #LATIN SMALL LETTER A WITH BREVE +0xE4 0x00E4 #LATIN SMALL LETTER A WITH DIAERESIS +0xE5 0x013A #LATIN SMALL LETTER L WITH ACUTE +0xE6 0x0107 #LATIN SMALL LETTER C WITH ACUTE +0xE7 0x00E7 #LATIN SMALL LETTER C WITH CEDILLA +0xE8 0x010D #LATIN SMALL LETTER C WITH CARON +0xE9 0x00E9 #LATIN SMALL LETTER E WITH ACUTE +0xEA 0x0119 #LATIN SMALL LETTER E WITH OGONEK +0xEB 0x00EB #LATIN SMALL LETTER E WITH DIAERESIS +0xEC 0x011B #LATIN SMALL LETTER E WITH CARON +0xED 0x00ED #LATIN SMALL LETTER I WITH ACUTE +0xEE 0x00EE #LATIN SMALL LETTER I WITH CIRCUMFLEX +0xEF 0x010F #LATIN SMALL LETTER D WITH CARON +0xF0 0x0111 #LATIN SMALL LETTER D WITH STROKE +0xF1 0x0144 #LATIN SMALL LETTER N WITH ACUTE +0xF2 0x0148 #LATIN SMALL LETTER N WITH CARON +0xF3 0x00F3 #LATIN SMALL LETTER O WITH ACUTE +0xF4 0x00F4 #LATIN SMALL LETTER O WITH CIRCUMFLEX +0xF5 0x0151 #LATIN SMALL LETTER O WITH DOUBLE ACUTE +0xF6 0x00F6 #LATIN SMALL LETTER O WITH DIAERESIS +0xF7 0x00F7 #DIVISION SIGN +0xF8 0x0159 #LATIN SMALL LETTER R WITH CARON +0xF9 0x016F #LATIN SMALL LETTER U WITH RING ABOVE +0xFA 0x00FA #LATIN SMALL LETTER U WITH ACUTE +0xFB 0x0171 #LATIN SMALL LETTER U WITH DOUBLE ACUTE +0xFC 0x00FC #LATIN SMALL LETTER U WITH DIAERESIS +0xFD 0x00FD #LATIN SMALL LETTER Y WITH ACUTE +0xFE 0x0163 #LATIN SMALL LETTER T WITH CEDILLA +0xFF 0x02D9 #DOT ABOVE diff --git a/basis/listener/listener.factor b/basis/listener/listener.factor index 57d1fd3964..a42eada563 100644 --- a/basis/listener/listener.factor +++ b/basis/listener/listener.factor @@ -163,8 +163,10 @@ SYMBOL: interactive-vocabs "syntax" "tools.annotations" "tools.crossref" + "tools.deprecation" "tools.destructors" "tools.disassembler" + "tools.dispatch" "tools.errors" "tools.memory" "tools.profiler" diff --git a/basis/math/vectors/simd/simd-tests.factor b/basis/math/vectors/simd/simd-tests.factor index 7ba9f243ce..396b8da22a 100644 --- a/basis/math/vectors/simd/simd-tests.factor +++ b/basis/math/vectors/simd/simd-tests.factor @@ -582,3 +582,20 @@ STRUCT: simd-struct float-4{ 1.0 0.0 1.0 0.0 } pi [ broken 3array ] [ compile-call ] [ call ] 3bi = ] unit-test + +! Spilling SIMD values -- this basically just tests that the +! stack was aligned properly by the runtime + +: simd-spill-test-1 ( a b c -- v ) + { float-4 float-4 float } declare + [ v+ ] dip sin v*n ; + +[ float-4{ 0 0 0 0 } ] +[ float-4{ 1 2 3 4 } float-4{ 4 5 6 7 } 0.0 simd-spill-test-1 ] unit-test + +: simd-spill-test-2 ( a b d c -- v ) + { float float-4 float-4 float } declare + [ [ 3.0 + ] 2dip v+ ] dip sin v*n n*v ; + +[ float-4{ 0 0 0 0 } ] +[ 5.0 float-4{ 1 2 3 4 } float-4{ 4 5 6 7 } 0.0 simd-spill-test-2 ] unit-test diff --git a/basis/prettyprint/prettyprint.factor b/basis/prettyprint/prettyprint.factor index 718de7e84c..6cff399201 100644 --- a/basis/prettyprint/prettyprint.factor +++ b/basis/prettyprint/prettyprint.factor @@ -110,3 +110,7 @@ SYMBOL: pprint-string-cells? ] with-row ] each ] tabular-output nl ; + +: object-table. ( obj alist -- ) + [ [ nip first ] [ second call( obj -- str ) ] 2bi 2array ] with map + simple-table. ; diff --git a/extra/sequences/merged/authors.txt b/basis/sequences/merged/authors.txt similarity index 100% rename from extra/sequences/merged/authors.txt rename to basis/sequences/merged/authors.txt diff --git a/extra/sequences/merged/merged-docs.factor b/basis/sequences/merged/merged-docs.factor similarity index 100% rename from extra/sequences/merged/merged-docs.factor rename to basis/sequences/merged/merged-docs.factor diff --git a/extra/sequences/merged/merged-tests.factor b/basis/sequences/merged/merged-tests.factor similarity index 100% rename from extra/sequences/merged/merged-tests.factor rename to basis/sequences/merged/merged-tests.factor diff --git a/extra/sequences/merged/merged.factor b/basis/sequences/merged/merged.factor similarity index 100% rename from extra/sequences/merged/merged.factor rename to basis/sequences/merged/merged.factor diff --git a/extra/sequences/merged/summary.txt b/basis/sequences/merged/summary.txt similarity index 100% rename from extra/sequences/merged/summary.txt rename to basis/sequences/merged/summary.txt diff --git a/extra/sequences/merged/tags.txt b/basis/sequences/merged/tags.txt similarity index 100% rename from extra/sequences/merged/tags.txt rename to basis/sequences/merged/tags.txt diff --git a/extra/sequences/product/authors.txt b/basis/sequences/product/authors.txt similarity index 100% rename from extra/sequences/product/authors.txt rename to basis/sequences/product/authors.txt diff --git a/extra/sequences/product/product-docs.factor b/basis/sequences/product/product-docs.factor similarity index 100% rename from extra/sequences/product/product-docs.factor rename to basis/sequences/product/product-docs.factor diff --git a/extra/sequences/product/product-tests.factor b/basis/sequences/product/product-tests.factor similarity index 100% rename from extra/sequences/product/product-tests.factor rename to basis/sequences/product/product-tests.factor diff --git a/extra/sequences/product/product.factor b/basis/sequences/product/product.factor similarity index 100% rename from extra/sequences/product/product.factor rename to basis/sequences/product/product.factor diff --git a/extra/sequences/product/summary.txt b/basis/sequences/product/summary.txt similarity index 100% rename from extra/sequences/product/summary.txt rename to basis/sequences/product/summary.txt diff --git a/basis/stack-checker/known-words/known-words.factor b/basis/stack-checker/known-words/known-words.factor index 62a9526e20..2c0ce853aa 100644 --- a/basis/stack-checker/known-words/known-words.factor +++ b/basis/stack-checker/known-words/known-words.factor @@ -13,7 +13,7 @@ words.private definitions assocs summary compiler.units system.private combinators combinators.short-circuit locals locals.backend locals.types combinators.private stack-checker.values generic.single generic.single.private -alien.libraries +alien.libraries tools.dispatch.private tools.profiler.private stack-checker.alien stack-checker.state stack-checker.errors @@ -501,16 +501,14 @@ M: bad-executable summary \ compact-gc { } { } define-primitive -\ gc-stats { } { array } define-primitive - \ (save-image) { byte-array } { } define-primitive \ (save-image-and-exit) { byte-array } { } define-primitive -\ data-room { } { integer integer array } define-primitive +\ data-room { } { byte-array } define-primitive \ data-room make-flushable -\ code-room { } { integer integer integer integer } define-primitive +\ code-room { } { byte-array } define-primitive \ code-room make-flushable \ micros { } { integer } define-primitive @@ -594,7 +592,7 @@ M: bad-executable summary \ set-alien-double { float c-ptr integer } { } define-primitive -\ alien-cell { c-ptr integer } { simple-c-ptr } define-primitive +\ alien-cell { c-ptr integer } { pinned-c-ptr } define-primitive \ alien-cell make-flushable \ set-alien-cell { c-ptr c-ptr integer } { } define-primitive @@ -701,21 +699,20 @@ M: bad-executable summary \ unimplemented { } { } define-primitive -\ gc-reset { } { } define-primitive - -\ gc-stats { } { array } define-primitive - \ jit-compile { quotation } { } define-primitive \ lookup-method { object array } { word } define-primitive \ reset-dispatch-stats { } { } define-primitive \ dispatch-stats { } { array } define-primitive -\ reset-inline-cache-stats { } { } define-primitive -\ inline-cache-stats { } { array } define-primitive \ optimized? { word } { object } define-primitive \ strip-stack-traces { } { } define-primitive \ { word } { alien } define-primitive + +\ enable-gc-events { } { } define-primitive +\ disable-gc-events { } { object } define-primitive + +\ profiling { object } { } define-primitive diff --git a/basis/tools/dispatch/authors.txt b/basis/tools/dispatch/authors.txt new file mode 100644 index 0000000000..d4f5d6b3ae --- /dev/null +++ b/basis/tools/dispatch/authors.txt @@ -0,0 +1 @@ +Slava Pestov \ No newline at end of file diff --git a/basis/tools/dispatch/dispatch-docs.factor b/basis/tools/dispatch/dispatch-docs.factor new file mode 100644 index 0000000000..e93ea165c1 --- /dev/null +++ b/basis/tools/dispatch/dispatch-docs.factor @@ -0,0 +1,8 @@ +IN: tools.dispatch +USING: help.markup help.syntax vm quotations ; + +HELP: last-dispatch-stats +{ $var-description "A " { $link dispatch-statistics } " instance, set by " { $link collect-dispatch-stats } "." } ; + +HELP: dispatch-stats. +{ $description "Prints method dispatch statistics from the last call to " { $link collect-dispatch-stats } "." } ; diff --git a/basis/tools/dispatch/dispatch.factor b/basis/tools/dispatch/dispatch.factor new file mode 100644 index 0000000000..7d30dac36b --- /dev/null +++ b/basis/tools/dispatch/dispatch.factor @@ -0,0 +1,24 @@ +! Copyright (C) 2009 Slava Pestov. +! See http://factorcode.org/license.txt for BSD license. +USING: accessors kernel namespaces prettyprint classes.struct +vm tools.dispatch.private ; +IN: tools.dispatch + +SYMBOL: last-dispatch-stats + +: dispatch-stats. ( -- ) + last-dispatch-stats get { + { "Megamorphic hits" [ megamorphic-cache-hits>> ] } + { "Megamorphic misses" [ megamorphic-cache-misses>> ] } + { "Cold to monomorphic" [ cold-call-to-ic-transitions>> ] } + { "Mono to polymorphic" [ ic-to-pic-transitions>> ] } + { "Poly to megamorphic" [ pic-to-mega-transitions>> ] } + { "Tag check count" [ pic-tag-count>> ] } + { "Tuple check count" [ pic-tuple-count>> ] } + } object-table. ; + +: collect-dispatch-stats ( quot -- ) + reset-dispatch-stats + call + dispatch-stats dispatch-statistics memory>struct + last-dispatch-stats set ; inline diff --git a/basis/tools/memory/memory-docs.factor b/basis/tools/memory/memory-docs.factor index 7ecbf402ab..f729e8945f 100644 --- a/basis/tools/memory/memory-docs.factor +++ b/basis/tools/memory/memory-docs.factor @@ -1,4 +1,4 @@ -USING: help.markup help.syntax memory sequences ; +USING: help.markup help.syntax memory sequences vm ; IN: tools.memory ARTICLE: "tools.memory" "Object memory tools" @@ -39,3 +39,15 @@ HELP: heap-stats. { $description "For each class, prints the number of instances and total memory consumed by those instances." } ; { heap-stats heap-stats. } related-words + +HELP: gc-events. +{ $description "Prints all garbage collection events that took place during the last call to " { $link collect-gc-events } "." } ; + +HELP: gc-stats. +{ $description "Prints a breakdown of different garbage collection events that took place during the last call to " { $link collect-gc-events } "." } ; + +HELP: gc-summary. +{ $description "Prints aggregate garbage collection statistics from the last call to " { $link collect-gc-events } "." } ; + +HELP: gc-events +{ $var-description "A sequence of " { $link gc-event } " instances, set by " { $link collect-gc-events } ". Can be inspected directly, or with the " { $link gc-events. } ", " { $link gc-stats. } " and " { $link gc-summary. } " words." } ; diff --git a/basis/tools/memory/memory-tests.factor b/basis/tools/memory/memory-tests.factor index 4b75cf0bfa..4711f472a3 100644 --- a/basis/tools/memory/memory-tests.factor +++ b/basis/tools/memory/memory-tests.factor @@ -1,5 +1,9 @@ -USING: tools.test tools.memory ; +USING: tools.test tools.memory memory ; IN: tools.memory.tests [ ] [ room. ] unit-test [ ] [ heap-stats. ] unit-test +[ ] [ [ gc gc ] collect-gc-events ] unit-test +[ ] [ gc-events. ] unit-test +[ ] [ gc-stats. ] unit-test +[ ] [ gc-summary. ] unit-test diff --git a/basis/tools/memory/memory.factor b/basis/tools/memory/memory.factor index 81785f7ea4..c147426a6f 100644 --- a/basis/tools/memory/memory.factor +++ b/basis/tools/memory/memory.factor @@ -1,55 +1,78 @@ -! Copyright (C) 2005, 2008 Slava Pestov. +! Copyright (C) 2005, 2009 Slava Pestov. ! See http://factorcode.org/license.txt for BSD license. -USING: kernel sequences arrays generic assocs io math -namespaces parser prettyprint strings io.styles words -system sorting splitting grouping math.parser classes memory -combinators fry ; +USING: accessors arrays assocs classes classes.struct +combinators combinators.smart continuations fry generalizations +generic grouping io io.styles kernel make math math.parser +math.statistics memory namespaces parser prettyprint sequences +sorting specialized-arrays splitting strings system vm words ; +SPECIALIZED-ARRAY: gc-event IN: tools.memory string - dup length 4 > [ 3 cut* "," glue ] when - " KB" append write-cell ; +: commas ( n -- str ) + dup 0 < [ neg commas "-" prepend ] [ + number>string + reverse 3 group "," join reverse + ] if ; -: write-total/used/free ( free total str -- ) - [ - write-cell - dup write-size - over - write-size - write-size - ] with-row ; +: kilobytes ( n -- str ) + 1024 /i commas " KB" append ; -: write-total ( n str -- ) - [ - write-cell - write-size - [ ] with-cell - [ ] with-cell - ] with-row ; +: micros>string ( n -- str ) + commas " µs" append ; -: write-headings ( seq -- ) - [ [ write-cell ] each ] with-row ; +: copying-room. ( copying-sizes -- ) + { + { "Size:" [ size>> kilobytes ] } + { "Occupied:" [ occupied>> kilobytes ] } + { "Free:" [ free>> kilobytes ] } + } object-table. ; -: (data-room.) ( -- ) - data-room 2 [ - [ first2 ] [ number>string "Generation " prepend ] bi* - write-total/used/free - ] each-index - "Decks" write-total - "Cards" write-total ; +: nursery-room. ( data-room -- ) + "- Nursery space" print nursery>> copying-room. ; -: write-labeled-size ( n string -- ) - [ write-cell write-size ] with-row ; +: aging-room. ( data-room -- ) + "- Aging space" print aging>> copying-room. ; -: (code-room.) ( -- ) - code-room { - [ "Size:" write-labeled-size ] - [ "Used:" write-labeled-size ] - [ "Total free space:" write-labeled-size ] - [ "Largest free block:" write-labeled-size ] - } spread ; +: mark-sweep-table. ( mark-sweep-sizes -- ) + { + { "Size:" [ size>> kilobytes ] } + { "Occupied:" [ occupied>> kilobytes ] } + { "Total free:" [ total-free>> kilobytes ] } + { "Contiguous free:" [ contiguous-free>> kilobytes ] } + { "Free block count:" [ free-block-count>> number>string ] } + } object-table. ; + +: tenured-room. ( data-room -- ) + "- Tenured space" print tenured>> mark-sweep-table. ; + +: misc-room. ( data-room -- ) + "- Miscellaneous buffers" print + { + { "Card array:" [ cards>> kilobytes ] } + { "Deck array:" [ decks>> kilobytes ] } + { "Mark stack:" [ mark-stack>> kilobytes ] } + } object-table. ; + +: data-room. ( -- ) + "== Data heap ==" print nl + data-room data-heap-room memory>struct { + [ nursery-room. nl ] + [ aging-room. nl ] + [ tenured-room. nl ] + [ misc-room. ] + } cleave ; + +: code-room. ( -- ) + "== Code heap ==" print nl + code-room mark-sweep-sizes memory>struct mark-sweep-table. ; + +PRIVATE> + +: room. ( -- ) data-room. nl code-room. ; + + -: room. ( -- ) - "==== DATA HEAP" print - standard-table-style [ - { "" "Total" "Used" "Free" } write-headings - (data-room.) - ] tabular-output - nl nl - "==== CODE HEAP" print - standard-table-style [ - (code-room.) - ] tabular-output - nl ; - : heap-stats ( -- counts sizes ) [ ] instances H{ } clone H{ } clone [ '[ _ _ heap-stat-step ] each ] 2keep ; : heap-stats. ( -- ) heap-stats dup keys natural-sort standard-table-style [ - { "Class" "Bytes" "Instances" } write-headings + [ { "Class" "Bytes" "Instances" } [ write-cell ] each ] with-row [ [ dup pprint-cell @@ -85,3 +95,104 @@ PRIVATE> ] with-row ] each 2drop ] tabular-output nl ; + +SYMBOL: gc-events + +: collect-gc-events ( quot -- ) + enable-gc-events + [ ] [ disable-gc-events drop ] cleanup + disable-gc-events byte-array>gc-event-array gc-events set ; inline + +> ] [ aging>> ] [ tenured>> ] tri [ occupied>> ] tri@ ] + [ occupied>> ] + bi* + ] sum-outputs ; + +: space-occupied-before ( event -- bytes ) + [ data-heap-before>> ] [ code-heap-before>> ] bi (space-occupied) ; + +: space-occupied-after ( event -- bytes ) + [ data-heap-after>> ] [ code-heap-after>> ] bi (space-occupied) ; + +: space-reclaimed ( event -- bytes ) + [ space-occupied-before ] [ space-occupied-after ] bi - ; + +TUPLE: gc-stats collections times ; + +: ( -- stats ) + gc-stats new + 0 >>collections + V{ } clone >>times ; inline + +: compute-gc-stats ( events -- stats ) + V{ } clone [ + '[ + dup op>> _ [ drop ] cache + [ 1 + ] change-collections + [ total-time>> ] dip times>> push + ] each + ] keep sort-keys ; + +: gc-stats-table-row ( pair -- row ) + [ + [ first gc-op-string ] [ + second + [ collections>> ] + [ + times>> { + [ sum micros>string ] + [ mean >integer micros>string ] + [ median >integer micros>string ] + [ infimum micros>string ] + [ supremum micros>string ] + } cleave + ] bi + ] bi + ] output>array ; + +: gc-stats-table ( stats -- table ) + [ gc-stats-table-row ] map + { "" "Number" "Total" "Mean" "Median" "Min" "Max" } prefix ; + +PRIVATE> + +: gc-event. ( event -- ) + { + { "Event type:" [ op>> gc-op-string ] } + { "Total time:" [ total-time>> micros>string ] } + { "Space reclaimed:" [ space-reclaimed kilobytes ] } + } object-table. ; + +: gc-events. ( -- ) + gc-events get [ gc-event. nl ] each ; + +: gc-stats. ( -- ) + gc-events get compute-gc-stats gc-stats-table simple-table. ; + +: gc-summary. ( -- ) + gc-events get { + { "Collections:" [ length commas ] } + { "Cards scanned:" [ [ cards-scanned>> ] map-sum commas ] } + { "Decks scanned:" [ [ decks-scanned>> ] map-sum commas ] } + { "Code blocks scanned:" [ [ code-blocks-scanned>> ] map-sum commas ] } + { "Total time:" [ [ total-time>> ] map-sum micros>string ] } + { "Card scan time:" [ [ card-scan-time>> ] map-sum micros>string ] } + { "Code block scan time:" [ [ code-scan-time>> ] map-sum micros>string ] } + { "Data heap sweep time:" [ [ data-sweep-time>> ] map-sum micros>string ] } + { "Code heap sweep time:" [ [ code-sweep-time>> ] map-sum micros>string ] } + { "Compaction time:" [ [ compaction-time>> ] map-sum micros>string ] } + } object-table. ; diff --git a/basis/tools/profiler/profiler-docs.factor b/basis/tools/profiler/profiler-docs.factor index 0fda4a65e5..66ae5d7bd3 100644 --- a/basis/tools/profiler/profiler-docs.factor +++ b/basis/tools/profiler/profiler-docs.factor @@ -25,7 +25,7 @@ $nl method-profile. "profiler-limitations" } -{ $see-also "ui.tools.profiler" } ; +{ $see-also "ui.tools.profiler" "tools.annotations" "timing" } ; ABOUT: "profiling" diff --git a/basis/tools/time/time-docs.factor b/basis/tools/time/time-docs.factor index 408592d0c6..9e892c33ec 100644 --- a/basis/tools/time/time-docs.factor +++ b/basis/tools/time/time-docs.factor @@ -1,28 +1,38 @@ -USING: help.markup help.syntax memory system ; +USING: help.markup help.syntax memory system tools.dispatch +tools.memory quotations vm ; IN: tools.time -ARTICLE: "timing" "Timing code" +ARTICLE: "timing" "Timing code and collecting statistics" "You can time the execution of a quotation in the listener:" { $subsections time } +"This word also collects statistics about method dispatch and garbage collection:" +{ $subsections dispatch-stats. gc-events. gc-stats. gc-summary. } "A lower-level word puts timings on the stack, intead of printing:" { $subsections benchmark } -"You can also read the system clock and garbage collection statistics directly:" -{ $subsections - micros - gc-stats -} -{ $see-also "profiling" } ; +"You can also read the system clock directly:" +{ $subsections micros } +{ $see-also "profiling" "calendar" } ; ABOUT: "timing" HELP: benchmark -{ $values { "quot" "a quotation" } +{ $values { "quot" quotation } { "runtime" "the runtime in microseconds" } } { $description "Runs a quotation, measuring the total wall clock time." } { $notes "A nicer word for interactive use is " { $link time } "." } ; HELP: time -{ $values { "quot" "a quotation" } } -{ $description "Runs a quotation and then prints the total run time and some garbage collection statistics." } ; +{ $values { "quot" quotation } } +{ $description "Runs a quotation, gathering statistics about method dispatch and garbage collection, and then prints the total run time." } ; { benchmark micros time } related-words + +HELP: collect-gc-events +{ $values { "quot" quotation } } +{ $description "Calls the quotation, storing an array of " { $link gc-event } " instances in the " { $link gc-events } " variable." } +{ $notes "The " { $link time } " combinator automatically calls this combinator." } ; + +HELP: collect-dispatch-stats +{ $values { "quot" quotation } } +{ $description "Calls the quotation, collecting method dispatch statistics and storing them in the " { $link last-dispatch-stats } " variable. " } +{ $notes "The " { $link time } " combinator automatically calls this combinator." } ; diff --git a/basis/tools/time/time.factor b/basis/tools/time/time.factor index 948c0d482d..3724a741b7 100644 --- a/basis/tools/time/time.factor +++ b/basis/tools/time/time.factor @@ -1,74 +1,22 @@ -! Copyright (C) 2003, 2008 Slava Pestov. +! Copyright (C) 2003, 2009 Slava Pestov. ! See http://factorcode.org/license.txt for BSD license. -USING: kernel math memory io io.styles prettyprint -namespaces system sequences splitting grouping assocs strings -generic.single combinators ; +USING: system kernel math io prettyprint tools.memory +tools.dispatch ; IN: tools.time : benchmark ( quot -- runtime ) micros [ call micros ] dip - ; inline : time. ( time -- ) - "== Running time ==" print nl 1000000 /f pprint " seconds" print ; + "Running time: " write 1000000 /f pprint " seconds" print ; -: gc-stats. ( stats -- ) - 5 cut* - "== Garbage collection ==" print nl - "Times are in microseconds." print nl - [ - 6 group - { - "GC count:" - "Total GC time:" - "Longest GC pause:" - "Average GC pause:" - "Objects copied:" - "Bytes copied:" - } prefix - flip - { "" "Nursery" "Aging" "Tenured" } prefix - simple-table. - ] - [ - nl - { - "Total GC time:" - "Cards scanned:" - "Decks scanned:" - "Card scan time:" - "Code heap literal scans:" - } swap zip simple-table. - ] bi* ; - -: dispatch-stats. ( stats -- ) - "== Megamorphic caches ==" print nl - { "Hits" "Misses" } swap zip simple-table. ; - -: inline-cache-stats. ( stats -- ) - nl "== Polymorphic inline caches ==" print nl - 3 cut - [ - "Transitions:" print - { "Cold to monomorphic" "Mono to polymorphic" "Poly to megamorphic" } swap zip - simple-table. nl - ] [ - "Type check stubs:" print - { "Tag only" "Hi-tag" "Tuple" "Hi-tag and tuple" } swap zip - simple-table. - ] bi* ; +: time-banner. ( -- ) + "Additional information was collected." print + "dispatch-stats. - Print method dispatch statistics" print + "gc-events. - Print all garbage collection events" print + "gc-stats. - Print breakdown of different garbage collection events" print + "gc-summary. - Print aggregate garbage collection statistics" print ; : time ( quot -- ) - gc-reset - reset-dispatch-stats - reset-inline-cache-stats - benchmark gc-stats dispatch-stats inline-cache-stats - H{ { table-gap { 20 20 } } } [ - [ - [ [ time. ] 3dip ] with-cell - [ ] with-cell - ] with-row - [ - [ [ gc-stats. ] 2dip ] with-cell - [ [ dispatch-stats. ] [ inline-cache-stats. ] bi* ] with-cell - ] with-row - ] tabular-output nl ; inline + [ [ benchmark ] collect-dispatch-stats ] collect-gc-events + time. nl time-banner. ; inline diff --git a/basis/vm/vm.factor b/basis/vm/vm.factor index 11d9dabb3d..ba057edffa 100644 --- a/basis/vm/vm.factor +++ b/basis/vm/vm.factor @@ -3,20 +3,77 @@ USING: classes.struct alien.c-types alien.syntax ; IN: vm -TYPEDEF: void* cell +TYPEDEF: intptr_t cell C-TYPE: context STRUCT: zone - { start cell } - { here cell } - { size cell } - { end cell } ; +{ start cell } +{ here cell } +{ size cell } +{ end cell } ; STRUCT: vm - { stack_chain context* } - { nursery zone } - { cards_offset cell } - { decks_offset cell } - { userenv cell[70] } ; +{ stack_chain context* } +{ nursery zone } +{ cards_offset cell } +{ decks_offset cell } +{ userenv cell[70] } ; : vm-field-offset ( field -- offset ) vm offset-of ; inline + +C-ENUM: +collect-nursery-op +collect-aging-op +collect-to-tenured-op +collect-full-op +collect-compact-op +collect-growing-heap-op ; + +STRUCT: copying-sizes +{ size cell } +{ occupied cell } +{ free cell } ; + +STRUCT: mark-sweep-sizes +{ size cell } +{ occupied cell } +{ total-free cell } +{ contiguous-free cell } +{ free-block-count cell } ; + +STRUCT: data-heap-room +{ nursery copying-sizes } +{ aging copying-sizes } +{ tenured mark-sweep-sizes } +{ cards cell } +{ decks cell } +{ mark-stack cell } ; + +STRUCT: gc-event +{ op uint } +{ data-heap-before data-heap-room } +{ code-heap-before mark-sweep-sizes } +{ data-heap-after data-heap-room } +{ code-heap-after mark-sweep-sizes } +{ cards-scanned cell } +{ decks-scanned cell } +{ code-blocks-scanned cell } +{ start-time ulonglong } +{ total-time cell } +{ card-scan-time cell } +{ code-scan-time cell } +{ data-sweep-time cell } +{ code-sweep-time cell } +{ compaction-time cell } +{ temp-time cell } ; + +STRUCT: dispatch-statistics +{ megamorphic-cache-hits cell } +{ megamorphic-cache-misses cell } + +{ cold-call-to-ic-transitions cell } +{ ic-to-pic-transitions cell } +{ pic-to-mega-transitions cell } + +{ pic-tag-count cell } +{ pic-tuple-count cell } ; diff --git a/core/alien/alien.factor b/core/alien/alien.factor index 3f2b5f95bf..f008a4bd59 100644 --- a/core/alien/alien.factor +++ b/core/alien/alien.factor @@ -4,19 +4,9 @@ USING: accessors assocs kernel math namespaces sequences system kernel.private byte-arrays arrays init ; IN: alien -! Some predicate classes used by the compiler for optimization -! purposes -PREDICATE: simple-alien < alien underlying>> not ; +PREDICATE: pinned-alien < alien underlying>> not ; -UNION: simple-c-ptr -simple-alien POSTPONE: f byte-array ; - -DEFER: pinned-c-ptr? - -PREDICATE: pinned-alien < alien underlying>> pinned-c-ptr? ; - -UNION: pinned-c-ptr - pinned-alien POSTPONE: f ; +UNION: pinned-c-ptr pinned-alien POSTPONE: f ; GENERIC: >c-ptr ( obj -- c-ptr ) @@ -33,7 +23,7 @@ M: alien expired? expired>> ; M: f expired? drop t ; : ( address -- alien ) - f { simple-c-ptr } declare ; inline + f { pinned-c-ptr } declare ; inline : ( -- alien ) -1 t >>expired ; inline @@ -49,7 +39,8 @@ M: alien equal? 2drop f ] if ; -M: simple-alien hashcode* nip dup expired>> [ drop 1234 ] [ alien-address ] if ; +M: pinned-alien hashcode* + nip dup expired>> [ drop 1234 ] [ alien-address ] if ; ERROR: alien-callback-error ; diff --git a/core/bootstrap/layouts/layouts.factor b/core/bootstrap/layouts/layouts.factor index 5ed92b7776..8b6547ce5c 100644 --- a/core/bootstrap/layouts/layouts.factor +++ b/core/bootstrap/layouts/layouts.factor @@ -5,32 +5,28 @@ hashtables vectors strings sbufs arrays quotations assocs layouts classes.tuple.private kernel.private ; -BIN: 111 tag-mask set -8 num-tags set -3 tag-bits set +16 data-alignment set -15 num-types set +BIN: 1111 tag-mask set +4 tag-bits set + +14 num-types set 32 mega-cache-size set H{ - { fixnum BIN: 000 } - { bignum BIN: 001 } - { array BIN: 010 } - { float BIN: 011 } - { quotation BIN: 100 } - { POSTPONE: f BIN: 101 } - { object BIN: 110 } - { hi-tag BIN: 110 } - { tuple BIN: 111 } -} tag-numbers set - -tag-numbers get H{ + { fixnum 0 } + { POSTPONE: f 1 } + { array 2 } + { float 3 } + { quotation 4 } + { bignum 5 } + { alien 6 } + { tuple 7 } { wrapper 8 } { byte-array 9 } { callstack 10 } { string 11 } { word 12 } { dll 13 } - { alien 14 } -} assoc-union type-numbers set +} type-numbers set diff --git a/core/bootstrap/primitives.factor b/core/bootstrap/primitives.factor index ef66cc3cd6..5d4144e354 100644 --- a/core/bootstrap/primitives.factor +++ b/core/bootstrap/primitives.factor @@ -99,6 +99,7 @@ bootstrapping? on "system" "system.private" "threads.private" + "tools.dispatch.private" "tools.profiler.private" "words" "words.private" @@ -177,10 +178,6 @@ bi "object?" "kernel" vocab-words delete-at -! Class of objects with object tag -"hi-tag" "kernel.private" create -builtins get num-tags get tail define-union-class - ! Empty class with no instances "null" "kernel" create [ f { } f union-class define-class ] @@ -423,7 +420,6 @@ tuple { "minor-gc" "memory" (( -- )) } { "gc" "memory" (( -- )) } { "compact-gc" "memory" (( -- )) } - { "gc-stats" "memory" f } { "(save-image)" "memory.private" (( path -- )) } { "(save-image-and-exit)" "memory.private" (( path -- )) } { "datastack" "kernel" (( -- ds )) } @@ -433,8 +429,8 @@ tuple { "set-retainstack" "kernel" (( rs -- )) } { "set-callstack" "kernel" (( cs -- )) } { "exit" "system" (( n -- )) } - { "data-room" "memory" (( -- cards decks generations )) } - { "code-room" "memory" (( -- code-total code-used code-free largest-free-block )) } + { "data-room" "memory" (( -- data-room )) } + { "code-room" "memory" (( -- code-room )) } { "micros" "system" (( -- us )) } { "modify-code-heap" "compiler.units" (( alist -- )) } { "(dlopen)" "alien.libraries" (( path -- dll )) } @@ -509,7 +505,6 @@ tuple { "resize-byte-array" "byte-arrays" (( n byte-array -- newbyte-array )) } { "dll-valid?" "alien.libraries" (( dll -- ? )) } { "unimplemented" "kernel.private" (( -- * )) } - { "gc-reset" "memory" (( -- )) } { "jit-compile" "quotations" (( quot -- )) } { "load-locals" "locals.backend" (( ... n -- )) } { "check-datastack" "kernel.private" (( array in# out# -- ? )) } @@ -517,15 +512,15 @@ tuple { "inline-cache-miss-tail" "generic.single.private" (( generic methods index cache -- )) } { "mega-cache-miss" "generic.single.private" (( methods index cache -- method )) } { "lookup-method" "generic.single.private" (( object methods -- method )) } - { "reset-dispatch-stats" "generic.single" (( -- )) } - { "dispatch-stats" "generic.single" (( -- stats )) } - { "reset-inline-cache-stats" "generic.single" (( -- )) } - { "inline-cache-stats" "generic.single" (( -- stats )) } + { "reset-dispatch-stats" "tools.dispatch.private" (( -- )) } + { "dispatch-stats" "tools.dispatch.private" (( -- stats )) } { "optimized?" "words" (( word -- ? )) } { "quot-compiled?" "quotations" (( quot -- ? )) } { "vm-ptr" "vm" (( -- ptr )) } { "strip-stack-traces" "kernel.private" (( -- )) } { "" "alien" (( word -- alien )) } + { "enable-gc-events" "memory" (( -- )) } + { "disable-gc-events" "memory" (( -- events )) } } [ [ first3 ] dip swap make-primitive ] each-index ! Bump build number diff --git a/core/classes/algebra/algebra-docs.factor b/core/classes/algebra/algebra-docs.factor index 1b2ea7dfd4..65e6f85678 100644 --- a/core/classes/algebra/algebra-docs.factor +++ b/core/classes/algebra/algebra-docs.factor @@ -17,7 +17,6 @@ ARTICLE: "class-operations" "Class operations" flatten-class flatten-builtin-class class-types - class-tags } ; ARTICLE: "class-linearization" "Class linearization" diff --git a/core/classes/algebra/algebra-tests.factor b/core/classes/algebra/algebra-tests.factor index 855a15b66f..72c2dd575c 100644 --- a/core/classes/algebra/algebra-tests.factor +++ b/core/classes/algebra/algebra-tests.factor @@ -95,8 +95,6 @@ UNION: z1 b1 c1 ; [ f ] [ a1 c1 class-or b1 c1 class-or class-and a1 b1 class-or classes-intersect? ] unit-test -[ f ] [ growable \ hi-tag classes-intersect? ] unit-test - [ t ] [ growable tuple sequence class-and class<= ] unit-test diff --git a/core/classes/algebra/algebra.factor b/core/classes/algebra/algebra.factor index afaae444bc..06857d3c71 100755 --- a/core/classes/algebra/algebra.factor +++ b/core/classes/algebra/algebra.factor @@ -237,11 +237,5 @@ M: anonymous-union (flatten-class) flatten-builtin-class keys [ "type" word-prop ] map natural-sort ; -: class-tags ( class -- seq ) - class-types [ - dup num-tags get >= - [ drop \ hi-tag tag-number ] when - ] map prune ; - -: class-tag ( class -- tag/f ) - class-tags dup length 1 = [ first ] [ drop f ] if ; +: class-type ( class -- tag/f ) + class-types dup length 1 = [ first ] [ drop f ] if ; diff --git a/core/classes/builtin/builtin.factor b/core/classes/builtin/builtin.factor index 8eeb4ce357..6185e4f24d 100644 --- a/core/classes/builtin/builtin.factor +++ b/core/classes/builtin/builtin.factor @@ -12,34 +12,20 @@ PREDICATE: builtin-class < class : class>type ( class -- n ) "type" word-prop ; foldable -PREDICATE: lo-tag-class < builtin-class class>type 7 <= ; - -PREDICATE: hi-tag-class < builtin-class class>type 7 > ; - : type>class ( n -- class ) builtins get-global nth ; : bootstrap-type>class ( n -- class ) builtins get nth ; -M: hi-tag class hi-tag type>class ; inline - M: object class tag type>class ; inline M: builtin-class rank-class drop 0 ; GENERIC: define-builtin-predicate ( class -- ) -M: lo-tag-class define-builtin-predicate +M: builtin-class define-builtin-predicate dup class>type [ eq? ] curry [ tag ] prepend define-predicate ; -M: hi-tag-class define-builtin-predicate - dup class>type [ eq? ] curry [ hi-tag ] prepend 1quotation - [ dup tag 6 eq? ] [ [ drop f ] if ] surround - define-predicate ; - -M: lo-tag-class instance? [ tag ] [ class>type ] bi* eq? ; - -M: hi-tag-class instance? - over tag 6 eq? [ [ hi-tag ] [ class>type ] bi* eq? ] [ 2drop f ] if ; +M: builtin-class instance? [ tag ] [ class>type ] bi* eq? ; M: builtin-class (flatten-class) dup set ; diff --git a/core/classes/classes-tests.factor b/core/classes/classes-tests.factor index 5607bc3a22..10a5f674bd 100644 --- a/core/classes/classes-tests.factor +++ b/core/classes/classes-tests.factor @@ -11,7 +11,6 @@ IN: classes.tests [ f ] [ 3 float instance? ] unit-test [ t ] [ 3 number instance? ] unit-test [ f ] [ 3 null instance? ] unit-test -[ t ] [ "hi" \ hi-tag instance? ] unit-test ! Regression GENERIC: method-forget-test ( obj -- obj ) diff --git a/core/generic/single/single.factor b/core/generic/single/single.factor index 9e773fe700..1434acf521 100644 --- a/core/generic/single/single.factor +++ b/core/generic/single/single.factor @@ -112,15 +112,6 @@ TUPLE: tuple-dispatch-engine echelons ; tuple bootstrap-word \ convert-methods ; -! 2.2 Convert hi-tag methods -TUPLE: hi-tag-dispatch-engine methods ; - -C: hi-tag-dispatch-engine - -: convert-hi-tag-methods ( assoc -- assoc' ) - \ hi-tag bootstrap-word - \ convert-methods ; - ! 3 Tag methods TUPLE: tag-dispatch-engine methods ; @@ -129,7 +120,6 @@ C: tag-dispatch-engine : ( assoc -- engine ) flatten-methods convert-tuple-methods - convert-hi-tag-methods ; ! ! ! Compile engine ! ! ! @@ -144,23 +134,12 @@ GENERIC: compile-engine ( engine -- obj ) : direct-dispatch-table ( assoc n -- table ) default get [ swap update ] keep ; -: lo-tag-number ( class -- n ) - "type" word-prop dup num-tags get iota member? - [ drop object tag-number ] unless ; +: tag-number ( class -- n ) "type" word-prop ; M: tag-dispatch-engine compile-engine methods>> compile-engines* - [ [ lo-tag-number ] dip ] assoc-map - num-tags get direct-dispatch-table ; - -: num-hi-tags ( -- n ) num-types get num-tags get - ; - -: hi-tag-number ( class -- n ) "type" word-prop ; - -M: hi-tag-dispatch-engine compile-engine - methods>> compile-engines* - [ [ hi-tag-number num-tags get - ] dip ] assoc-map - num-hi-tags direct-dispatch-table ; + [ [ tag-number ] dip ] assoc-map + num-types get direct-dispatch-table ; : build-fast-hash ( methods -- buckets ) >alist V{ } clone [ hashcode 1array ] distribute-buckets diff --git a/core/kernel/kernel-docs.factor b/core/kernel/kernel-docs.factor index f7ae292630..f70d9d4214 100644 --- a/core/kernel/kernel-docs.factor +++ b/core/kernel/kernel-docs.factor @@ -651,7 +651,7 @@ HELP: declare HELP: tag ( object -- n ) { $values { "object" object } { "n" "a tag number" } } -{ $description "Outputs an object's tag number, between zero and one less than " { $link num-tags } ". This is implementation detail and user code should call " { $link class } " instead." } ; +{ $description "Outputs an object's tag number, between zero and one less than " { $link num-types } ". This is implementation detail and user code should call " { $link class } " instead." } ; HELP: getenv ( n -- obj ) { $values { "n" "a non-negative integer" } { "obj" object } } diff --git a/core/kernel/kernel.factor b/core/kernel/kernel.factor index 22c96c4318..a0934c2b17 100644 --- a/core/kernel/kernel.factor +++ b/core/kernel/kernel.factor @@ -230,8 +230,6 @@ ERROR: assert got expect ; : declare ( spec -- ) drop ; -: hi-tag ( obj -- n ) { hi-tag } declare 0 slot ; inline - : do-primitive ( number -- ) "Improper primitive call" throw ; PRIVATE> diff --git a/core/layouts/layouts-docs.factor b/core/layouts/layouts-docs.factor index 8dd1e6901f..efea1ffb4e 100644 --- a/core/layouts/layouts-docs.factor +++ b/core/layouts/layouts-docs.factor @@ -7,18 +7,11 @@ HELP: tag-bits { $var-description "Number of least significant bits reserved for a type tag in a tagged pointer." } { $see-also tag } ; -HELP: num-tags -{ $var-description "Number of distinct pointer tags. This is one more than the maximum value from the " { $link tag } " primitive." } ; - HELP: tag-mask { $var-description "Taking the bitwise and of a tagged pointer with this mask leaves the tag." } ; HELP: num-types -{ $var-description "Number of distinct built-in types. This is one more than the maximum value from the " { $link hi-tag } " primitive." } ; - -HELP: tag-number -{ $values { "class" class } { "n" "an integer or " { $link f } } } -{ $description "Outputs the pointer tag for pointers to instances of " { $link class } ". Will output " { $link f } " if instances of this class are not identified by a distinct pointer tag." } ; +{ $var-description "Number of distinct built-in types. This is one more than the maximum value from the " { $link tag } " primitive." } ; HELP: type-number { $values { "class" class } { "n" "an integer or " { $link f } } } @@ -76,7 +69,7 @@ HELP: bootstrap-cell-bits ARTICLE: "layouts-types" "Type numbers" "Corresponding to every built-in class is a built-in type number. An object can be asked for its built-in type number:" -{ $subsections hi-tag } +{ $subsections tag } "Built-in type numbers can be converted to classes, and vice versa:" { $subsections type>class @@ -88,14 +81,10 @@ ARTICLE: "layouts-types" "Type numbers" ARTICLE: "layouts-tags" "Tagged pointers" "Every pointer stored on the stack or in the heap has a " { $emphasis "tag" } ", which is a small integer identifying the type of the pointer. If the tag is not equal to one of the two special tags, the remaining bits contain the memory address of a heap-allocated object. The two special tags are the " { $link fixnum } " tag and the " { $link f } " tag." $nl -"Getting the tag of an object:" -{ $link tag } "Words for working with tagged pointers:" { $subsections tag-bits - num-tags tag-mask - tag-number } "The Factor VM does not actually expose any words for working with tagged pointers directly. The above words operate on integers; they are used in the bootstrap image generator and the optimizing compiler." ; diff --git a/core/layouts/layouts.factor b/core/layouts/layouts.factor index be6276a684..7518dbf0cb 100644 --- a/core/layouts/layouts.factor +++ b/core/layouts/layouts.factor @@ -4,16 +4,14 @@ USING: namespaces math words kernel assocs classes math.order kernel.private ; IN: layouts -SYMBOL: tag-mask +SYMBOL: data-alignment -SYMBOL: num-tags +SYMBOL: tag-mask SYMBOL: tag-bits SYMBOL: num-types -SYMBOL: tag-numbers - SYMBOL: type-numbers SYMBOL: mega-cache-size @@ -21,9 +19,6 @@ SYMBOL: mega-cache-size : type-number ( class -- n ) type-numbers get at ; -: tag-number ( class -- n ) - type-number dup num-tags get >= [ drop object tag-number ] when ; - : tag-fixnum ( n -- tagged ) tag-bits get shift ; @@ -58,7 +53,7 @@ SYMBOL: mega-cache-size first-bignum neg >fixnum ; inline : (max-array-capacity) ( b -- n ) - 5 - 2^ 1 - ; inline + 6 - 2^ 1 - ; inline : max-array-capacity ( -- n ) cell-bits (max-array-capacity) ; inline diff --git a/core/slots/slots-docs.factor b/core/slots/slots-docs.factor index ce29c14b01..92b34db6ec 100644 --- a/core/slots/slots-docs.factor +++ b/core/slots/slots-docs.factor @@ -71,7 +71,7 @@ $nl { { { $link float } } { $snippet "0.0" } } { { { $link string } } { $snippet "\"\"" } } { { { $link byte-array } } { $snippet "B{ }" } } - { { { $link simple-alien } } { $snippet "BAD-ALIEN" } } + { { { $link pinned-alien } } { $snippet "BAD-ALIEN" } } } "All other classes are handled with one of two cases:" { $list diff --git a/core/slots/slots.factor b/core/slots/slots.factor index 95a854f493..0422478884 100755 --- a/core/slots/slots.factor +++ b/core/slots/slots.factor @@ -173,7 +173,7 @@ M: class initial-value* no-initial-value ; { [ string bootstrap-word over class<= ] [ "" ] } { [ array bootstrap-word over class<= ] [ { } ] } { [ byte-array bootstrap-word over class<= ] [ B{ } ] } - { [ simple-alien bootstrap-word over class<= ] [ ] } + { [ pinned-alien bootstrap-word over class<= ] [ ] } { [ quotation bootstrap-word over class<= ] [ [ ] ] } [ dup initial-value* ] } cond nip ; diff --git a/basis/images/testing/bmp/1bit.bmp b/extra/images/testing/bmp/1bit.bmp similarity index 100% rename from basis/images/testing/bmp/1bit.bmp rename to extra/images/testing/bmp/1bit.bmp diff --git a/basis/images/testing/bmp/42red_24bit.bmp b/extra/images/testing/bmp/42red_24bit.bmp similarity index 100% rename from basis/images/testing/bmp/42red_24bit.bmp rename to extra/images/testing/bmp/42red_24bit.bmp diff --git a/basis/images/testing/bmp/42red_24bit.fig b/extra/images/testing/bmp/42red_24bit.fig similarity index 100% rename from basis/images/testing/bmp/42red_24bit.fig rename to extra/images/testing/bmp/42red_24bit.fig diff --git a/basis/images/testing/bmp/rgb_4bit.bmp b/extra/images/testing/bmp/rgb_4bit.bmp similarity index 100% rename from basis/images/testing/bmp/rgb_4bit.bmp rename to extra/images/testing/bmp/rgb_4bit.bmp diff --git a/basis/images/testing/bmp/rgb_8bit.bmp b/extra/images/testing/bmp/rgb_8bit.bmp similarity index 100% rename from basis/images/testing/bmp/rgb_8bit.bmp rename to extra/images/testing/bmp/rgb_8bit.bmp diff --git a/basis/images/testing/bmp/rgb_8bit.fig b/extra/images/testing/bmp/rgb_8bit.fig similarity index 100% rename from basis/images/testing/bmp/rgb_8bit.fig rename to extra/images/testing/bmp/rgb_8bit.fig diff --git a/basis/images/testing/gif/alpha.fig b/extra/images/testing/gif/alpha.fig similarity index 100% rename from basis/images/testing/gif/alpha.fig rename to extra/images/testing/gif/alpha.fig diff --git a/basis/images/testing/gif/alpha.gif b/extra/images/testing/gif/alpha.gif similarity index 100% rename from basis/images/testing/gif/alpha.gif rename to extra/images/testing/gif/alpha.gif diff --git a/basis/images/testing/gif/astronaut_animation.fig b/extra/images/testing/gif/astronaut_animation.fig similarity index 100% rename from basis/images/testing/gif/astronaut_animation.fig rename to extra/images/testing/gif/astronaut_animation.fig diff --git a/basis/images/testing/gif/astronaut_animation.gif b/extra/images/testing/gif/astronaut_animation.gif similarity index 100% rename from basis/images/testing/gif/astronaut_animation.gif rename to extra/images/testing/gif/astronaut_animation.gif diff --git a/basis/images/testing/gif/checkmark.fig b/extra/images/testing/gif/checkmark.fig similarity index 100% rename from basis/images/testing/gif/checkmark.fig rename to extra/images/testing/gif/checkmark.fig diff --git a/basis/images/testing/gif/checkmark.gif b/extra/images/testing/gif/checkmark.gif similarity index 100% rename from basis/images/testing/gif/checkmark.gif rename to extra/images/testing/gif/checkmark.gif diff --git a/basis/images/testing/gif/circle.fig b/extra/images/testing/gif/circle.fig similarity index 100% rename from basis/images/testing/gif/circle.fig rename to extra/images/testing/gif/circle.fig diff --git a/basis/images/testing/gif/circle.gif b/extra/images/testing/gif/circle.gif similarity index 100% rename from basis/images/testing/gif/circle.gif rename to extra/images/testing/gif/circle.gif diff --git a/basis/images/testing/gif/monochrome.fig b/extra/images/testing/gif/monochrome.fig similarity index 100% rename from basis/images/testing/gif/monochrome.fig rename to extra/images/testing/gif/monochrome.fig diff --git a/basis/images/testing/gif/monochrome.gif b/extra/images/testing/gif/monochrome.gif similarity index 100% rename from basis/images/testing/gif/monochrome.gif rename to extra/images/testing/gif/monochrome.gif diff --git a/basis/images/testing/gif/noise.fig b/extra/images/testing/gif/noise.fig similarity index 100% rename from basis/images/testing/gif/noise.fig rename to extra/images/testing/gif/noise.fig diff --git a/basis/images/testing/gif/noise.gif b/extra/images/testing/gif/noise.gif similarity index 100% rename from basis/images/testing/gif/noise.gif rename to extra/images/testing/gif/noise.gif diff --git a/basis/images/testing/png/basn2c08.fig b/extra/images/testing/png/basn2c08.fig similarity index 100% rename from basis/images/testing/png/basn2c08.fig rename to extra/images/testing/png/basn2c08.fig diff --git a/basis/images/testing/png/basn2c08.png b/extra/images/testing/png/basn2c08.png similarity index 100% rename from basis/images/testing/png/basn2c08.png rename to extra/images/testing/png/basn2c08.png diff --git a/basis/images/testing/png/basn6a08.fig b/extra/images/testing/png/basn6a08.fig similarity index 100% rename from basis/images/testing/png/basn6a08.fig rename to extra/images/testing/png/basn6a08.fig diff --git a/basis/images/testing/png/basn6a08.png b/extra/images/testing/png/basn6a08.png similarity index 100% rename from basis/images/testing/png/basn6a08.png rename to extra/images/testing/png/basn6a08.png diff --git a/basis/images/testing/png/f00n2c08.fig b/extra/images/testing/png/f00n2c08.fig similarity index 100% rename from basis/images/testing/png/f00n2c08.fig rename to extra/images/testing/png/f00n2c08.fig diff --git a/basis/images/testing/png/f00n2c08.png b/extra/images/testing/png/f00n2c08.png similarity index 100% rename from basis/images/testing/png/f00n2c08.png rename to extra/images/testing/png/f00n2c08.png diff --git a/basis/images/testing/png/f01n2c08.fig b/extra/images/testing/png/f01n2c08.fig similarity index 100% rename from basis/images/testing/png/f01n2c08.fig rename to extra/images/testing/png/f01n2c08.fig diff --git a/basis/images/testing/png/f01n2c08.png b/extra/images/testing/png/f01n2c08.png similarity index 100% rename from basis/images/testing/png/f01n2c08.png rename to extra/images/testing/png/f01n2c08.png diff --git a/basis/images/testing/png/f02n2c08.fig b/extra/images/testing/png/f02n2c08.fig similarity index 100% rename from basis/images/testing/png/f02n2c08.fig rename to extra/images/testing/png/f02n2c08.fig diff --git a/basis/images/testing/png/f02n2c08.png b/extra/images/testing/png/f02n2c08.png similarity index 100% rename from basis/images/testing/png/f02n2c08.png rename to extra/images/testing/png/f02n2c08.png diff --git a/basis/images/testing/png/f03n2c08.fig b/extra/images/testing/png/f03n2c08.fig similarity index 100% rename from basis/images/testing/png/f03n2c08.fig rename to extra/images/testing/png/f03n2c08.fig diff --git a/basis/images/testing/png/f03n2c08.png b/extra/images/testing/png/f03n2c08.png similarity index 100% rename from basis/images/testing/png/f03n2c08.png rename to extra/images/testing/png/f03n2c08.png diff --git a/basis/images/testing/png/f04n2c08.fig b/extra/images/testing/png/f04n2c08.fig similarity index 100% rename from basis/images/testing/png/f04n2c08.fig rename to extra/images/testing/png/f04n2c08.fig diff --git a/basis/images/testing/png/f04n2c08.png b/extra/images/testing/png/f04n2c08.png similarity index 100% rename from basis/images/testing/png/f04n2c08.png rename to extra/images/testing/png/f04n2c08.png diff --git a/basis/images/testing/png/suite/basi0g01.png b/extra/images/testing/png/suite/basi0g01.png similarity index 100% rename from basis/images/testing/png/suite/basi0g01.png rename to extra/images/testing/png/suite/basi0g01.png diff --git a/basis/images/testing/png/suite/basi0g02.png b/extra/images/testing/png/suite/basi0g02.png similarity index 100% rename from basis/images/testing/png/suite/basi0g02.png rename to extra/images/testing/png/suite/basi0g02.png diff --git a/basis/images/testing/png/suite/basi0g04.png b/extra/images/testing/png/suite/basi0g04.png similarity index 100% rename from basis/images/testing/png/suite/basi0g04.png rename to extra/images/testing/png/suite/basi0g04.png diff --git a/basis/images/testing/png/suite/basi0g08.png b/extra/images/testing/png/suite/basi0g08.png similarity index 100% rename from basis/images/testing/png/suite/basi0g08.png rename to extra/images/testing/png/suite/basi0g08.png diff --git a/basis/images/testing/png/suite/basi0g16.png b/extra/images/testing/png/suite/basi0g16.png similarity index 100% rename from basis/images/testing/png/suite/basi0g16.png rename to extra/images/testing/png/suite/basi0g16.png diff --git a/basis/images/testing/png/suite/basi2c08.png b/extra/images/testing/png/suite/basi2c08.png similarity index 100% rename from basis/images/testing/png/suite/basi2c08.png rename to extra/images/testing/png/suite/basi2c08.png diff --git a/basis/images/testing/png/suite/basi2c16.png b/extra/images/testing/png/suite/basi2c16.png similarity index 100% rename from basis/images/testing/png/suite/basi2c16.png rename to extra/images/testing/png/suite/basi2c16.png diff --git a/basis/images/testing/png/suite/basi3p01.png b/extra/images/testing/png/suite/basi3p01.png similarity index 100% rename from basis/images/testing/png/suite/basi3p01.png rename to extra/images/testing/png/suite/basi3p01.png diff --git a/basis/images/testing/png/suite/basi3p02.png b/extra/images/testing/png/suite/basi3p02.png similarity index 100% rename from basis/images/testing/png/suite/basi3p02.png rename to extra/images/testing/png/suite/basi3p02.png diff --git a/basis/images/testing/png/suite/basi3p04.png b/extra/images/testing/png/suite/basi3p04.png similarity index 100% rename from basis/images/testing/png/suite/basi3p04.png rename to extra/images/testing/png/suite/basi3p04.png diff --git a/basis/images/testing/png/suite/basi3p08.png b/extra/images/testing/png/suite/basi3p08.png similarity index 100% rename from basis/images/testing/png/suite/basi3p08.png rename to extra/images/testing/png/suite/basi3p08.png diff --git a/basis/images/testing/png/suite/basi4a08.png b/extra/images/testing/png/suite/basi4a08.png similarity index 100% rename from basis/images/testing/png/suite/basi4a08.png rename to extra/images/testing/png/suite/basi4a08.png diff --git a/basis/images/testing/png/suite/basi4a16.png b/extra/images/testing/png/suite/basi4a16.png similarity index 100% rename from basis/images/testing/png/suite/basi4a16.png rename to extra/images/testing/png/suite/basi4a16.png diff --git a/basis/images/testing/png/suite/basi6a08.png b/extra/images/testing/png/suite/basi6a08.png similarity index 100% rename from basis/images/testing/png/suite/basi6a08.png rename to extra/images/testing/png/suite/basi6a08.png diff --git a/basis/images/testing/png/suite/basi6a16.png b/extra/images/testing/png/suite/basi6a16.png similarity index 100% rename from basis/images/testing/png/suite/basi6a16.png rename to extra/images/testing/png/suite/basi6a16.png diff --git a/basis/images/testing/png/suite/basn0g01.png b/extra/images/testing/png/suite/basn0g01.png similarity index 100% rename from basis/images/testing/png/suite/basn0g01.png rename to extra/images/testing/png/suite/basn0g01.png diff --git a/basis/images/testing/png/suite/basn0g02.png b/extra/images/testing/png/suite/basn0g02.png similarity index 100% rename from basis/images/testing/png/suite/basn0g02.png rename to extra/images/testing/png/suite/basn0g02.png diff --git a/basis/images/testing/png/suite/basn0g04.png b/extra/images/testing/png/suite/basn0g04.png similarity index 100% rename from basis/images/testing/png/suite/basn0g04.png rename to extra/images/testing/png/suite/basn0g04.png diff --git a/basis/images/testing/png/suite/basn0g08.png b/extra/images/testing/png/suite/basn0g08.png similarity index 100% rename from basis/images/testing/png/suite/basn0g08.png rename to extra/images/testing/png/suite/basn0g08.png diff --git a/basis/images/testing/png/suite/basn0g16.png b/extra/images/testing/png/suite/basn0g16.png similarity index 100% rename from basis/images/testing/png/suite/basn0g16.png rename to extra/images/testing/png/suite/basn0g16.png diff --git a/basis/images/testing/png/suite/basn2c08.png b/extra/images/testing/png/suite/basn2c08.png similarity index 100% rename from basis/images/testing/png/suite/basn2c08.png rename to extra/images/testing/png/suite/basn2c08.png diff --git a/basis/images/testing/png/suite/basn2c16.png b/extra/images/testing/png/suite/basn2c16.png similarity index 100% rename from basis/images/testing/png/suite/basn2c16.png rename to extra/images/testing/png/suite/basn2c16.png diff --git a/basis/images/testing/png/suite/basn3p01.png b/extra/images/testing/png/suite/basn3p01.png similarity index 100% rename from basis/images/testing/png/suite/basn3p01.png rename to extra/images/testing/png/suite/basn3p01.png diff --git a/basis/images/testing/png/suite/basn3p02.png b/extra/images/testing/png/suite/basn3p02.png similarity index 100% rename from basis/images/testing/png/suite/basn3p02.png rename to extra/images/testing/png/suite/basn3p02.png diff --git a/basis/images/testing/png/suite/basn3p04.png b/extra/images/testing/png/suite/basn3p04.png similarity index 100% rename from basis/images/testing/png/suite/basn3p04.png rename to extra/images/testing/png/suite/basn3p04.png diff --git a/basis/images/testing/png/suite/basn3p08.png b/extra/images/testing/png/suite/basn3p08.png similarity index 100% rename from basis/images/testing/png/suite/basn3p08.png rename to extra/images/testing/png/suite/basn3p08.png diff --git a/basis/images/testing/png/suite/basn4a08.png b/extra/images/testing/png/suite/basn4a08.png similarity index 100% rename from basis/images/testing/png/suite/basn4a08.png rename to extra/images/testing/png/suite/basn4a08.png diff --git a/basis/images/testing/png/suite/basn4a16.png b/extra/images/testing/png/suite/basn4a16.png similarity index 100% rename from basis/images/testing/png/suite/basn4a16.png rename to extra/images/testing/png/suite/basn4a16.png diff --git a/basis/images/testing/png/suite/basn6a08.png b/extra/images/testing/png/suite/basn6a08.png similarity index 100% rename from basis/images/testing/png/suite/basn6a08.png rename to extra/images/testing/png/suite/basn6a08.png diff --git a/basis/images/testing/png/suite/basn6a16.png b/extra/images/testing/png/suite/basn6a16.png similarity index 100% rename from basis/images/testing/png/suite/basn6a16.png rename to extra/images/testing/png/suite/basn6a16.png diff --git a/basis/images/testing/png/suite/bgai4a08.png b/extra/images/testing/png/suite/bgai4a08.png similarity index 100% rename from basis/images/testing/png/suite/bgai4a08.png rename to extra/images/testing/png/suite/bgai4a08.png diff --git a/basis/images/testing/png/suite/bgai4a16.png b/extra/images/testing/png/suite/bgai4a16.png similarity index 100% rename from basis/images/testing/png/suite/bgai4a16.png rename to extra/images/testing/png/suite/bgai4a16.png diff --git a/basis/images/testing/png/suite/bgan6a08.png b/extra/images/testing/png/suite/bgan6a08.png similarity index 100% rename from basis/images/testing/png/suite/bgan6a08.png rename to extra/images/testing/png/suite/bgan6a08.png diff --git a/basis/images/testing/png/suite/bgan6a16.png b/extra/images/testing/png/suite/bgan6a16.png similarity index 100% rename from basis/images/testing/png/suite/bgan6a16.png rename to extra/images/testing/png/suite/bgan6a16.png diff --git a/basis/images/testing/png/suite/bgbn4a08.png b/extra/images/testing/png/suite/bgbn4a08.png similarity index 100% rename from basis/images/testing/png/suite/bgbn4a08.png rename to extra/images/testing/png/suite/bgbn4a08.png diff --git a/basis/images/testing/png/suite/bggn4a16.png b/extra/images/testing/png/suite/bggn4a16.png similarity index 100% rename from basis/images/testing/png/suite/bggn4a16.png rename to extra/images/testing/png/suite/bggn4a16.png diff --git a/basis/images/testing/png/suite/bgwn6a08.png b/extra/images/testing/png/suite/bgwn6a08.png similarity index 100% rename from basis/images/testing/png/suite/bgwn6a08.png rename to extra/images/testing/png/suite/bgwn6a08.png diff --git a/basis/images/testing/png/suite/bgyn6a16.png b/extra/images/testing/png/suite/bgyn6a16.png similarity index 100% rename from basis/images/testing/png/suite/bgyn6a16.png rename to extra/images/testing/png/suite/bgyn6a16.png diff --git a/basis/images/testing/png/suite/ccwn2c08.png b/extra/images/testing/png/suite/ccwn2c08.png similarity index 100% rename from basis/images/testing/png/suite/ccwn2c08.png rename to extra/images/testing/png/suite/ccwn2c08.png diff --git a/basis/images/testing/png/suite/ccwn3p08.png b/extra/images/testing/png/suite/ccwn3p08.png similarity index 100% rename from basis/images/testing/png/suite/ccwn3p08.png rename to extra/images/testing/png/suite/ccwn3p08.png diff --git a/basis/images/testing/png/suite/cdfn2c08.png b/extra/images/testing/png/suite/cdfn2c08.png similarity index 100% rename from basis/images/testing/png/suite/cdfn2c08.png rename to extra/images/testing/png/suite/cdfn2c08.png diff --git a/basis/images/testing/png/suite/cdhn2c08.png b/extra/images/testing/png/suite/cdhn2c08.png similarity index 100% rename from basis/images/testing/png/suite/cdhn2c08.png rename to extra/images/testing/png/suite/cdhn2c08.png diff --git a/basis/images/testing/png/suite/cdsn2c08.png b/extra/images/testing/png/suite/cdsn2c08.png similarity index 100% rename from basis/images/testing/png/suite/cdsn2c08.png rename to extra/images/testing/png/suite/cdsn2c08.png diff --git a/basis/images/testing/png/suite/cdun2c08.png b/extra/images/testing/png/suite/cdun2c08.png similarity index 100% rename from basis/images/testing/png/suite/cdun2c08.png rename to extra/images/testing/png/suite/cdun2c08.png diff --git a/basis/images/testing/png/suite/ch1n3p04.png b/extra/images/testing/png/suite/ch1n3p04.png similarity index 100% rename from basis/images/testing/png/suite/ch1n3p04.png rename to extra/images/testing/png/suite/ch1n3p04.png diff --git a/basis/images/testing/png/suite/ch2n3p08.png b/extra/images/testing/png/suite/ch2n3p08.png similarity index 100% rename from basis/images/testing/png/suite/ch2n3p08.png rename to extra/images/testing/png/suite/ch2n3p08.png diff --git a/basis/images/testing/png/suite/cm0n0g04.png b/extra/images/testing/png/suite/cm0n0g04.png similarity index 100% rename from basis/images/testing/png/suite/cm0n0g04.png rename to extra/images/testing/png/suite/cm0n0g04.png diff --git a/basis/images/testing/png/suite/cm7n0g04.png b/extra/images/testing/png/suite/cm7n0g04.png similarity index 100% rename from basis/images/testing/png/suite/cm7n0g04.png rename to extra/images/testing/png/suite/cm7n0g04.png diff --git a/basis/images/testing/png/suite/cm9n0g04.png b/extra/images/testing/png/suite/cm9n0g04.png similarity index 100% rename from basis/images/testing/png/suite/cm9n0g04.png rename to extra/images/testing/png/suite/cm9n0g04.png diff --git a/basis/images/testing/png/suite/cs3n2c16.png b/extra/images/testing/png/suite/cs3n2c16.png similarity index 100% rename from basis/images/testing/png/suite/cs3n2c16.png rename to extra/images/testing/png/suite/cs3n2c16.png diff --git a/basis/images/testing/png/suite/cs3n3p08.png b/extra/images/testing/png/suite/cs3n3p08.png similarity index 100% rename from basis/images/testing/png/suite/cs3n3p08.png rename to extra/images/testing/png/suite/cs3n3p08.png diff --git a/basis/images/testing/png/suite/cs5n2c08.png b/extra/images/testing/png/suite/cs5n2c08.png similarity index 100% rename from basis/images/testing/png/suite/cs5n2c08.png rename to extra/images/testing/png/suite/cs5n2c08.png diff --git a/basis/images/testing/png/suite/cs5n3p08.png b/extra/images/testing/png/suite/cs5n3p08.png similarity index 100% rename from basis/images/testing/png/suite/cs5n3p08.png rename to extra/images/testing/png/suite/cs5n3p08.png diff --git a/basis/images/testing/png/suite/cs8n2c08.png b/extra/images/testing/png/suite/cs8n2c08.png similarity index 100% rename from basis/images/testing/png/suite/cs8n2c08.png rename to extra/images/testing/png/suite/cs8n2c08.png diff --git a/basis/images/testing/png/suite/cs8n3p08.png b/extra/images/testing/png/suite/cs8n3p08.png similarity index 100% rename from basis/images/testing/png/suite/cs8n3p08.png rename to extra/images/testing/png/suite/cs8n3p08.png diff --git a/basis/images/testing/png/suite/ct0n0g04.png b/extra/images/testing/png/suite/ct0n0g04.png similarity index 100% rename from basis/images/testing/png/suite/ct0n0g04.png rename to extra/images/testing/png/suite/ct0n0g04.png diff --git a/basis/images/testing/png/suite/ct1n0g04.png b/extra/images/testing/png/suite/ct1n0g04.png similarity index 100% rename from basis/images/testing/png/suite/ct1n0g04.png rename to extra/images/testing/png/suite/ct1n0g04.png diff --git a/basis/images/testing/png/suite/ctzn0g04.png b/extra/images/testing/png/suite/ctzn0g04.png similarity index 100% rename from basis/images/testing/png/suite/ctzn0g04.png rename to extra/images/testing/png/suite/ctzn0g04.png diff --git a/basis/images/testing/png/suite/f00n0g08.png b/extra/images/testing/png/suite/f00n0g08.png similarity index 100% rename from basis/images/testing/png/suite/f00n0g08.png rename to extra/images/testing/png/suite/f00n0g08.png diff --git a/basis/images/testing/png/suite/f00n2c08.png b/extra/images/testing/png/suite/f00n2c08.png similarity index 100% rename from basis/images/testing/png/suite/f00n2c08.png rename to extra/images/testing/png/suite/f00n2c08.png diff --git a/basis/images/testing/png/suite/f01n0g08.png b/extra/images/testing/png/suite/f01n0g08.png similarity index 100% rename from basis/images/testing/png/suite/f01n0g08.png rename to extra/images/testing/png/suite/f01n0g08.png diff --git a/basis/images/testing/png/suite/f01n2c08.png b/extra/images/testing/png/suite/f01n2c08.png similarity index 100% rename from basis/images/testing/png/suite/f01n2c08.png rename to extra/images/testing/png/suite/f01n2c08.png diff --git a/basis/images/testing/png/suite/f02n0g08.png b/extra/images/testing/png/suite/f02n0g08.png similarity index 100% rename from basis/images/testing/png/suite/f02n0g08.png rename to extra/images/testing/png/suite/f02n0g08.png diff --git a/basis/images/testing/png/suite/f02n2c08.png b/extra/images/testing/png/suite/f02n2c08.png similarity index 100% rename from basis/images/testing/png/suite/f02n2c08.png rename to extra/images/testing/png/suite/f02n2c08.png diff --git a/basis/images/testing/png/suite/f03n0g08.png b/extra/images/testing/png/suite/f03n0g08.png similarity index 100% rename from basis/images/testing/png/suite/f03n0g08.png rename to extra/images/testing/png/suite/f03n0g08.png diff --git a/basis/images/testing/png/suite/f03n2c08.png b/extra/images/testing/png/suite/f03n2c08.png similarity index 100% rename from basis/images/testing/png/suite/f03n2c08.png rename to extra/images/testing/png/suite/f03n2c08.png diff --git a/basis/images/testing/png/suite/f04n0g08.png b/extra/images/testing/png/suite/f04n0g08.png similarity index 100% rename from basis/images/testing/png/suite/f04n0g08.png rename to extra/images/testing/png/suite/f04n0g08.png diff --git a/basis/images/testing/png/suite/f04n2c08.png b/extra/images/testing/png/suite/f04n2c08.png similarity index 100% rename from basis/images/testing/png/suite/f04n2c08.png rename to extra/images/testing/png/suite/f04n2c08.png diff --git a/basis/images/testing/png/suite/g03n0g16.png b/extra/images/testing/png/suite/g03n0g16.png similarity index 100% rename from basis/images/testing/png/suite/g03n0g16.png rename to extra/images/testing/png/suite/g03n0g16.png diff --git a/basis/images/testing/png/suite/g03n2c08.png b/extra/images/testing/png/suite/g03n2c08.png similarity index 100% rename from basis/images/testing/png/suite/g03n2c08.png rename to extra/images/testing/png/suite/g03n2c08.png diff --git a/basis/images/testing/png/suite/g03n3p04.png b/extra/images/testing/png/suite/g03n3p04.png similarity index 100% rename from basis/images/testing/png/suite/g03n3p04.png rename to extra/images/testing/png/suite/g03n3p04.png diff --git a/basis/images/testing/png/suite/g04n0g16.png b/extra/images/testing/png/suite/g04n0g16.png similarity index 100% rename from basis/images/testing/png/suite/g04n0g16.png rename to extra/images/testing/png/suite/g04n0g16.png diff --git a/basis/images/testing/png/suite/g04n2c08.png b/extra/images/testing/png/suite/g04n2c08.png similarity index 100% rename from basis/images/testing/png/suite/g04n2c08.png rename to extra/images/testing/png/suite/g04n2c08.png diff --git a/basis/images/testing/png/suite/g04n3p04.png b/extra/images/testing/png/suite/g04n3p04.png similarity index 100% rename from basis/images/testing/png/suite/g04n3p04.png rename to extra/images/testing/png/suite/g04n3p04.png diff --git a/basis/images/testing/png/suite/g05n0g16.png b/extra/images/testing/png/suite/g05n0g16.png similarity index 100% rename from basis/images/testing/png/suite/g05n0g16.png rename to extra/images/testing/png/suite/g05n0g16.png diff --git a/basis/images/testing/png/suite/g05n2c08.png b/extra/images/testing/png/suite/g05n2c08.png similarity index 100% rename from basis/images/testing/png/suite/g05n2c08.png rename to extra/images/testing/png/suite/g05n2c08.png diff --git a/basis/images/testing/png/suite/g05n3p04.png b/extra/images/testing/png/suite/g05n3p04.png similarity index 100% rename from basis/images/testing/png/suite/g05n3p04.png rename to extra/images/testing/png/suite/g05n3p04.png diff --git a/basis/images/testing/png/suite/g07n0g16.png b/extra/images/testing/png/suite/g07n0g16.png similarity index 100% rename from basis/images/testing/png/suite/g07n0g16.png rename to extra/images/testing/png/suite/g07n0g16.png diff --git a/basis/images/testing/png/suite/g07n2c08.png b/extra/images/testing/png/suite/g07n2c08.png similarity index 100% rename from basis/images/testing/png/suite/g07n2c08.png rename to extra/images/testing/png/suite/g07n2c08.png diff --git a/basis/images/testing/png/suite/g07n3p04.png b/extra/images/testing/png/suite/g07n3p04.png similarity index 100% rename from basis/images/testing/png/suite/g07n3p04.png rename to extra/images/testing/png/suite/g07n3p04.png diff --git a/basis/images/testing/png/suite/g10n0g16.png b/extra/images/testing/png/suite/g10n0g16.png similarity index 100% rename from basis/images/testing/png/suite/g10n0g16.png rename to extra/images/testing/png/suite/g10n0g16.png diff --git a/basis/images/testing/png/suite/g10n2c08.png b/extra/images/testing/png/suite/g10n2c08.png similarity index 100% rename from basis/images/testing/png/suite/g10n2c08.png rename to extra/images/testing/png/suite/g10n2c08.png diff --git a/basis/images/testing/png/suite/g10n3p04.png b/extra/images/testing/png/suite/g10n3p04.png similarity index 100% rename from basis/images/testing/png/suite/g10n3p04.png rename to extra/images/testing/png/suite/g10n3p04.png diff --git a/basis/images/testing/png/suite/g25n0g16.png b/extra/images/testing/png/suite/g25n0g16.png similarity index 100% rename from basis/images/testing/png/suite/g25n0g16.png rename to extra/images/testing/png/suite/g25n0g16.png diff --git a/basis/images/testing/png/suite/g25n2c08.png b/extra/images/testing/png/suite/g25n2c08.png similarity index 100% rename from basis/images/testing/png/suite/g25n2c08.png rename to extra/images/testing/png/suite/g25n2c08.png diff --git a/basis/images/testing/png/suite/g25n3p04.png b/extra/images/testing/png/suite/g25n3p04.png similarity index 100% rename from basis/images/testing/png/suite/g25n3p04.png rename to extra/images/testing/png/suite/g25n3p04.png diff --git a/basis/images/testing/png/suite/oi1n0g16.png b/extra/images/testing/png/suite/oi1n0g16.png similarity index 100% rename from basis/images/testing/png/suite/oi1n0g16.png rename to extra/images/testing/png/suite/oi1n0g16.png diff --git a/basis/images/testing/png/suite/oi1n2c16.png b/extra/images/testing/png/suite/oi1n2c16.png similarity index 100% rename from basis/images/testing/png/suite/oi1n2c16.png rename to extra/images/testing/png/suite/oi1n2c16.png diff --git a/basis/images/testing/png/suite/oi2n0g16.png b/extra/images/testing/png/suite/oi2n0g16.png similarity index 100% rename from basis/images/testing/png/suite/oi2n0g16.png rename to extra/images/testing/png/suite/oi2n0g16.png diff --git a/basis/images/testing/png/suite/oi2n2c16.png b/extra/images/testing/png/suite/oi2n2c16.png similarity index 100% rename from basis/images/testing/png/suite/oi2n2c16.png rename to extra/images/testing/png/suite/oi2n2c16.png diff --git a/basis/images/testing/png/suite/oi4n0g16.png b/extra/images/testing/png/suite/oi4n0g16.png similarity index 100% rename from basis/images/testing/png/suite/oi4n0g16.png rename to extra/images/testing/png/suite/oi4n0g16.png diff --git a/basis/images/testing/png/suite/oi4n2c16.png b/extra/images/testing/png/suite/oi4n2c16.png similarity index 100% rename from basis/images/testing/png/suite/oi4n2c16.png rename to extra/images/testing/png/suite/oi4n2c16.png diff --git a/basis/images/testing/png/suite/oi9n0g16.png b/extra/images/testing/png/suite/oi9n0g16.png similarity index 100% rename from basis/images/testing/png/suite/oi9n0g16.png rename to extra/images/testing/png/suite/oi9n0g16.png diff --git a/basis/images/testing/png/suite/oi9n2c16.png b/extra/images/testing/png/suite/oi9n2c16.png similarity index 100% rename from basis/images/testing/png/suite/oi9n2c16.png rename to extra/images/testing/png/suite/oi9n2c16.png diff --git a/basis/images/testing/png/suite/pngsuite.doc b/extra/images/testing/png/suite/pngsuite.doc similarity index 100% rename from basis/images/testing/png/suite/pngsuite.doc rename to extra/images/testing/png/suite/pngsuite.doc diff --git a/basis/images/testing/png/suite/pngsuite_logo.png b/extra/images/testing/png/suite/pngsuite_logo.png similarity index 100% rename from basis/images/testing/png/suite/pngsuite_logo.png rename to extra/images/testing/png/suite/pngsuite_logo.png diff --git a/basis/images/testing/png/suite/pp0n2c16.png b/extra/images/testing/png/suite/pp0n2c16.png similarity index 100% rename from basis/images/testing/png/suite/pp0n2c16.png rename to extra/images/testing/png/suite/pp0n2c16.png diff --git a/basis/images/testing/png/suite/pp0n6a08.png b/extra/images/testing/png/suite/pp0n6a08.png similarity index 100% rename from basis/images/testing/png/suite/pp0n6a08.png rename to extra/images/testing/png/suite/pp0n6a08.png diff --git a/basis/images/testing/png/suite/ps1n0g08.png b/extra/images/testing/png/suite/ps1n0g08.png similarity index 100% rename from basis/images/testing/png/suite/ps1n0g08.png rename to extra/images/testing/png/suite/ps1n0g08.png diff --git a/basis/images/testing/png/suite/ps1n2c16.png b/extra/images/testing/png/suite/ps1n2c16.png similarity index 100% rename from basis/images/testing/png/suite/ps1n2c16.png rename to extra/images/testing/png/suite/ps1n2c16.png diff --git a/basis/images/testing/png/suite/ps2n0g08.png b/extra/images/testing/png/suite/ps2n0g08.png similarity index 100% rename from basis/images/testing/png/suite/ps2n0g08.png rename to extra/images/testing/png/suite/ps2n0g08.png diff --git a/basis/images/testing/png/suite/ps2n2c16.png b/extra/images/testing/png/suite/ps2n2c16.png similarity index 100% rename from basis/images/testing/png/suite/ps2n2c16.png rename to extra/images/testing/png/suite/ps2n2c16.png diff --git a/basis/images/testing/png/suite/s01i3p01.png b/extra/images/testing/png/suite/s01i3p01.png similarity index 100% rename from basis/images/testing/png/suite/s01i3p01.png rename to extra/images/testing/png/suite/s01i3p01.png diff --git a/basis/images/testing/png/suite/s01n3p01.png b/extra/images/testing/png/suite/s01n3p01.png similarity index 100% rename from basis/images/testing/png/suite/s01n3p01.png rename to extra/images/testing/png/suite/s01n3p01.png diff --git a/basis/images/testing/png/suite/s02i3p01.png b/extra/images/testing/png/suite/s02i3p01.png similarity index 100% rename from basis/images/testing/png/suite/s02i3p01.png rename to extra/images/testing/png/suite/s02i3p01.png diff --git a/basis/images/testing/png/suite/s02n3p01.png b/extra/images/testing/png/suite/s02n3p01.png similarity index 100% rename from basis/images/testing/png/suite/s02n3p01.png rename to extra/images/testing/png/suite/s02n3p01.png diff --git a/basis/images/testing/png/suite/s03i3p01.png b/extra/images/testing/png/suite/s03i3p01.png similarity index 100% rename from basis/images/testing/png/suite/s03i3p01.png rename to extra/images/testing/png/suite/s03i3p01.png diff --git a/basis/images/testing/png/suite/s03n3p01.png b/extra/images/testing/png/suite/s03n3p01.png similarity index 100% rename from basis/images/testing/png/suite/s03n3p01.png rename to extra/images/testing/png/suite/s03n3p01.png diff --git a/basis/images/testing/png/suite/s04i3p01.png b/extra/images/testing/png/suite/s04i3p01.png similarity index 100% rename from basis/images/testing/png/suite/s04i3p01.png rename to extra/images/testing/png/suite/s04i3p01.png diff --git a/basis/images/testing/png/suite/s04n3p01.png b/extra/images/testing/png/suite/s04n3p01.png similarity index 100% rename from basis/images/testing/png/suite/s04n3p01.png rename to extra/images/testing/png/suite/s04n3p01.png diff --git a/basis/images/testing/png/suite/s05i3p02.png b/extra/images/testing/png/suite/s05i3p02.png similarity index 100% rename from basis/images/testing/png/suite/s05i3p02.png rename to extra/images/testing/png/suite/s05i3p02.png diff --git a/basis/images/testing/png/suite/s05n3p02.png b/extra/images/testing/png/suite/s05n3p02.png similarity index 100% rename from basis/images/testing/png/suite/s05n3p02.png rename to extra/images/testing/png/suite/s05n3p02.png diff --git a/basis/images/testing/png/suite/s06i3p02.png b/extra/images/testing/png/suite/s06i3p02.png similarity index 100% rename from basis/images/testing/png/suite/s06i3p02.png rename to extra/images/testing/png/suite/s06i3p02.png diff --git a/basis/images/testing/png/suite/s06n3p02.png b/extra/images/testing/png/suite/s06n3p02.png similarity index 100% rename from basis/images/testing/png/suite/s06n3p02.png rename to extra/images/testing/png/suite/s06n3p02.png diff --git a/basis/images/testing/png/suite/s07i3p02.png b/extra/images/testing/png/suite/s07i3p02.png similarity index 100% rename from basis/images/testing/png/suite/s07i3p02.png rename to extra/images/testing/png/suite/s07i3p02.png diff --git a/basis/images/testing/png/suite/s07n3p02.png b/extra/images/testing/png/suite/s07n3p02.png similarity index 100% rename from basis/images/testing/png/suite/s07n3p02.png rename to extra/images/testing/png/suite/s07n3p02.png diff --git a/basis/images/testing/png/suite/s08i3p02.png b/extra/images/testing/png/suite/s08i3p02.png similarity index 100% rename from basis/images/testing/png/suite/s08i3p02.png rename to extra/images/testing/png/suite/s08i3p02.png diff --git a/basis/images/testing/png/suite/s08n3p02.png b/extra/images/testing/png/suite/s08n3p02.png similarity index 100% rename from basis/images/testing/png/suite/s08n3p02.png rename to extra/images/testing/png/suite/s08n3p02.png diff --git a/basis/images/testing/png/suite/s09i3p02.png b/extra/images/testing/png/suite/s09i3p02.png similarity index 100% rename from basis/images/testing/png/suite/s09i3p02.png rename to extra/images/testing/png/suite/s09i3p02.png diff --git a/basis/images/testing/png/suite/s09n3p02.png b/extra/images/testing/png/suite/s09n3p02.png similarity index 100% rename from basis/images/testing/png/suite/s09n3p02.png rename to extra/images/testing/png/suite/s09n3p02.png diff --git a/basis/images/testing/png/suite/s32i3p04.png b/extra/images/testing/png/suite/s32i3p04.png similarity index 100% rename from basis/images/testing/png/suite/s32i3p04.png rename to extra/images/testing/png/suite/s32i3p04.png diff --git a/basis/images/testing/png/suite/s32n3p04.png b/extra/images/testing/png/suite/s32n3p04.png similarity index 100% rename from basis/images/testing/png/suite/s32n3p04.png rename to extra/images/testing/png/suite/s32n3p04.png diff --git a/basis/images/testing/png/suite/s33i3p04.png b/extra/images/testing/png/suite/s33i3p04.png similarity index 100% rename from basis/images/testing/png/suite/s33i3p04.png rename to extra/images/testing/png/suite/s33i3p04.png diff --git a/basis/images/testing/png/suite/s33n3p04.png b/extra/images/testing/png/suite/s33n3p04.png similarity index 100% rename from basis/images/testing/png/suite/s33n3p04.png rename to extra/images/testing/png/suite/s33n3p04.png diff --git a/basis/images/testing/png/suite/s34i3p04.png b/extra/images/testing/png/suite/s34i3p04.png similarity index 100% rename from basis/images/testing/png/suite/s34i3p04.png rename to extra/images/testing/png/suite/s34i3p04.png diff --git a/basis/images/testing/png/suite/s34n3p04.png b/extra/images/testing/png/suite/s34n3p04.png similarity index 100% rename from basis/images/testing/png/suite/s34n3p04.png rename to extra/images/testing/png/suite/s34n3p04.png diff --git a/basis/images/testing/png/suite/s35i3p04.png b/extra/images/testing/png/suite/s35i3p04.png similarity index 100% rename from basis/images/testing/png/suite/s35i3p04.png rename to extra/images/testing/png/suite/s35i3p04.png diff --git a/basis/images/testing/png/suite/s35n3p04.png b/extra/images/testing/png/suite/s35n3p04.png similarity index 100% rename from basis/images/testing/png/suite/s35n3p04.png rename to extra/images/testing/png/suite/s35n3p04.png diff --git a/basis/images/testing/png/suite/s36i3p04.png b/extra/images/testing/png/suite/s36i3p04.png similarity index 100% rename from basis/images/testing/png/suite/s36i3p04.png rename to extra/images/testing/png/suite/s36i3p04.png diff --git a/basis/images/testing/png/suite/s36n3p04.png b/extra/images/testing/png/suite/s36n3p04.png similarity index 100% rename from basis/images/testing/png/suite/s36n3p04.png rename to extra/images/testing/png/suite/s36n3p04.png diff --git a/basis/images/testing/png/suite/s37i3p04.png b/extra/images/testing/png/suite/s37i3p04.png similarity index 100% rename from basis/images/testing/png/suite/s37i3p04.png rename to extra/images/testing/png/suite/s37i3p04.png diff --git a/basis/images/testing/png/suite/s37n3p04.png b/extra/images/testing/png/suite/s37n3p04.png similarity index 100% rename from basis/images/testing/png/suite/s37n3p04.png rename to extra/images/testing/png/suite/s37n3p04.png diff --git a/basis/images/testing/png/suite/s38i3p04.png b/extra/images/testing/png/suite/s38i3p04.png similarity index 100% rename from basis/images/testing/png/suite/s38i3p04.png rename to extra/images/testing/png/suite/s38i3p04.png diff --git a/basis/images/testing/png/suite/s38n3p04.png b/extra/images/testing/png/suite/s38n3p04.png similarity index 100% rename from basis/images/testing/png/suite/s38n3p04.png rename to extra/images/testing/png/suite/s38n3p04.png diff --git a/basis/images/testing/png/suite/s39i3p04.png b/extra/images/testing/png/suite/s39i3p04.png similarity index 100% rename from basis/images/testing/png/suite/s39i3p04.png rename to extra/images/testing/png/suite/s39i3p04.png diff --git a/basis/images/testing/png/suite/s39n3p04.png b/extra/images/testing/png/suite/s39n3p04.png similarity index 100% rename from basis/images/testing/png/suite/s39n3p04.png rename to extra/images/testing/png/suite/s39n3p04.png diff --git a/basis/images/testing/png/suite/s40i3p04.png b/extra/images/testing/png/suite/s40i3p04.png similarity index 100% rename from basis/images/testing/png/suite/s40i3p04.png rename to extra/images/testing/png/suite/s40i3p04.png diff --git a/basis/images/testing/png/suite/s40n3p04.png b/extra/images/testing/png/suite/s40n3p04.png similarity index 100% rename from basis/images/testing/png/suite/s40n3p04.png rename to extra/images/testing/png/suite/s40n3p04.png diff --git a/basis/images/testing/png/suite/tbbn1g04.png b/extra/images/testing/png/suite/tbbn1g04.png similarity index 100% rename from basis/images/testing/png/suite/tbbn1g04.png rename to extra/images/testing/png/suite/tbbn1g04.png diff --git a/basis/images/testing/png/suite/tbbn2c16.png b/extra/images/testing/png/suite/tbbn2c16.png similarity index 100% rename from basis/images/testing/png/suite/tbbn2c16.png rename to extra/images/testing/png/suite/tbbn2c16.png diff --git a/basis/images/testing/png/suite/tbbn3p08.png b/extra/images/testing/png/suite/tbbn3p08.png similarity index 100% rename from basis/images/testing/png/suite/tbbn3p08.png rename to extra/images/testing/png/suite/tbbn3p08.png diff --git a/basis/images/testing/png/suite/tbgn2c16.png b/extra/images/testing/png/suite/tbgn2c16.png similarity index 100% rename from basis/images/testing/png/suite/tbgn2c16.png rename to extra/images/testing/png/suite/tbgn2c16.png diff --git a/basis/images/testing/png/suite/tbgn3p08.png b/extra/images/testing/png/suite/tbgn3p08.png similarity index 100% rename from basis/images/testing/png/suite/tbgn3p08.png rename to extra/images/testing/png/suite/tbgn3p08.png diff --git a/basis/images/testing/png/suite/tbrn2c08.png b/extra/images/testing/png/suite/tbrn2c08.png similarity index 100% rename from basis/images/testing/png/suite/tbrn2c08.png rename to extra/images/testing/png/suite/tbrn2c08.png diff --git a/basis/images/testing/png/suite/tbwn1g16.png b/extra/images/testing/png/suite/tbwn1g16.png similarity index 100% rename from basis/images/testing/png/suite/tbwn1g16.png rename to extra/images/testing/png/suite/tbwn1g16.png diff --git a/basis/images/testing/png/suite/tbwn3p08.png b/extra/images/testing/png/suite/tbwn3p08.png similarity index 100% rename from basis/images/testing/png/suite/tbwn3p08.png rename to extra/images/testing/png/suite/tbwn3p08.png diff --git a/basis/images/testing/png/suite/tbyn3p08.png b/extra/images/testing/png/suite/tbyn3p08.png similarity index 100% rename from basis/images/testing/png/suite/tbyn3p08.png rename to extra/images/testing/png/suite/tbyn3p08.png diff --git a/basis/images/testing/png/suite/tp0n1g08.png b/extra/images/testing/png/suite/tp0n1g08.png similarity index 100% rename from basis/images/testing/png/suite/tp0n1g08.png rename to extra/images/testing/png/suite/tp0n1g08.png diff --git a/basis/images/testing/png/suite/tp0n2c08.png b/extra/images/testing/png/suite/tp0n2c08.png similarity index 100% rename from basis/images/testing/png/suite/tp0n2c08.png rename to extra/images/testing/png/suite/tp0n2c08.png diff --git a/basis/images/testing/png/suite/tp0n3p08.png b/extra/images/testing/png/suite/tp0n3p08.png similarity index 100% rename from basis/images/testing/png/suite/tp0n3p08.png rename to extra/images/testing/png/suite/tp0n3p08.png diff --git a/basis/images/testing/png/suite/tp1n3p08.png b/extra/images/testing/png/suite/tp1n3p08.png similarity index 100% rename from basis/images/testing/png/suite/tp1n3p08.png rename to extra/images/testing/png/suite/tp1n3p08.png diff --git a/basis/images/testing/png/suite/x00n0g01.png b/extra/images/testing/png/suite/x00n0g01.png similarity index 100% rename from basis/images/testing/png/suite/x00n0g01.png rename to extra/images/testing/png/suite/x00n0g01.png diff --git a/basis/images/testing/png/suite/xcrn0g04.png b/extra/images/testing/png/suite/xcrn0g04.png similarity index 100% rename from basis/images/testing/png/suite/xcrn0g04.png rename to extra/images/testing/png/suite/xcrn0g04.png diff --git a/basis/images/testing/png/suite/xlfn0g04.png b/extra/images/testing/png/suite/xlfn0g04.png similarity index 100% rename from basis/images/testing/png/suite/xlfn0g04.png rename to extra/images/testing/png/suite/xlfn0g04.png diff --git a/basis/images/testing/png/suite/z00n2c08.png b/extra/images/testing/png/suite/z00n2c08.png similarity index 100% rename from basis/images/testing/png/suite/z00n2c08.png rename to extra/images/testing/png/suite/z00n2c08.png diff --git a/basis/images/testing/png/suite/z03n2c08.png b/extra/images/testing/png/suite/z03n2c08.png similarity index 100% rename from basis/images/testing/png/suite/z03n2c08.png rename to extra/images/testing/png/suite/z03n2c08.png diff --git a/basis/images/testing/png/suite/z06n2c08.png b/extra/images/testing/png/suite/z06n2c08.png similarity index 100% rename from basis/images/testing/png/suite/z06n2c08.png rename to extra/images/testing/png/suite/z06n2c08.png diff --git a/basis/images/testing/png/suite/z09n2c08.png b/extra/images/testing/png/suite/z09n2c08.png similarity index 100% rename from basis/images/testing/png/suite/z09n2c08.png rename to extra/images/testing/png/suite/z09n2c08.png diff --git a/basis/images/testing/png/z00n2c08.fig b/extra/images/testing/png/z00n2c08.fig similarity index 100% rename from basis/images/testing/png/z00n2c08.fig rename to extra/images/testing/png/z00n2c08.fig diff --git a/basis/images/testing/png/z00n2c08.png b/extra/images/testing/png/z00n2c08.png similarity index 100% rename from basis/images/testing/png/z00n2c08.png rename to extra/images/testing/png/z00n2c08.png diff --git a/basis/images/testing/png/z03n2c08.fig b/extra/images/testing/png/z03n2c08.fig similarity index 100% rename from basis/images/testing/png/z03n2c08.fig rename to extra/images/testing/png/z03n2c08.fig diff --git a/basis/images/testing/png/z03n2c08.png b/extra/images/testing/png/z03n2c08.png similarity index 100% rename from basis/images/testing/png/z03n2c08.png rename to extra/images/testing/png/z03n2c08.png diff --git a/basis/images/testing/png/z06n2c08.fig b/extra/images/testing/png/z06n2c08.fig similarity index 100% rename from basis/images/testing/png/z06n2c08.fig rename to extra/images/testing/png/z06n2c08.fig diff --git a/basis/images/testing/png/z06n2c08.png b/extra/images/testing/png/z06n2c08.png similarity index 100% rename from basis/images/testing/png/z06n2c08.png rename to extra/images/testing/png/z06n2c08.png diff --git a/basis/images/testing/png/z09n2c08.fig b/extra/images/testing/png/z09n2c08.fig similarity index 100% rename from basis/images/testing/png/z09n2c08.fig rename to extra/images/testing/png/z09n2c08.fig diff --git a/basis/images/testing/png/z09n2c08.png b/extra/images/testing/png/z09n2c08.png similarity index 100% rename from basis/images/testing/png/z09n2c08.png rename to extra/images/testing/png/z09n2c08.png diff --git a/basis/images/testing/testing-docs.factor b/extra/images/testing/testing-docs.factor similarity index 100% rename from basis/images/testing/testing-docs.factor rename to extra/images/testing/testing-docs.factor diff --git a/basis/images/testing/testing.factor b/extra/images/testing/testing.factor similarity index 100% rename from basis/images/testing/testing.factor rename to extra/images/testing/testing.factor diff --git a/basis/images/testing/tiff/alpha.fig b/extra/images/testing/tiff/alpha.fig similarity index 100% rename from basis/images/testing/tiff/alpha.fig rename to extra/images/testing/tiff/alpha.fig diff --git a/basis/images/testing/tiff/alpha.tiff b/extra/images/testing/tiff/alpha.tiff similarity index 100% rename from basis/images/testing/tiff/alpha.tiff rename to extra/images/testing/tiff/alpha.tiff diff --git a/basis/images/testing/tiff/color_spectrum.fig b/extra/images/testing/tiff/color_spectrum.fig similarity index 100% rename from basis/images/testing/tiff/color_spectrum.fig rename to extra/images/testing/tiff/color_spectrum.fig diff --git a/basis/images/testing/tiff/color_spectrum.tiff b/extra/images/testing/tiff/color_spectrum.tiff similarity index 100% rename from basis/images/testing/tiff/color_spectrum.tiff rename to extra/images/testing/tiff/color_spectrum.tiff diff --git a/basis/images/testing/tiff/elephants.tiff b/extra/images/testing/tiff/elephants.tiff similarity index 100% rename from basis/images/testing/tiff/elephants.tiff rename to extra/images/testing/tiff/elephants.tiff diff --git a/basis/images/testing/tiff/noise.fig b/extra/images/testing/tiff/noise.fig similarity index 100% rename from basis/images/testing/tiff/noise.fig rename to extra/images/testing/tiff/noise.fig diff --git a/basis/images/testing/tiff/noise.tiff b/extra/images/testing/tiff/noise.tiff similarity index 100% rename from basis/images/testing/tiff/noise.tiff rename to extra/images/testing/tiff/noise.tiff diff --git a/basis/images/testing/tiff/octagon.fig b/extra/images/testing/tiff/octagon.fig similarity index 100% rename from basis/images/testing/tiff/octagon.fig rename to extra/images/testing/tiff/octagon.fig diff --git a/basis/images/testing/tiff/octagon.tiff b/extra/images/testing/tiff/octagon.tiff similarity index 100% rename from basis/images/testing/tiff/octagon.tiff rename to extra/images/testing/tiff/octagon.tiff diff --git a/basis/images/testing/tiff/rgb.fig b/extra/images/testing/tiff/rgb.fig similarity index 100% rename from basis/images/testing/tiff/rgb.fig rename to extra/images/testing/tiff/rgb.fig diff --git a/basis/images/testing/tiff/rgb.tiff b/extra/images/testing/tiff/rgb.tiff similarity index 100% rename from basis/images/testing/tiff/rgb.tiff rename to extra/images/testing/tiff/rgb.tiff diff --git a/vm/aging_collector.cpp b/vm/aging_collector.cpp index 5e284be587..3572677aa6 100644 --- a/vm/aging_collector.cpp +++ b/vm/aging_collector.cpp @@ -6,7 +6,6 @@ namespace factor aging_collector::aging_collector(factor_vm *parent_) : copying_collector( parent_, - &parent_->gc_stats.aging_stats, parent_->data->aging, aging_policy(parent_)) {} @@ -22,28 +21,40 @@ void factor_vm::collect_aging() current_gc->op = collect_to_tenured_op; to_tenured_collector collector(this); + + current_gc->event->started_code_scan(); collector.trace_cards(data->tenured, card_points_to_aging, - simple_unmarker(card_mark_mask)); - collector.cheneys_algorithm(); + full_unmarker()); + current_gc->event->ended_card_scan(collector.cards_scanned,collector.decks_scanned); + + current_gc->event->started_code_scan(); + collector.trace_code_heap_roots(&code->points_to_aging); + current_gc->event->ended_code_scan(collector.code_blocks_scanned); + + collector.tenure_reachable_objects(); + + current_gc->event->started_code_sweep(); + update_code_heap_for_minor_gc(&code->points_to_aging); + current_gc->event->ended_code_sweep(); } { /* If collection fails here, do a to_tenured collection. */ current_gc->op = collect_aging_op; std::swap(data->aging,data->aging_semispace); - reset_generation(data->aging); + data->reset_generation(data->aging); aging_collector collector(this); collector.trace_roots(); collector.trace_contexts(); - collector.trace_code_heap_roots(&code->points_to_aging); - collector.cheneys_algorithm(); - update_code_heap_for_minor_gc(&code->points_to_aging); - nursery.here = nursery.start; + collector.cheneys_algorithm(); + + data->reset_generation(&nursery); code->points_to_nursery.clear(); + code->points_to_aging.clear(); } } diff --git a/vm/aging_collector.hpp b/vm/aging_collector.hpp index 1fa82972ff..56550b211a 100644 --- a/vm/aging_collector.hpp +++ b/vm/aging_collector.hpp @@ -3,9 +3,10 @@ namespace factor struct aging_policy { factor_vm *parent; - zone *aging, *tenured; + aging_space *aging; + tenured_space *tenured; - aging_policy(factor_vm *parent_) : + explicit aging_policy(factor_vm *parent_) : parent(parent_), aging(parent->data->aging), tenured(parent->data->tenured) {} @@ -14,10 +15,14 @@ struct aging_policy { { return !(aging->contains_p(untagged) || tenured->contains_p(untagged)); } + + void promoted_object(object *obj) {} + + void visited_object(object *obj) {} }; struct aging_collector : copying_collector { - aging_collector(factor_vm *parent_); + explicit aging_collector(factor_vm *parent_); }; } diff --git a/vm/aging_space.hpp b/vm/aging_space.hpp index c2ec2a645e..7a28f54ebf 100644 --- a/vm/aging_space.hpp +++ b/vm/aging_space.hpp @@ -1,8 +1,29 @@ namespace factor { -struct aging_space : old_space { - aging_space(cell size, cell start) : old_space(size,start) {} +struct aging_space : bump_allocator { + object_start_map starts; + + explicit aging_space(cell size, cell start) : + bump_allocator(size,start), starts(size,start) {} + + object *allot(cell size) + { + if(here + size > end) return NULL; + + object *obj = bump_allocator::allot(size); + starts.record_object_start_offset(obj); + return obj; + } + + cell next_object_after(cell scan) + { + cell size = ((object *)scan)->size(); + if(scan + size < here) + return scan + size; + else + return 0; + } }; } diff --git a/vm/alien.cpp b/vm/alien.cpp index ed3adf5c9b..d07b6e353b 100755 --- a/vm/alien.cpp +++ b/vm/alien.cpp @@ -14,7 +14,10 @@ char *factor_vm::pinned_alien_offset(cell obj) alien *ptr = untag(obj); if(to_boolean(ptr->expired)) general_error(ERROR_EXPIRED,obj,false_object,NULL); - return pinned_alien_offset(ptr->base) + ptr->displacement; + if(to_boolean(ptr->base)) + type_error(ALIEN_TYPE,obj); + else + return (char *)ptr->address; } case F_TYPE: return NULL; @@ -27,8 +30,8 @@ char *factor_vm::pinned_alien_offset(cell obj) /* make an alien */ cell factor_vm::allot_alien(cell delegate_, cell displacement) { - gc_root delegate(delegate_,this); - gc_root new_alien(allot(sizeof(alien)),this); + data_root delegate(delegate_,this); + data_root new_alien(allot(sizeof(alien)),this); if(delegate.type_p(ALIEN_TYPE)) { @@ -41,6 +44,7 @@ cell factor_vm::allot_alien(cell delegate_, cell displacement) new_alien->displacement = displacement; new_alien->expired = false_object; + new_alien->update_address(); return new_alien.value(); } @@ -113,9 +117,9 @@ DEFINE_ALIEN_ACCESSOR(cell,void *,box_alien,pinned_alien_offset) /* open a native library and push a handle */ void factor_vm::primitive_dlopen() { - gc_root path(dpop(),this); + data_root path(dpop(),this); path.untag_check(this); - gc_root library(allot(sizeof(dll)),this); + data_root library(allot(sizeof(dll)),this); library->path = path.value(); ffi_dlopen(library.untagged()); dpush(library.value()); @@ -124,8 +128,8 @@ void factor_vm::primitive_dlopen() /* look up a symbol in a native library */ void factor_vm::primitive_dlsym() { - gc_root library(dpop(),this); - gc_root name(dpop(),this); + data_root library(dpop(),this); + data_root name(dpop(),this); name.untag_check(this); symbol_char *sym = name->data(); @@ -168,12 +172,7 @@ char *factor_vm::alien_offset(cell obj) case BYTE_ARRAY_TYPE: return untag(obj)->data(); case ALIEN_TYPE: - { - alien *ptr = untag(obj); - if(to_boolean(ptr->expired)) - general_error(ERROR_EXPIRED,obj,false_object,NULL); - return alien_offset(ptr->base) + ptr->displacement; - } + return (char *)untag(obj)->address; case F_TYPE: return NULL; default: diff --git a/vm/arrays.cpp b/vm/arrays.cpp index 1f60515bb8..3060ac70a3 100644 --- a/vm/arrays.cpp +++ b/vm/arrays.cpp @@ -6,8 +6,8 @@ namespace factor /* make a new array with an initial element */ array *factor_vm::allot_array(cell capacity, cell fill_) { - gc_root fill(fill_,this); - gc_root new_array(allot_array_internal(capacity),this); + data_root fill(fill_,this); + data_root new_array(allot_uninitialized_array(capacity),this); memset_cell(new_array->data(),fill.value(),capacity * sizeof(cell)); return new_array.untagged(); } @@ -22,17 +22,17 @@ void factor_vm::primitive_array() cell factor_vm::allot_array_1(cell obj_) { - gc_root obj(obj_,this); - gc_root a(allot_array_internal(1),this); + data_root obj(obj_,this); + data_root a(allot_uninitialized_array(1),this); set_array_nth(a.untagged(),0,obj.value()); return a.value(); } cell factor_vm::allot_array_2(cell v1_, cell v2_) { - gc_root v1(v1_,this); - gc_root v2(v2_,this); - gc_root a(allot_array_internal(2),this); + data_root v1(v1_,this); + data_root v2(v2_,this); + data_root a(allot_uninitialized_array(2),this); set_array_nth(a.untagged(),0,v1.value()); set_array_nth(a.untagged(),1,v2.value()); return a.value(); @@ -40,11 +40,11 @@ cell factor_vm::allot_array_2(cell v1_, cell v2_) cell factor_vm::allot_array_4(cell v1_, cell v2_, cell v3_, cell v4_) { - gc_root v1(v1_,this); - gc_root v2(v2_,this); - gc_root v3(v3_,this); - gc_root v4(v4_,this); - gc_root a(allot_array_internal(4),this); + data_root v1(v1_,this); + data_root v2(v2_,this); + data_root v3(v3_,this); + data_root v4(v4_,this); + data_root a(allot_uninitialized_array(4),this); set_array_nth(a.untagged(),0,v1.value()); set_array_nth(a.untagged(),1,v2.value()); set_array_nth(a.untagged(),2,v3.value()); @@ -62,7 +62,7 @@ void factor_vm::primitive_resize_array() void growable_array::add(cell elt_) { factor_vm *parent = elements.parent; - gc_root elt(elt_,parent); + data_root elt(elt_,parent); if(count == array_capacity(elements.untagged())) elements = parent->reallot_array(elements.untagged(),count * 2); @@ -72,7 +72,7 @@ void growable_array::add(cell elt_) void growable_array::append(array *elts_) { factor_vm *parent = elements.parent; - gc_root elts(elts_,parent); + data_root elts(elts_,parent); cell capacity = array_capacity(elts.untagged()); if(count + capacity > array_capacity(elements.untagged())) { diff --git a/vm/arrays.hpp b/vm/arrays.hpp index 48be881230..8eb2b530b0 100755 --- a/vm/arrays.hpp +++ b/vm/arrays.hpp @@ -15,7 +15,6 @@ inline void factor_vm::set_array_nth(array *array, cell slot, cell value) #ifdef FACTOR_DEBUG assert(slot < array_capacity(array)); assert(array->h.hi_tag() == ARRAY_TYPE); - check_tagged_pointer(value); #endif cell *slot_ptr = &array->data()[slot]; *slot_ptr = value; @@ -24,7 +23,7 @@ inline void factor_vm::set_array_nth(array *array, cell slot, cell value) struct growable_array { cell count; - gc_root elements; + data_root elements; explicit growable_array(factor_vm *parent, cell capacity = 10) : count(0), elements(parent->allot_array(capacity,false_object),parent) {} diff --git a/vm/bignum.cpp b/vm/bignum.cpp index d8c5452b08..5a391e7625 100755 --- a/vm/bignum.cpp +++ b/vm/bignum.cpp @@ -1299,7 +1299,7 @@ bignum *factor_vm::bignum_digit_to_bignum(bignum_digit_type digit, int negative_ bignum *factor_vm::allot_bignum(bignum_length_type length, int negative_p) { BIGNUM_ASSERT ((length >= 0) || (length < BIGNUM_RADIX)); - bignum * result = allot_array_internal(length + 1); + bignum * result = allot_uninitialized_array(length + 1); BIGNUM_SET_NEGATIVE_P (result, negative_p); return (result); } diff --git a/vm/bitwise_hacks.hpp b/vm/bitwise_hacks.hpp new file mode 100644 index 0000000000..dc685bb28c --- /dev/null +++ b/vm/bitwise_hacks.hpp @@ -0,0 +1,67 @@ +namespace factor +{ + +/* These algorithms were snarfed from various places. I did not come up with them myself */ + +inline cell popcount(u64 x) +{ + u64 k1 = 0x5555555555555555ll; + u64 k2 = 0x3333333333333333ll; + u64 k4 = 0x0f0f0f0f0f0f0f0fll; + u64 kf = 0x0101010101010101ll; + x = x - ((x >> 1) & k1); // put count of each 2 bits into those 2 bits + x = (x & k2) + ((x >> 2) & k2); // put count of each 4 bits into those 4 bits + x = (x + (x >> 4)) & k4 ; // put count of each 8 bits into those 8 bits + x = (x * kf) >> 56; // returns 8 most significant bits of x + (x<<8) + (x<<16) + (x<<24) + ... + + return (cell)x; +} + +inline cell log2(u64 x) +{ +#ifdef FACTOR_AMD64 + cell n; + asm ("bsr %1, %0;":"=r"(n):"r"((cell)x)); +#else + cell n = 0; + if (x >= (u64)1 << 32) { x >>= 32; n += 32; } + if (x >= (u64)1 << 16) { x >>= 16; n += 16; } + if (x >= (u64)1 << 8) { x >>= 8; n += 8; } + if (x >= (u64)1 << 4) { x >>= 4; n += 4; } + if (x >= (u64)1 << 2) { x >>= 2; n += 2; } + if (x >= (u64)1 << 1) { n += 1; } +#endif + return n; +} + +inline cell log2(u16 x) +{ +#if defined(FACTOR_X86) || defined(FACTOR_AMD64) + cell n; + asm ("bsr %1, %0;":"=r"(n):"r"((cell)x)); +#else + cell n = 0; + if (x >= 1 << 8) { x >>= 8; n += 8; } + if (x >= 1 << 4) { x >>= 4; n += 4; } + if (x >= 1 << 2) { x >>= 2; n += 2; } + if (x >= 1 << 1) { n += 1; } +#endif + return n; +} + +inline cell rightmost_clear_bit(u64 x) +{ + return log2(~x & (x + 1)); +} + +inline cell rightmost_set_bit(u64 x) +{ + return log2(x & -x); +} + +inline cell rightmost_set_bit(u16 x) +{ + return log2((u16)(x & -x)); +} + +} diff --git a/vm/bump_allocator.hpp b/vm/bump_allocator.hpp new file mode 100644 index 0000000000..5488c65323 --- /dev/null +++ b/vm/bump_allocator.hpp @@ -0,0 +1,37 @@ +namespace factor +{ + +template struct bump_allocator { + /* offset of 'here' and 'end' is hardcoded in compiler backends */ + cell here; + cell start; + cell end; + cell size; + + explicit bump_allocator(cell size_, cell start_) : + here(start_), start(start_), end(start_ + size_), size(size_) {} + + bool contains_p(Block *block) + { + return ((cell)block - start) < size; + } + + Block *allot(cell size) + { + cell h = here; + here = h + align(size,data_alignment); + return (Block *)h; + } + + cell occupied_space() + { + return here - start; + } + + cell free_space() + { + return end - here; + } +}; + +} diff --git a/vm/byte_arrays.cpp b/vm/byte_arrays.cpp index 56b5db7ad8..b317c39f62 100644 --- a/vm/byte_arrays.cpp +++ b/vm/byte_arrays.cpp @@ -5,7 +5,7 @@ namespace factor byte_array *factor_vm::allot_byte_array(cell size) { - byte_array *array = allot_array_internal(size); + byte_array *array = allot_uninitialized_array(size); memset(array + 1,0,size); return array; } @@ -19,7 +19,7 @@ void factor_vm::primitive_byte_array() void factor_vm::primitive_uninitialized_byte_array() { cell size = unbox_array_size(); - dpush(tag(allot_array_internal(size))); + dpush(tag(allot_uninitialized_array(size))); } void factor_vm::primitive_resize_byte_array() @@ -43,7 +43,7 @@ void growable_byte_array::append_bytes(void *elts, cell len) void growable_byte_array::append_byte_array(cell byte_array_) { - gc_root byte_array(byte_array_,elements.parent); + data_root byte_array(byte_array_,elements.parent); cell len = array_capacity(byte_array.untagged()); cell new_size = count + len; diff --git a/vm/byte_arrays.hpp b/vm/byte_arrays.hpp index 8ca95d9d1e..a3d6fcf941 100755 --- a/vm/byte_arrays.hpp +++ b/vm/byte_arrays.hpp @@ -3,7 +3,7 @@ namespace factor struct growable_byte_array { cell count; - gc_root elements; + data_root elements; explicit growable_byte_array(factor_vm *parent,cell capacity = 40) : count(0), elements(parent->allot_byte_array(capacity),parent) { } @@ -13,4 +13,17 @@ struct growable_byte_array { void trim(); }; +template byte_array *factor_vm::byte_array_from_value(Type *value) +{ + return byte_array_from_values(value,1); +} + +template byte_array *factor_vm::byte_array_from_values(Type *values, cell len) +{ + cell size = sizeof(Type) * len; + byte_array *data = allot_uninitialized_array(size); + memcpy(data->data(),values,size); + return data; +} + } diff --git a/vm/callbacks.cpp b/vm/callbacks.cpp index dca0eb6c24..4fe19c0bc0 100644 --- a/vm/callbacks.cpp +++ b/vm/callbacks.cpp @@ -21,7 +21,7 @@ void factor_vm::init_callbacks(cell size) void callback_heap::update(callback *stub) { - tagged code_template(parent->userenv[CALLBACK_STUB]); + tagged code_template(parent->special_objects[CALLBACK_STUB]); cell rel_class = untag_fixnum(array_nth(code_template.untagged(),1)); cell offset = untag_fixnum(array_nth(code_template.untagged(),3)); @@ -35,18 +35,18 @@ void callback_heap::update(callback *stub) callback *callback_heap::add(code_block *compiled) { - tagged code_template(parent->userenv[CALLBACK_STUB]); + tagged code_template(parent->special_objects[CALLBACK_STUB]); tagged insns(array_nth(code_template.untagged(),0)); cell size = array_capacity(insns.untagged()); - cell bump = align8(size) + sizeof(callback); + cell bump = align(size,sizeof(cell)) + sizeof(callback); if(here + bump > seg->end) fatal_error("Out of callback space",0); callback *stub = (callback *)here; stub->compiled = compiled; memcpy(stub + 1,insns->data(),size); - stub->size = align8(size); + stub->size = align(size,sizeof(cell)); here += bump; update(stub); diff --git a/vm/callstack.cpp b/vm/callstack.cpp index 4721fc4ece..85f392af0e 100755 --- a/vm/callstack.cpp +++ b/vm/callstack.cpp @@ -76,7 +76,7 @@ code_block *factor_vm::frame_code(stack_frame *frame) return (code_block *)frame->xt - 1; } -cell factor_vm::frame_type(stack_frame *frame) +code_block_type factor_vm::frame_type(stack_frame *frame) { return frame_code(frame)->type(); } @@ -97,7 +97,7 @@ cell factor_vm::frame_scan(stack_frame *frame) { switch(frame_type(frame)) { - case QUOTATION_TYPE: + case code_block_unoptimized: { cell quot = frame_executing(frame); if(to_boolean(quot)) @@ -111,7 +111,7 @@ cell factor_vm::frame_scan(stack_frame *frame) else return false_object; } - case WORD_TYPE: + case code_block_optimized: return false_object; default: critical_error("Bad frame type",frame_type(frame)); @@ -130,8 +130,8 @@ struct stack_frame_accumulator { void operator()(stack_frame *frame) { - gc_root executing(parent->frame_executing(frame),parent); - gc_root scan(parent->frame_scan(frame),parent); + data_root executing(parent->frame_executing(frame),parent); + data_root scan(parent->frame_scan(frame),parent); frames.add(executing.value()); frames.add(scan.value()); @@ -142,7 +142,7 @@ struct stack_frame_accumulator { void factor_vm::primitive_callstack_to_array() { - gc_root callstack(dpop(),this); + data_root callstack(dpop(),this); stack_frame_accumulator accum(this); iterate_callstack_object(callstack.untagged(),accum); @@ -184,8 +184,8 @@ void factor_vm::primitive_innermost_stack_frame_scan() void factor_vm::primitive_set_innermost_stack_frame_quot() { - gc_root callstack(dpop(),this); - gc_root quot(dpop(),this); + data_root callstack(dpop(),this); + data_root quot(dpop(),this); callstack.untag_check(this); quot.untag_check(this); diff --git a/vm/callstack.hpp b/vm/callstack.hpp index 4d44904281..76bf3ecea4 100755 --- a/vm/callstack.hpp +++ b/vm/callstack.hpp @@ -12,7 +12,7 @@ VM_ASM_API void save_callstack_bottom(stack_frame *callstack_bottom, factor_vm * keep the callstack in a GC root and use relative offsets */ template void factor_vm::iterate_callstack_object(callstack *stack_, Iterator &iterator) { - gc_root stack(stack_,this); + data_root stack(stack_,this); fixnum frame_offset = untag_fixnum(stack->length) - sizeof(stack_frame); while(frame_offset >= 0) diff --git a/vm/code_block.cpp b/vm/code_block.cpp index 1f77148b5c..dc6a488e26 100755 --- a/vm/code_block.cpp +++ b/vm/code_block.cpp @@ -127,9 +127,8 @@ void *factor_vm::get_rel_symbol(array *literals, cell index) } case ARRAY_TYPE: { - cell i; array *names = untag(symbol); - for(i = 0; i < array_capacity(names); i++) + for(cell i = 0; i < array_capacity(names); i++) { symbol_char *name = alien_offset(array_nth(names,i)); void *sym = ffi_dlsym(d,name); @@ -179,7 +178,7 @@ cell factor_vm::compute_relocation(relocation_entry rel, cell index, code_block case RT_UNTAGGED: return untag_fixnum(ARG); case RT_MEGAMORPHIC_CACHE_HITS: - return (cell)&megamorphic_cache_hits; + return (cell)&dispatch_stats.megamorphic_cache_hits; case RT_VM: return (cell)this + untag_fixnum(ARG); case RT_CARDS_OFFSET: @@ -286,7 +285,7 @@ struct literal_references_updater { if(parent->relocation_type_of(rel) == RT_IMMEDIATE) { cell offset = parent->relocation_offset_of(rel) + (cell)(compiled + 1); - array *literals = parent->untag(compiled->literals); + array *literals = untag(compiled->literals); fixnum absolute_value = array_nth(literals,index); parent->store_address_in_code_block(parent->relocation_class_of(rel),offset,absolute_value); } @@ -346,7 +345,7 @@ void factor_vm::update_word_references(code_block *compiled) are referenced after this is done. So instead of polluting the code heap with dead PICs that will be freed on the next GC, we add them to the free list immediately. */ - else if(compiled->type() == PIC_TYPE) + else if(compiled->pic_p()) code->code_heap_free(compiled); else { @@ -379,7 +378,7 @@ struct literal_and_word_references_updater { } }; -void factor_vm::update_code_block_for_full_gc(code_block *compiled) +void factor_vm::update_code_block_words_and_literals(code_block *compiled) { if(code->needs_fixup_p(compiled)) relocate_code_block(compiled); @@ -437,9 +436,9 @@ void factor_vm::fixup_labels(array *labels, code_block *compiled) } /* Might GC */ -code_block *factor_vm::allot_code_block(cell size, cell type) +code_block *factor_vm::allot_code_block(cell size, code_block_type type) { - heap_block *block = code->heap_allot(size + sizeof(code_block),type); + code_block *block = code->allocator->allot(size + sizeof(code_block)); /* If allocation failed, do a full GC and compact the code heap. A full GC that occurs as a result of the data heap filling up does not @@ -449,35 +448,31 @@ code_block *factor_vm::allot_code_block(cell size, cell type) if(block == NULL) { primitive_compact_gc(); - block = code->heap_allot(size + sizeof(code_block),type); + block = code->allocator->allot(size + sizeof(code_block)); /* Insufficient room even after code GC, give up */ if(block == NULL) { - cell used, total_free, max_free; - code->heap_usage(&used,&total_free,&max_free); - - print_string("Code heap stats:\n"); - print_string("Used: "); print_cell(used); nl(); - print_string("Total free space: "); print_cell(total_free); nl(); - print_string("Largest free block: "); print_cell(max_free); nl(); + std::cout << "Code heap used: " << code->allocator->occupied_space() << "\n"; + std::cout << "Code heap free: " << code->allocator->free_space() << "\n"; fatal_error("Out of memory in add-compiled-block",0); } } - return (code_block *)block; + block->set_type(type); + return block; } /* Might GC */ -code_block *factor_vm::add_code_block(cell type, cell code_, cell labels_, cell owner_, cell relocation_, cell literals_) +code_block *factor_vm::add_code_block(code_block_type type, cell code_, cell labels_, cell owner_, cell relocation_, cell literals_) { - gc_root code(code_,this); - gc_root labels(labels_,this); - gc_root owner(owner_,this); - gc_root relocation(relocation_,this); - gc_root literals(literals_,this); + data_root code(code_,this); + data_root labels(labels_,this); + data_root owner(owner_,this); + data_root relocation(relocation_,this); + data_root literals(literals_,this); - cell code_length = align8(array_capacity(code.untagged())); + cell code_length = array_capacity(code.untagged()); code_block *compiled = allot_code_block(code_length,type); compiled->owner = owner.value(); diff --git a/vm/code_block_visitor.hpp b/vm/code_block_visitor.hpp new file mode 100644 index 0000000000..33e889c940 --- /dev/null +++ b/vm/code_block_visitor.hpp @@ -0,0 +1,89 @@ +namespace factor +{ + +template struct call_frame_code_block_visitor { + factor_vm *parent; + Visitor visitor; + + explicit call_frame_code_block_visitor(factor_vm *parent_, Visitor visitor_) : + parent(parent_), visitor(visitor_) {} + + void operator()(stack_frame *frame) + { + cell offset = (cell)FRAME_RETURN_ADDRESS(frame,parent) - (cell)frame->xt; + + code_block *new_block = visitor(parent->frame_code(frame)); + frame->xt = new_block->xt(); + + FRAME_RETURN_ADDRESS(frame,parent) = (void *)((cell)frame->xt + offset); + } +}; + +template struct callback_code_block_visitor { + callback_heap *callbacks; + Visitor visitor; + + explicit callback_code_block_visitor(callback_heap *callbacks_, Visitor visitor_) : + callbacks(callbacks_), visitor(visitor_) {} + + void operator()(callback *stub) + { + stub->compiled = visitor(stub->compiled); + callbacks->update(stub); + } +}; + +template struct code_block_visitor { + factor_vm *parent; + Visitor visitor; + + explicit code_block_visitor(factor_vm *parent_, Visitor visitor_) : + parent(parent_), visitor(visitor_) {} + + void visit_object_code_block(object *obj) + { + switch(obj->h.hi_tag()) + { + case WORD_TYPE: + { + word *w = (word *)obj; + if(w->code) + w->code = visitor(w->code); + if(w->profiling) + w->code = visitor(w->profiling); + + parent->update_word_xt(w); + break; + } + case QUOTATION_TYPE: + { + quotation *q = (quotation *)obj; + if(q->code) + parent->set_quot_xt(q,visitor(q->code)); + break; + } + case CALLSTACK_TYPE: + { + callstack *stack = (callstack *)obj; + call_frame_code_block_visitor call_frame_visitor(parent,visitor); + parent->iterate_callstack_object(stack,call_frame_visitor); + break; + } + } + } + + void visit_context_code_blocks() + { + call_frame_code_block_visitor call_frame_visitor(parent,visitor); + parent->iterate_active_frames(call_frame_visitor); + } + + void visit_callback_code_blocks() + { + callback_code_block_visitor callback_visitor(parent->callbacks,visitor); + parent->callbacks->iterate(callback_visitor); + } + +}; + +} diff --git a/vm/code_heap.cpp b/vm/code_heap.cpp index 288c2221f2..b4e071d644 100755 --- a/vm/code_heap.cpp +++ b/vm/code_heap.cpp @@ -3,7 +3,21 @@ namespace factor { -code_heap::code_heap(bool secure_gc, cell size) : heap(secure_gc,size,true) {} +code_heap::code_heap(cell size) +{ + if(size > (1L << (sizeof(cell) * 8 - 6))) fatal_error("Heap too large",size); + seg = new segment(align_page(size),true); + if(!seg) fatal_error("Out of memory in heap allocator",size); + allocator = new free_list_allocator(size,seg->start); +} + +code_heap::~code_heap() +{ + delete allocator; + allocator = NULL; + delete seg; + seg = NULL; +} void code_heap::write_barrier(code_block *compiled) { @@ -22,18 +36,33 @@ bool code_heap::needs_fixup_p(code_block *compiled) return needs_fixup.count(compiled) > 0; } +bool code_heap::marked_p(code_block *compiled) +{ + return allocator->state.marked_p(compiled); +} + +void code_heap::set_marked_p(code_block *compiled) +{ + allocator->state.set_marked_p(compiled); +} + +void code_heap::clear_mark_bits() +{ + allocator->state.clear_mark_bits(); +} + void code_heap::code_heap_free(code_block *compiled) { points_to_nursery.erase(compiled); points_to_aging.erase(compiled); needs_fixup.erase(compiled); - heap_free(compiled); + allocator->free(compiled); } /* Allocate a code heap during startup */ void factor_vm::init_code_heap(cell size) { - code = new code_heap(secure_gc,size); + code = new code_heap(size); } bool factor_vm::in_code_heap_p(cell ptr) @@ -44,8 +73,8 @@ bool factor_vm::in_code_heap_p(cell ptr) /* Compile a word definition with the non-optimizing compiler. Allocates memory */ void factor_vm::jit_compile_word(cell word_, cell def_, bool relocate) { - gc_root word(word_,this); - gc_root def(def_,this); + data_root word(word_,this); + data_root def(def_,this); jit_compile(def.value(),relocate); @@ -59,7 +88,8 @@ struct word_updater { factor_vm *parent; explicit word_updater(factor_vm *parent_) : parent(parent_) {} - void operator()(code_block *compiled) + + void operator()(code_block *compiled, cell size) { parent->update_word_references(compiled); } @@ -73,9 +103,49 @@ void factor_vm::update_code_heap_words() iterate_code_heap(updater); } +/* After a full GC that did not grow the heap, we have to update references +to literals and other words. */ +struct word_and_literal_code_heap_updater { + factor_vm *parent; + + explicit word_and_literal_code_heap_updater(factor_vm *parent_) : parent(parent_) {} + + void operator()(code_block *block, cell size) + { + parent->update_code_block_words_and_literals(block); + } +}; + +void factor_vm::update_code_heap_words_and_literals() +{ + current_gc->event->started_code_sweep(); + word_and_literal_code_heap_updater updater(this); + code->allocator->sweep(updater); + current_gc->event->ended_code_sweep(); +} + +/* After growing the heap, we have to perform a full relocation to update +references to card and deck arrays. */ +struct code_heap_relocator { + factor_vm *parent; + + explicit code_heap_relocator(factor_vm *parent_) : parent(parent_) {} + + void operator()(code_block *block, cell size) + { + parent->relocate_code_block(block); + } +}; + +void factor_vm::relocate_code_heap() +{ + code_heap_relocator relocator(this); + code->allocator->sweep(relocator); +} + void factor_vm::primitive_modify_code_heap() { - gc_root alist(dpop(),this); + data_root alist(dpop(),this); cell count = array_capacity(alist.untagged()); @@ -85,10 +155,10 @@ void factor_vm::primitive_modify_code_heap() cell i; for(i = 0; i < count; i++) { - gc_root pair(array_nth(alist.untagged(),i),this); + data_root pair(array_nth(alist.untagged(),i),this); - gc_root word(array_nth(pair.untagged(),0),this); - gc_root data(array_nth(pair.untagged(),1),this); + data_root word(array_nth(pair.untagged(),0),this); + data_root data(array_nth(pair.untagged(),1),this); switch(data.type()) { @@ -105,7 +175,7 @@ void factor_vm::primitive_modify_code_heap() cell code = array_nth(compiled_data,4); code_block *compiled = add_code_block( - WORD_TYPE, + code_block_optimized, code, labels, owner, @@ -120,136 +190,35 @@ void factor_vm::primitive_modify_code_heap() break; } - update_word_xt(word.value()); + update_word_xt(word.untagged()); } update_code_heap_words(); } -/* Push the free space and total size of the code heap */ +code_heap_room factor_vm::code_room() +{ + code_heap_room room; + + room.size = code->allocator->size; + room.occupied_space = code->allocator->occupied_space(); + room.total_free = code->allocator->free_space(); + room.contiguous_free = code->allocator->largest_free_block(); + room.free_block_count = code->allocator->free_block_count(); + + return room; +} + void factor_vm::primitive_code_room() { - cell used, total_free, max_free; - code->heap_usage(&used,&total_free,&max_free); - dpush(tag_fixnum(code->seg->size / 1024)); - dpush(tag_fixnum(used / 1024)); - dpush(tag_fixnum(total_free / 1024)); - dpush(tag_fixnum(max_free / 1024)); -} - -code_block *code_heap::forward_code_block(code_block *compiled) -{ - return (code_block *)forwarding[compiled]; -} - -struct callframe_forwarder { - factor_vm *parent; - - explicit callframe_forwarder(factor_vm *parent_) : parent(parent_) {} - - void operator()(stack_frame *frame) - { - cell offset = (cell)FRAME_RETURN_ADDRESS(frame,parent) - (cell)frame->xt; - - code_block *forwarded = parent->code->forward_code_block(parent->frame_code(frame)); - frame->xt = forwarded->xt(); - - FRAME_RETURN_ADDRESS(frame,parent) = (void *)((cell)frame->xt + offset); - } -}; - -void factor_vm::forward_object_xts() -{ - begin_scan(); - - cell obj; - - while(to_boolean(obj = next_object())) - { - switch(tagged(obj).type()) - { - case WORD_TYPE: - { - word *w = untag(obj); - - if(w->code) - w->code = code->forward_code_block(w->code); - if(w->profiling) - w->profiling = code->forward_code_block(w->profiling); - - update_word_xt(obj); - } - break; - case QUOTATION_TYPE: - { - quotation *quot = untag(obj); - - if(quot->code) - { - quot->code = code->forward_code_block(quot->code); - set_quot_xt(quot,quot->code); - } - } - break; - case CALLSTACK_TYPE: - { - callstack *stack = untag(obj); - callframe_forwarder forwarder(this); - iterate_callstack_object(stack,forwarder); - } - break; - default: - break; - } - } - - end_scan(); -} - -void factor_vm::forward_context_xts() -{ - callframe_forwarder forwarder(this); - iterate_active_frames(forwarder); -} - -struct callback_forwarder { - code_heap *code; - callback_heap *callbacks; - - callback_forwarder(code_heap *code_, callback_heap *callbacks_) : - code(code_), callbacks(callbacks_) {} - - void operator()(callback *stub) - { - stub->compiled = code->forward_code_block(stub->compiled); - callbacks->update(stub); - } -}; - -void factor_vm::forward_callback_xts() -{ - callback_forwarder forwarder(code,callbacks); - callbacks->iterate(forwarder); -} - -/* Move all free space to the end of the code heap. Live blocks must be marked -on entry to this function. XTs in code blocks must be updated after this -function returns. */ -void factor_vm::compact_code_heap(bool trace_contexts_p) -{ - code->compact_heap(); - forward_object_xts(); - if(trace_contexts_p) - { - forward_context_xts(); - forward_callback_xts(); - } + code_heap_room room = code_room(); + dpush(tag(byte_array_from_value(&room))); } struct stack_trace_stripper { explicit stack_trace_stripper() {} - void operator()(code_block *compiled) + void operator()(code_block *compiled, cell size) { compiled->owner = false_object; } diff --git a/vm/code_heap.hpp b/vm/code_heap.hpp index 0a96a0b27b..8f4790d2f9 100755 --- a/vm/code_heap.hpp +++ b/vm/code_heap.hpp @@ -1,7 +1,13 @@ namespace factor { -struct code_heap : heap { +struct code_heap { + /* The actual memory area */ + segment *seg; + + /* Memory allocator */ + free_list_allocator *allocator; + /* Set of blocks which need full relocation. */ std::set needs_fixup; @@ -11,12 +17,23 @@ struct code_heap : heap { /* Code blocks which may reference objects in aging space or the nursery */ std::set points_to_aging; - explicit code_heap(bool secure_gc, cell size); + explicit code_heap(cell size); + ~code_heap(); void write_barrier(code_block *compiled); void clear_remembered_set(); bool needs_fixup_p(code_block *compiled); + bool marked_p(code_block *compiled); + void set_marked_p(code_block *compiled); + void clear_mark_bits(); void code_heap_free(code_block *compiled); - code_block *forward_code_block(code_block *compiled); +}; + +struct code_heap_room { + cell size; + cell occupied_space; + cell total_free; + cell contiguous_free; + cell free_block_count; }; } diff --git a/vm/code_roots.hpp b/vm/code_roots.hpp new file mode 100644 index 0000000000..64f2b0c0ed --- /dev/null +++ b/vm/code_roots.hpp @@ -0,0 +1,29 @@ +namespace factor +{ + +struct code_root { + cell value; + bool valid; + factor_vm *parent; + + void push() + { + parent->code_roots.push_back(this); + } + + explicit code_root(cell value_, factor_vm *parent_) : + value(value_), valid(true), parent(parent_) + { + push(); + } + + ~code_root() + { +#ifdef FACTOR_DEBUG + assert(parent->code_roots.back() == this); +#endif + parent->code_roots.pop_back(); + } +}; + +} diff --git a/vm/collector.hpp b/vm/collector.hpp index bbaad1d570..29711aeb9c 100644 --- a/vm/collector.hpp +++ b/vm/collector.hpp @@ -1,21 +1,13 @@ namespace factor { -template struct collector { +template struct collector_workhorse { factor_vm *parent; - data_heap *data; - code_heap *code; - gc_state *current_gc; - generation_statistics *stats; TargetGeneration *target; Policy policy; - explicit collector(factor_vm *parent_, generation_statistics *stats_, TargetGeneration *target_, Policy policy_) : + explicit collector_workhorse(factor_vm *parent_, TargetGeneration *target_, Policy policy_) : parent(parent_), - data(parent_->data), - code(parent_->code), - current_gc(parent_->current_gc), - stats(stats_), target(target_), policy(policy_) {} @@ -32,117 +24,239 @@ template struct collector { return untagged; } - void trace_handle(cell *handle) - { - cell pointer = *handle; - - if(immediate_p(pointer)) return; - - object *untagged = parent->untag(pointer); - if(!policy.should_copy_p(untagged)) - return; - - object *forwarding = resolve_forwarding(untagged); - - if(forwarding == untagged) - untagged = promote_object(untagged); - else if(policy.should_copy_p(forwarding)) - untagged = promote_object(forwarding); - else - untagged = forwarding; - - *handle = RETAG(untagged,TAG(pointer)); - } - - void trace_slots(object *ptr) - { - cell *slot = (cell *)ptr; - cell *end = (cell *)((cell)ptr + parent->binary_payload_start(ptr)); - - if(slot != end) - { - slot++; - for(; slot < end; slot++) trace_handle(slot); - } - } - object *promote_object(object *untagged) { - cell size = parent->untagged_object_size(untagged); + cell size = untagged->size(); object *newpointer = target->allot(size); /* XXX not exception-safe */ - if(!newpointer) longjmp(current_gc->gc_unwind,1); + if(!newpointer) longjmp(parent->current_gc->gc_unwind,1); memcpy(newpointer,untagged,size); untagged->h.forward_to(newpointer); - stats->object_count++; - stats->bytes_copied += size; + policy.promoted_object(newpointer); return newpointer; } - void trace_stack_elements(segment *region, cell *top) + object *operator()(object *obj) { - for(cell *ptr = (cell *)region->start; ptr <= top; ptr++) - trace_handle(ptr); - } - - void trace_registered_locals() - { - std::vector::const_iterator iter = parent->gc_locals.begin(); - std::vector::const_iterator end = parent->gc_locals.end(); - - for(; iter < end; iter++) - trace_handle((cell *)(*iter)); - } - - void trace_registered_bignums() - { - std::vector::const_iterator iter = parent->gc_bignums.begin(); - std::vector::const_iterator end = parent->gc_bignums.end(); - - for(; iter < end; iter++) + if(!policy.should_copy_p(obj)) { - cell *handle = (cell *)(*iter); + policy.visited_object(obj); + return obj; + } - if(*handle) - { - *handle |= BIGNUM_TYPE; - trace_handle(handle); - *handle &= ~BIGNUM_TYPE; - } + object *forwarding = resolve_forwarding(obj); + + if(forwarding == obj) + return promote_object(obj); + else if(policy.should_copy_p(forwarding)) + return promote_object(forwarding); + else + { + policy.visited_object(forwarding); + return forwarding; } } +}; + +template +inline static slot_visitor > make_collector_workhorse( + factor_vm *parent, + TargetGeneration *target, + Policy policy) +{ + return slot_visitor >(parent, + collector_workhorse(parent,target,policy)); +} + +struct dummy_unmarker { + void operator()(card *ptr) {} +}; + +struct simple_unmarker { + card unmask; + explicit simple_unmarker(card unmask_) : unmask(unmask_) {} + void operator()(card *ptr) { *ptr &= ~unmask; } +}; + +struct full_unmarker { + explicit full_unmarker() {} + void operator()(card *ptr) { *ptr = 0; } +}; + +template struct collector { + factor_vm *parent; + data_heap *data; + code_heap *code; + TargetGeneration *target; + slot_visitor > workhorse; + cell cards_scanned; + cell decks_scanned; + cell code_blocks_scanned; + + explicit collector(factor_vm *parent_, TargetGeneration *target_, Policy policy_) : + parent(parent_), + data(parent_->data), + code(parent_->code), + target(target_), + workhorse(make_collector_workhorse(parent_,target_,policy_)), + cards_scanned(0), + decks_scanned(0), + code_blocks_scanned(0) {} + + void trace_handle(cell *handle) + { + workhorse.visit_handle(handle); + } + + void trace_object(object *ptr) + { + workhorse.visit_slots(ptr); + if(ptr->h.hi_tag() == ALIEN_TYPE) + ((alien *)ptr)->update_address(); + } - /* Copy roots over at the start of GC, namely various constants, stacks, - the user environment and extra roots registered by local_roots.hpp */ void trace_roots() { - trace_handle(&parent->true_object); - trace_handle(&parent->bignum_zero); - trace_handle(&parent->bignum_pos_one); - trace_handle(&parent->bignum_neg_one); - - trace_registered_locals(); - trace_registered_bignums(); - - for(int i = 0; i < USER_ENV; i++) trace_handle(&parent->userenv[i]); + workhorse.visit_roots(); } void trace_contexts() { - context *ctx = parent->ctx; + workhorse.visit_contexts(); + } - while(ctx) + /* Trace all literals referenced from a code block. Only for aging and nursery collections */ + void trace_literal_references(code_block *compiled) + { + workhorse.visit_literal_references(compiled); + } + + void trace_code_heap_roots(std::set *remembered_set) + { + std::set::const_iterator iter = remembered_set->begin(); + std::set::const_iterator end = remembered_set->end(); + + for(; iter != end; iter++) { - trace_stack_elements(ctx->datastack_region,(cell *)ctx->datastack); - trace_stack_elements(ctx->retainstack_region,(cell *)ctx->retainstack); + trace_literal_references(*iter); + code_blocks_scanned++; + } + } - trace_handle(&ctx->catchstack_save); - trace_handle(&ctx->current_callback_save); + inline cell first_card_in_deck(cell deck) + { + return deck << (deck_bits - card_bits); + } - ctx = ctx->next; + inline cell last_card_in_deck(cell deck) + { + return first_card_in_deck(deck + 1); + } + + inline cell card_deck_for_address(cell a) + { + return addr_to_deck(a - data->start); + } + + inline cell card_start_address(cell card) + { + return (card << card_bits) + data->start; + } + + inline cell card_end_address(cell card) + { + return ((card + 1) << card_bits) + data->start; + } + + void trace_partial_objects(cell start, cell end, cell card_start, cell card_end) + { + if(card_start < end) + { + start += sizeof(cell); + + if(start < card_start) start = card_start; + if(end > card_end) end = card_end; + + cell *slot_ptr = (cell *)start; + cell *end_ptr = (cell *)end; + + if(slot_ptr != end_ptr) + { + for(; slot_ptr < end_ptr; slot_ptr++) + workhorse.visit_handle(slot_ptr); + } + } + } + + template + void trace_cards(SourceGeneration *gen, card mask, Unmarker unmarker) + { + card_deck *decks = data->decks; + card_deck *cards = data->cards; + + cell gen_start_card = addr_to_card(gen->start - data->start); + + cell first_deck = card_deck_for_address(gen->start); + cell last_deck = card_deck_for_address(gen->end); + + cell start = 0, binary_start = 0, end = 0; + + for(cell deck_index = first_deck; deck_index < last_deck; deck_index++) + { + if(decks[deck_index] & mask) + { + decks_scanned++; + + cell first_card = first_card_in_deck(deck_index); + cell last_card = last_card_in_deck(deck_index); + + for(cell card_index = first_card; card_index < last_card; card_index++) + { + if(cards[card_index] & mask) + { + cards_scanned++; + + if(end < card_start_address(card_index)) + { + start = gen->starts.find_object_containing_card(card_index - gen_start_card); + binary_start = start + ((object *)start)->binary_payload_start(); + end = start + ((object *)start)->size(); + } + +#ifdef FACTOR_DEBUG + assert(addr_to_card(start - data->start) <= card_index); + assert(start < card_end_address(card_index)); +#endif + +scan_next_object: { + trace_partial_objects( + start, + binary_start, + card_start_address(card_index), + card_end_address(card_index)); + if(end < card_end_address(card_index)) + { + start = gen->next_object_after(start); + if(start) + { + binary_start = start + ((object *)start)->binary_payload_start(); + end = start + ((object *)start)->size(); + goto scan_next_object; + } + } + } + + unmarker(&cards[card_index]); + + if(!start) return; + } + } + + unmarker(&decks[deck_index]); + } } } }; diff --git a/vm/compaction.cpp b/vm/compaction.cpp new file mode 100644 index 0000000000..10e37db263 --- /dev/null +++ b/vm/compaction.cpp @@ -0,0 +1,191 @@ +#include "master.hpp" + +namespace factor { + +template struct forwarder { + mark_bits *forwarding_map; + + explicit forwarder(mark_bits *forwarding_map_) : + forwarding_map(forwarding_map_) {} + + Block *operator()(Block *block) + { + return forwarding_map->forward_block(block); + } +}; + +static inline cell tuple_size_with_forwarding(mark_bits *forwarding_map, object *obj) +{ + /* The tuple layout may or may not have been forwarded already. Tricky. */ + object *layout_obj = (object *)UNTAG(((tuple *)obj)->layout); + tuple_layout *layout; + + if(layout_obj < obj) + { + /* It's already been moved up; dereference through forwarding + map to get the size */ + layout = (tuple_layout *)forwarding_map->forward_block(layout_obj); + } + else + { + /* It hasn't been moved up yet; dereference directly */ + layout = (tuple_layout *)layout_obj; + } + + return tuple_size(layout); +} + +struct compaction_sizer { + mark_bits *forwarding_map; + + explicit compaction_sizer(mark_bits *forwarding_map_) : + forwarding_map(forwarding_map_) {} + + cell operator()(object *obj) + { + if(!forwarding_map->marked_p(obj)) + return forwarding_map->unmarked_block_size(obj); + else if(obj->h.hi_tag() == TUPLE_TYPE) + return align(tuple_size_with_forwarding(forwarding_map,obj),data_alignment); + else + return obj->size(); + } +}; + +struct object_compaction_updater { + factor_vm *parent; + slot_visitor > slot_forwarder; + code_block_visitor > code_forwarder; + mark_bits *data_forwarding_map; + object_start_map *starts; + + explicit object_compaction_updater(factor_vm *parent_, + slot_visitor > slot_forwarder_, + code_block_visitor > code_forwarder_, + mark_bits *data_forwarding_map_) : + parent(parent_), + slot_forwarder(slot_forwarder_), + code_forwarder(code_forwarder_), + data_forwarding_map(data_forwarding_map_), + starts(&parent->data->tenured->starts) {} + + void operator()(object *old_address, object *new_address, cell size) + { + cell payload_start; + if(old_address->h.hi_tag() == TUPLE_TYPE) + payload_start = tuple_size_with_forwarding(data_forwarding_map,old_address); + else + payload_start = old_address->binary_payload_start(); + + memmove(new_address,old_address,size); + + slot_forwarder.visit_slots(new_address,payload_start); + code_forwarder.visit_object_code_block(new_address); + starts->record_object_start_offset(new_address); + } +}; + +template struct code_block_compaction_updater { + factor_vm *parent; + SlotForwarder slot_forwarder; + + explicit code_block_compaction_updater(factor_vm *parent_, SlotForwarder slot_forwarder_) : + parent(parent_), slot_forwarder(slot_forwarder_) {} + + void operator()(code_block *old_address, code_block *new_address, cell size) + { + memmove(new_address,old_address,size); + slot_forwarder.visit_literal_references(new_address); + parent->relocate_code_block(new_address); + } +}; + +/* Compact data and code heaps */ +void factor_vm::collect_compact_impl(bool trace_contexts_p) +{ + current_gc->event->started_compaction(); + + tenured_space *tenured = data->tenured; + mark_bits *data_forwarding_map = &tenured->state; + mark_bits *code_forwarding_map = &code->allocator->state; + + /* Figure out where blocks are going to go */ + data_forwarding_map->compute_forwarding(); + code_forwarding_map->compute_forwarding(); + + slot_visitor > slot_forwarder(this,forwarder(data_forwarding_map)); + code_block_visitor > code_forwarder(this,forwarder(code_forwarding_map)); + + /* Object start offsets get recomputed by the object_compaction_updater */ + data->tenured->starts.clear_object_start_offsets(); + + /* Slide everything in tenured space up, and update data and code heap + pointers inside objects. */ + object_compaction_updater object_updater(this,slot_forwarder,code_forwarder,data_forwarding_map); + compaction_sizer object_sizer(data_forwarding_map); + tenured->compact(object_updater,object_sizer); + + /* Slide everything in the code heap up, and update data and code heap + pointers inside code blocks. */ + code_block_compaction_updater > > code_block_updater(this,slot_forwarder); + standard_sizer code_block_sizer; + code->allocator->compact(code_block_updater,code_block_sizer); + + slot_forwarder.visit_roots(); + if(trace_contexts_p) + { + slot_forwarder.visit_contexts(); + code_forwarder.visit_context_code_blocks(); + code_forwarder.visit_callback_code_blocks(); + } + + update_code_roots_for_compaction(); + + current_gc->event->ended_compaction(); +} + +struct object_code_block_updater { + code_block_visitor > *visitor; + + explicit object_code_block_updater(code_block_visitor > *visitor_) : + visitor(visitor_) {} + + void operator()(cell obj) + { + visitor->visit_object_code_block(tagged(obj).untagged()); + } +}; + +struct dummy_slot_forwarder { + void visit_literal_references(code_block *compiled) {} +}; + +/* Compact just the code heap */ +void factor_vm::collect_compact_code_impl(bool trace_contexts_p) +{ + /* Figure out where blocks are going to go */ + mark_bits *code_forwarding_map = &code->allocator->state; + code_forwarding_map->compute_forwarding(); + code_block_visitor > code_forwarder(this,forwarder(code_forwarding_map)); + + if(trace_contexts_p) + { + code_forwarder.visit_context_code_blocks(); + code_forwarder.visit_callback_code_blocks(); + } + + /* Update code heap references in data heap */ + object_code_block_updater updater(&code_forwarder); + each_object(updater); + + /* Slide everything in the code heap up, and update code heap + pointers inside code blocks. */ + dummy_slot_forwarder slot_forwarder; + code_block_compaction_updater code_block_updater(this,slot_forwarder); + standard_sizer code_block_sizer; + code->allocator->compact(code_block_updater,code_block_sizer); + + update_code_roots_for_compaction(); +} + +} diff --git a/vm/compaction.hpp b/vm/compaction.hpp new file mode 100644 index 0000000000..412ef35bb4 --- /dev/null +++ b/vm/compaction.hpp @@ -0,0 +1,4 @@ +namespace factor +{ + +} diff --git a/vm/contexts.cpp b/vm/contexts.cpp index cc7029e7f1..7af7fdaa57 100644 --- a/vm/contexts.cpp +++ b/vm/contexts.cpp @@ -80,9 +80,9 @@ void factor_vm::nest_stacks(stack_frame *magic_frame) new_ctx->magic_frame = magic_frame; - /* save per-callback userenv */ - new_ctx->current_callback_save = userenv[CURRENT_CALLBACK_ENV]; - new_ctx->catchstack_save = userenv[CATCHSTACK_ENV]; + /* save per-callback special_objects */ + new_ctx->current_callback_save = special_objects[OBJ_CURRENT_CALLBACK]; + new_ctx->catchstack_save = special_objects[OBJ_CATCHSTACK]; new_ctx->next = ctx; ctx = new_ctx; @@ -102,9 +102,9 @@ void factor_vm::unnest_stacks() ds = ctx->datastack_save; rs = ctx->retainstack_save; - /* restore per-callback userenv */ - userenv[CURRENT_CALLBACK_ENV] = ctx->current_callback_save; - userenv[CATCHSTACK_ENV] = ctx->catchstack_save; + /* restore per-callback special_objects */ + special_objects[OBJ_CURRENT_CALLBACK] = ctx->current_callback_save; + special_objects[OBJ_CATCHSTACK] = ctx->catchstack_save; context *old_ctx = ctx; ctx = old_ctx->next; @@ -133,7 +133,7 @@ bool factor_vm::stack_to_array(cell bottom, cell top) return false; else { - array *a = allot_array_internal(depth / sizeof(cell)); + array *a = allot_uninitialized_array(depth / sizeof(cell)); memcpy(a + 1,(void*)bottom,depth); dpush(tag(a)); return true; diff --git a/vm/contexts.hpp b/vm/contexts.hpp index f66b5d0fe2..aa6f9ec8ce 100644 --- a/vm/contexts.hpp +++ b/vm/contexts.hpp @@ -41,7 +41,7 @@ struct context { /* memory region holding current retain stack */ segment *retainstack_region; - /* saved userenv slots on entry to callback */ + /* saved special_objects slots on entry to callback */ cell catchstack_save; cell current_callback_save; diff --git a/vm/copying_collector.hpp b/vm/copying_collector.hpp index 640d355bf4..89501a3a4a 100644 --- a/vm/copying_collector.hpp +++ b/vm/copying_collector.hpp @@ -1,164 +1,19 @@ namespace factor { -struct dummy_unmarker { - void operator()(card *ptr) {} -}; - -struct simple_unmarker { - card unmask; - simple_unmarker(card unmask_) : unmask(unmask_) {} - void operator()(card *ptr) { *ptr &= ~unmask; } -}; - template struct copying_collector : collector { cell scan; - explicit copying_collector(factor_vm *parent_, generation_statistics *stats_, TargetGeneration *target_, Policy policy_) : - collector(parent_,stats_,target_,policy_), scan(target_->here) {} - - inline cell first_card_in_deck(cell deck) - { - return deck << (deck_bits - card_bits); - } - - inline cell last_card_in_deck(cell deck) - { - return first_card_in_deck(deck + 1); - } - - inline cell card_deck_for_address(cell a) - { - return addr_to_deck(a - this->data->start); - } - - inline cell card_start_address(cell card) - { - return (card << card_bits) + this->data->start; - } - - inline cell card_end_address(cell card) - { - return ((card + 1) << card_bits) + this->data->start; - } - - void trace_partial_objects(cell start, cell end, cell card_start, cell card_end) - { - if(card_start < end) - { - start += sizeof(cell); - - if(start < card_start) start = card_start; - if(end > card_end) end = card_end; - - cell *slot_ptr = (cell *)start; - cell *end_ptr = (cell *)end; - - if(slot_ptr != end_ptr) - { - for(; slot_ptr < end_ptr; slot_ptr++) - this->trace_handle(slot_ptr); - } - } - } - - template - void trace_cards(SourceGeneration *gen, card mask, Unmarker unmarker) - { - u64 start_time = current_micros(); - - card_deck *decks = this->data->decks; - card_deck *cards = this->data->cards; - - cell gen_start_card = addr_to_card(gen->start - this->data->start); - - cell first_deck = card_deck_for_address(gen->start); - cell last_deck = card_deck_for_address(gen->end); - - cell start = 0, binary_start = 0, end = 0; - - for(cell deck_index = first_deck; deck_index < last_deck; deck_index++) - { - if(decks[deck_index] & mask) - { - this->parent->gc_stats.decks_scanned++; - - cell first_card = first_card_in_deck(deck_index); - cell last_card = last_card_in_deck(deck_index); - - for(cell card_index = first_card; card_index < last_card; card_index++) - { - if(cards[card_index] & mask) - { - this->parent->gc_stats.cards_scanned++; - - if(end < card_start_address(card_index)) - { - start = gen->find_object_containing_card(card_index - gen_start_card); - binary_start = start + this->parent->binary_payload_start((object *)start); - end = start + this->parent->untagged_object_size((object *)start); - } - -#ifdef FACTOR_DEBUG - assert(addr_to_card(start - this->data->start) <= card_index); - assert(start < card_end_address(card_index)); -#endif - -scan_next_object: { - trace_partial_objects( - start, - binary_start, - card_start_address(card_index), - card_end_address(card_index)); - if(end < card_end_address(card_index)) - { - start = gen->next_object_after(this->parent,start); - if(start) - { - binary_start = start + this->parent->binary_payload_start((object *)start); - end = start + this->parent->untagged_object_size((object *)start); - goto scan_next_object; - } - } - } - - unmarker(&cards[card_index]); - - if(!start) goto end; - } - } - - unmarker(&decks[deck_index]); - } - } - -end: this->parent->gc_stats.card_scan_time += (current_micros() - start_time); - } - - /* Trace all literals referenced from a code block. Only for aging and nursery collections */ - void trace_literal_references(code_block *compiled) - { - this->trace_handle(&compiled->owner); - this->trace_handle(&compiled->literals); - this->trace_handle(&compiled->relocation); - this->parent->gc_stats.code_blocks_scanned++; - } - - void trace_code_heap_roots(std::set *remembered_set) - { - std::set::const_iterator iter = remembered_set->begin(); - std::set::const_iterator end = remembered_set->end(); - - for(; iter != end; iter++) trace_literal_references(*iter); - } + explicit copying_collector(factor_vm *parent_, TargetGeneration *target_, Policy policy_) : + collector(parent_,target_,policy_), scan(target_->here) {} void cheneys_algorithm() { while(scan && scan < this->target->here) { - this->trace_slots((object *)scan); - scan = this->target->next_object_after(this->parent,scan); + this->trace_object((object *)scan); + scan = this->target->next_object_after(scan); } } }; diff --git a/vm/cpu-ppc.S b/vm/cpu-ppc.S index 61b05a1735..40f2521e50 100644 --- a/vm/cpu-ppc.S +++ b/vm/cpu-ppc.S @@ -37,13 +37,13 @@ DEF(void,primitive_fixnum_multiply,(void *vm)): lwz r3,0(DS_REG) lwz r4,-4(DS_REG) subi DS_REG,DS_REG,4 - srawi r3,r3,3 + srawi r3,r3,4 mullwo. r6,r3,r4 bso multiply_overflow stw r6,0(DS_REG) blr multiply_overflow: - srawi r4,r4,3 + srawi r4,r4,4 b MANGLE(overflow_fixnum_multiply) /* Note that the XT is passed to the quotation in r11 */ diff --git a/vm/cpu-x86.32.S b/vm/cpu-x86.32.S index 2e85be0f81..c0532f0ece 100644 --- a/vm/cpu-x86.32.S +++ b/vm/cpu-x86.32.S @@ -19,11 +19,9 @@ #define PUSH_NONVOLATILE \ push %ebx ; \ - push %ebp ; \ push %ebp #define POP_NONVOLATILE \ - pop %ebp ; \ pop %ebp ; \ pop %ebx diff --git a/vm/cpu-x86.64.S b/vm/cpu-x86.64.S index 5e307f0500..8ccd703bfe 100644 --- a/vm/cpu-x86.64.S +++ b/vm/cpu-x86.64.S @@ -27,11 +27,9 @@ push %rdi ; \ push %rsi ; \ push %rbx ; \ - push %rbp ; \ push %rbp #define POP_NONVOLATILE \ - pop %rbp ; \ pop %rbp ; \ pop %rbx ; \ pop %rsi ; \ @@ -50,11 +48,9 @@ push %rbx ; \ push %rbp ; \ push %r12 ; \ - push %r13 ; \ push %r13 #define POP_NONVOLATILE \ - pop %r13 ; \ pop %r13 ; \ pop %r12 ; \ pop %rbp ; \ diff --git a/vm/cpu-x86.S b/vm/cpu-x86.S index c497a0aad2..411a0cdaa6 100644 --- a/vm/cpu-x86.S +++ b/vm/cpu-x86.S @@ -25,7 +25,7 @@ DEF(void,primitive_fixnum_multiply,(void *myvm)): mov (DS_REG),ARITH_TEMP_1 mov ARITH_TEMP_1,DIV_RESULT mov -CELL_SIZE(DS_REG),ARITH_TEMP_2 - sar $3,ARITH_TEMP_2 + sar $4,ARITH_TEMP_2 sub $CELL_SIZE,DS_REG imul ARITH_TEMP_2 jo multiply_overflow @@ -33,7 +33,7 @@ DEF(void,primitive_fixnum_multiply,(void *myvm)): pop ARG2 ret multiply_overflow: - sar $3,ARITH_TEMP_1 + sar $4,ARITH_TEMP_1 mov ARITH_TEMP_1,ARG0 mov ARITH_TEMP_2,ARG1 pop ARG2 @@ -43,14 +43,20 @@ DEF(F_FASTCALL void,c_to_factor,(CELL quot, void *vm)): PUSH_NONVOLATILE mov ARG0,NV0 mov ARG1,NV1 - + + /* Save old stack pointer and align */ + mov STACK_REG,ARG0 + and $-16,STACK_REG + add $CELL_SIZE,STACK_REG + push ARG0 + /* Create register shadow area for Win64 */ sub $32,STACK_REG - + /* Save stack pointer */ lea -CELL_SIZE(STACK_REG),ARG0 call MANGLE(save_callstack_bottom) - + /* Call quot-xt */ mov NV0,ARG0 mov NV1,ARG1 @@ -59,6 +65,9 @@ DEF(F_FASTCALL void,c_to_factor,(CELL quot, void *vm)): /* Tear down register shadow area */ add $32,STACK_REG + /* Undo stack alignment */ + mov (STACK_REG),STACK_REG + POP_NONVOLATILE ret diff --git a/vm/data_heap.cpp b/vm/data_heap.cpp index 335938acab..bb705e276c 100755 --- a/vm/data_heap.cpp +++ b/vm/data_heap.cpp @@ -9,7 +9,9 @@ void factor_vm::init_card_decks() decks_offset = (cell)data->decks - addr_to_deck(data->start); } -data_heap::data_heap(cell young_size_, cell aging_size_, cell tenured_size_) +data_heap::data_heap(cell young_size_, + cell aging_size_, + cell tenured_size_) { young_size_ = align(young_size_,deck_size); aging_size_ = align(aging_size_,deck_size); @@ -19,30 +21,27 @@ data_heap::data_heap(cell young_size_, cell aging_size_, cell tenured_size_) aging_size = aging_size_; tenured_size = tenured_size_; - cell total_size = young_size + 2 * aging_size + 2 * tenured_size; - - total_size += deck_size; - + cell total_size = young_size + 2 * aging_size + tenured_size + deck_size; seg = new segment(total_size,false); cell cards_size = addr_to_card(total_size); - cards = new card[cards_size]; cards_end = cards + cards_size; + memset(cards,0,cards_size); cell decks_size = addr_to_deck(total_size); decks = new card_deck[decks_size]; decks_end = decks + decks_size; + memset(decks,0,decks_size); start = align(seg->start,deck_size); tenured = new tenured_space(tenured_size,start); - tenured_semispace = new tenured_space(tenured_size,tenured->end); - aging = new aging_space(aging_size,tenured_semispace->end); + aging = new aging_space(aging_size,tenured->end); aging_semispace = new aging_space(aging_size,aging->end); - nursery = new zone(young_size,aging_semispace->end); + nursery = new nursery_space(young_size,aging_semispace->end); assert(seg->end - nursery->end <= deck_size); } @@ -54,7 +53,6 @@ data_heap::~data_heap() delete aging; delete aging_semispace; delete tenured; - delete tenured_semispace; delete[] cards; delete[] decks; } @@ -62,49 +60,59 @@ data_heap::~data_heap() data_heap *data_heap::grow(cell requested_bytes) { cell new_tenured_size = (tenured_size * 2) + requested_bytes; - return new data_heap(young_size,aging_size,new_tenured_size); + return new data_heap(young_size, + aging_size, + new_tenured_size); } -void factor_vm::clear_cards(old_space *gen) +template void data_heap::clear_cards(Generation *gen) { - cell first_card = addr_to_card(gen->start - data->start); - cell last_card = addr_to_card(gen->end - data->start); - memset(&data->cards[first_card],0,last_card - first_card); + cell first_card = addr_to_card(gen->start - start); + cell last_card = addr_to_card(gen->end - start); + memset(&cards[first_card],0,last_card - first_card); } -void factor_vm::clear_decks(old_space *gen) +template void data_heap::clear_decks(Generation *gen) { - cell first_deck = addr_to_deck(gen->start - data->start); - cell last_deck = addr_to_deck(gen->end - data->start); - memset(&data->decks[first_deck],0,last_deck - first_deck); + cell first_deck = addr_to_deck(gen->start - start); + cell last_deck = addr_to_deck(gen->end - start); + memset(&decks[first_deck],0,last_deck - first_deck); } -/* After garbage collection, any generations which are now empty need to have -their allocation pointers and cards reset. */ -void factor_vm::reset_generation(old_space *gen) +void data_heap::reset_generation(nursery_space *gen) { gen->here = gen->start; - if(secure_gc) memset((void*)gen->start,69,gen->size); +} +void data_heap::reset_generation(aging_space *gen) +{ + gen->here = gen->start; clear_cards(gen); clear_decks(gen); - gen->clear_object_start_offsets(); + gen->starts.clear_object_start_offsets(); +} + +void data_heap::reset_generation(tenured_space *gen) +{ + clear_cards(gen); + clear_decks(gen); +} + +bool data_heap::low_memory_p() +{ + return (tenured->free_space() <= nursery->size + aging->size); } void factor_vm::set_data_heap(data_heap *data_) { data = data_; nursery = *data->nursery; - nursery.here = nursery.start; init_card_decks(); - reset_generation(data->aging); - reset_generation(data->tenured); } -void factor_vm::init_data_heap(cell young_size, cell aging_size, cell tenured_size, bool secure_gc_) +void factor_vm::init_data_heap(cell young_size, cell aging_size, cell tenured_size) { set_data_heap(new data_heap(young_size,aging_size,tenured_size)); - secure_gc = secure_gc_; } /* Size of the object pointed to by a tagged pointer */ @@ -113,61 +121,55 @@ cell factor_vm::object_size(cell tagged) if(immediate_p(tagged)) return 0; else - return untagged_object_size(untag(tagged)); + return untag(tagged)->size(); } /* Size of the object pointed to by an untagged pointer */ -cell factor_vm::untagged_object_size(object *pointer) +cell object::size() const { - return align8(unaligned_object_size(pointer)); -} + if(free_p()) return ((free_heap_block *)this)->size(); -/* Size of the data area of an object pointed to by an untagged pointer */ -cell factor_vm::unaligned_object_size(object *pointer) -{ - switch(pointer->h.hi_tag()) + switch(h.hi_tag()) { case ARRAY_TYPE: - return array_size((array*)pointer); + return align(array_size((array*)this),data_alignment); case BIGNUM_TYPE: - return array_size((bignum*)pointer); + return align(array_size((bignum*)this),data_alignment); case BYTE_ARRAY_TYPE: - return array_size((byte_array*)pointer); + return align(array_size((byte_array*)this),data_alignment); case STRING_TYPE: - return string_size(string_capacity((string*)pointer)); + return align(string_size(string_capacity((string*)this)),data_alignment); case TUPLE_TYPE: - return tuple_size(untag(((tuple *)pointer)->layout)); + { + tuple_layout *layout = (tuple_layout *)UNTAG(((tuple *)this)->layout); + return align(tuple_size(layout),data_alignment); + } case QUOTATION_TYPE: - return sizeof(quotation); + return align(sizeof(quotation),data_alignment); case WORD_TYPE: - return sizeof(word); + return align(sizeof(word),data_alignment); case FLOAT_TYPE: - return sizeof(boxed_float); + return align(sizeof(boxed_float),data_alignment); case DLL_TYPE: - return sizeof(dll); + return align(sizeof(dll),data_alignment); case ALIEN_TYPE: - return sizeof(alien); + return align(sizeof(alien),data_alignment); case WRAPPER_TYPE: - return sizeof(wrapper); + return align(sizeof(wrapper),data_alignment); case CALLSTACK_TYPE: - return callstack_size(untag_fixnum(((callstack *)pointer)->length)); + return align(callstack_size(untag_fixnum(((callstack *)this)->length)),data_alignment); default: - critical_error("Invalid header",(cell)pointer); + critical_error("Invalid header",(cell)this); return 0; /* can't happen */ } } -void factor_vm::primitive_size() -{ - box_unsigned_cell(object_size(dpop())); -} - /* The number of cells from the start of the object which should be scanned by the GC. Some types have a binary payload at the end (string, word, DLL) which we ignore. */ -cell factor_vm::binary_payload_start(object *pointer) +cell object::binary_payload_start() const { - switch(pointer->h.hi_tag()) + switch(h.hi_tag()) { /* these objects do not refer to other objects at all */ case FLOAT_TYPE: @@ -188,42 +190,54 @@ cell factor_vm::binary_payload_start(object *pointer) return sizeof(string); /* everything else consists entirely of pointers */ case ARRAY_TYPE: - return array_size(array_capacity((array*)pointer)); + return array_size(array_capacity((array*)this)); case TUPLE_TYPE: - return tuple_size(untag(((tuple *)pointer)->layout)); + return tuple_size(untag(((tuple *)this)->layout)); case WRAPPER_TYPE: return sizeof(wrapper); default: - critical_error("Invalid header",(cell)pointer); + critical_error("Invalid header",(cell)this); return 0; /* can't happen */ } } -/* Push memory usage statistics in data heap */ +void factor_vm::primitive_size() +{ + box_unsigned_cell(object_size(dpop())); +} + +data_heap_room factor_vm::data_room() +{ + data_heap_room room; + + room.nursery_size = nursery.size; + room.nursery_occupied = nursery.occupied_space(); + room.nursery_free = nursery.free_space(); + room.aging_size = data->aging->size; + room.aging_occupied = data->aging->occupied_space(); + room.aging_free = data->aging->free_space(); + room.tenured_size = data->tenured->size; + room.tenured_occupied = data->tenured->occupied_space(); + room.tenured_total_free = data->tenured->free_space(); + room.tenured_contiguous_free = data->tenured->largest_free_block(); + room.tenured_free_block_count = data->tenured->free_block_count(); + room.cards = data->cards_end - data->cards; + room.decks = data->decks_end - data->decks; + room.mark_stack = data->tenured->mark_stack.capacity(); + + return room; +} + void factor_vm::primitive_data_room() { - dpush(tag_fixnum((data->cards_end - data->cards) >> 10)); - dpush(tag_fixnum((data->decks_end - data->decks) >> 10)); - - growable_array a(this); - - a.add(tag_fixnum((nursery.end - nursery.here) >> 10)); - a.add(tag_fixnum((nursery.size) >> 10)); - - a.add(tag_fixnum((data->aging->end - data->aging->here) >> 10)); - a.add(tag_fixnum((data->aging->size) >> 10)); - - a.add(tag_fixnum((data->tenured->end - data->tenured->here) >> 10)); - a.add(tag_fixnum((data->tenured->size) >> 10)); - - a.trim(); - dpush(a.elements.value()); + data_heap_room room = data_room(); + dpush(tag(byte_array_from_value(&room))); } /* Disables GC and activates next-object ( -- obj ) primitive */ void factor_vm::begin_scan() { - heap_scan_ptr = data->tenured->start; + heap_scan_ptr = data->tenured->first_object(); gc_off = true; } @@ -242,12 +256,14 @@ cell factor_vm::next_object() if(!gc_off) general_error(ERROR_HEAP_SCAN,false_object,false_object,NULL); - if(heap_scan_ptr >= data->tenured->here) + if(heap_scan_ptr) + { + cell current = heap_scan_ptr; + heap_scan_ptr = data->tenured->next_object_after(heap_scan_ptr); + return tag_dynamic((object *)current); + } + else return false_object; - - object *obj = (object *)heap_scan_ptr; - heap_scan_ptr += untagged_object_size(obj); - return tag_dynamic(obj); } /* Push object at heap scan cursor and advance; pushes f when done */ @@ -262,25 +278,28 @@ void factor_vm::primitive_end_scan() gc_off = false; } -template void factor_vm::each_object(Iterator &iterator) -{ - begin_scan(); - cell obj; - while(to_boolean(obj = next_object())) - iterator(tagged(obj)); - end_scan(); -} - struct word_counter { cell count; + explicit word_counter() : count(0) {} - void operator()(tagged obj) { if(obj.type_p(WORD_TYPE)) count++; } + + void operator()(cell obj) + { + if(tagged(obj).type_p(WORD_TYPE)) + count++; + } }; struct word_accumulator { growable_array words; + explicit word_accumulator(int count,factor_vm *vm) : words(vm,count) {} - void operator()(tagged obj) { if(obj.type_p(WORD_TYPE)) words.add(obj.value()); } + + void operator()(cell obj) + { + if(tagged(obj).type_p(WORD_TYPE)) + words.add(obj); + } }; cell factor_vm::find_all_words() diff --git a/vm/data_heap.hpp b/vm/data_heap.hpp index 10f3698e74..760a10942e 100755 --- a/vm/data_heap.hpp +++ b/vm/data_heap.hpp @@ -10,11 +10,10 @@ struct data_heap { segment *seg; - zone *nursery; + nursery_space *nursery; aging_space *aging; aging_space *aging_semispace; tenured_space *tenured; - tenured_space *tenured_semispace; card *cards; card *cards_end; @@ -25,6 +24,29 @@ struct data_heap { explicit data_heap(cell young_size, cell aging_size, cell tenured_size); ~data_heap(); data_heap *grow(cell requested_size); + template void clear_cards(Generation *gen); + template void clear_decks(Generation *gen); + void reset_generation(nursery_space *gen); + void reset_generation(aging_space *gen); + void reset_generation(tenured_space *gen); + bool low_memory_p(); +}; + +struct data_heap_room { + cell nursery_size; + cell nursery_occupied; + cell nursery_free; + cell aging_size; + cell aging_occupied; + cell aging_free; + cell tenured_size; + cell tenured_occupied; + cell tenured_total_free; + cell tenured_contiguous_free; + cell tenured_free_block_count; + cell cards; + cell decks; + cell mark_stack; }; } diff --git a/vm/data_roots.hpp b/vm/data_roots.hpp new file mode 100644 index 0000000000..52654c49f0 --- /dev/null +++ b/vm/data_roots.hpp @@ -0,0 +1,59 @@ +namespace factor +{ + +template +struct data_root : public tagged { + factor_vm *parent; + + void push() + { + parent->data_roots.push_back((cell)this); + } + + explicit data_root(cell value_, factor_vm *parent_) + : tagged(value_), parent(parent_) + { + push(); + } + + explicit data_root(Type *value_, factor_vm *parent_) : + tagged(value_), parent(parent_) + { + push(); + } + + const data_root& operator=(const Type *x) { tagged::operator=(x); return *this; } + const data_root& operator=(const cell &x) { tagged::operator=(x); return *this; } + + ~data_root() + { +#ifdef FACTOR_DEBUG + assert(parent->data_roots.back() == (cell)this); +#endif + parent->data_roots.pop_back(); + } +}; + +/* A similar hack for the bignum implementation */ +struct gc_bignum { + bignum **addr; + factor_vm *parent; + + gc_bignum(bignum **addr_, factor_vm *parent_) : addr(addr_), parent(parent_) + { + if(*addr_) parent->check_data_pointer(*addr_); + parent->bignum_roots.push_back((cell)addr); + } + + ~gc_bignum() + { +#ifdef FACTOR_DEBUG + assert(parent->bignum_roots.back() == (cell)addr); +#endif + parent->bignum_roots.pop_back(); + } +}; + +#define GC_BIGNUM(x) gc_bignum x##__data_root(&x,this) + +} diff --git a/vm/debug.cpp b/vm/debug.cpp index 4b47e24221..fee3e6a257 100755 --- a/vm/debug.cpp +++ b/vm/debug.cpp @@ -3,36 +3,31 @@ namespace factor { -void factor_vm::print_chars(string* str) +std::ostream &operator<<(std::ostream &out, const string *str) { - cell i; - for(i = 0; i < string_capacity(str); i++) - putchar(string_nth(str,i)); + for(cell i = 0; i < string_capacity(str); i++) + out << (char)str->nth(i); + return out; } void factor_vm::print_word(word* word, cell nesting) { if(tagged(word->vocabulary).type_p(STRING_TYPE)) - { - print_chars(untag(word->vocabulary)); - print_string(":"); - } + std::cout << untag(word->vocabulary) << ":"; if(tagged(word->name).type_p(STRING_TYPE)) - print_chars(untag(word->name)); + std::cout << untag(word->name); else { - print_string("#name,nesting); - print_string(">"); + std::cout << ">"; } } -void factor_vm::print_factor_string(string* str) +void factor_vm::print_factor_string(string *str) { - putchar('"'); - print_chars(str); - putchar('"'); + std::cout << '"' << str << '"'; } void factor_vm::print_array(array* array, cell nesting) @@ -51,12 +46,12 @@ void factor_vm::print_array(array* array, cell nesting) for(i = 0; i < length; i++) { - print_string(" "); + std::cout << " "; print_nested_obj(array_nth(array,i),nesting); } if(trimmed) - print_string("..."); + std::cout << "..."; } void factor_vm::print_tuple(tuple *tuple, cell nesting) @@ -64,12 +59,10 @@ void factor_vm::print_tuple(tuple *tuple, cell nesting) tuple_layout *layout = untag(tuple->layout); cell length = to_fixnum(layout->size); - print_string(" "); + std::cout << " "; print_nested_obj(layout->klass,nesting); - cell i; bool trimmed; - if(length > 10 && !full_output) { trimmed = true; @@ -78,21 +71,21 @@ void factor_vm::print_tuple(tuple *tuple, cell nesting) else trimmed = false; - for(i = 0; i < length; i++) + for(cell i = 0; i < length; i++) { - print_string(" "); + std::cout << " "; print_nested_obj(tuple->data()[i],nesting); } if(trimmed) - print_string("..."); + std::cout << "..."; } void factor_vm::print_nested_obj(cell obj, fixnum nesting) { if(nesting <= 0 && !full_output) { - print_string(" ... "); + std::cout << " ... "; return; } @@ -101,7 +94,7 @@ void factor_vm::print_nested_obj(cell obj, fixnum nesting) switch(tagged(obj).type()) { case FIXNUM_TYPE: - print_fixnum(untag_fixnum(obj)); + std::cout << untag_fixnum(obj); break; case WORD_TYPE: print_word(untag(obj),nesting - 1); @@ -110,30 +103,27 @@ void factor_vm::print_nested_obj(cell obj, fixnum nesting) print_factor_string(untag(obj)); break; case F_TYPE: - print_string("f"); + std::cout << "f"; break; case TUPLE_TYPE: - print_string("T{"); + std::cout << "T{"; print_tuple(untag(obj),nesting - 1); - print_string(" }"); + std::cout << " }"; break; case ARRAY_TYPE: - print_string("{"); + std::cout << "{"; print_array(untag(obj),nesting - 1); - print_string(" }"); + std::cout << " }"; break; case QUOTATION_TYPE: - print_string("["); + std::cout << "["; quot = untag(obj); print_array(untag(quot->array),nesting - 1); - print_string(" ]"); + std::cout << " ]"; break; default: - print_string("#(obj).type()); - print_string(" @ "); - print_cell_hex(obj); - print_string(">"); + std::cout << "#(obj).type() << " @ "; + std::cout << std::hex << obj << std::dec << ">"; break; } } @@ -148,19 +138,19 @@ void factor_vm::print_objects(cell *start, cell *end) for(; start <= end; start++) { print_obj(*start); - nl(); + std::cout << std::endl; } } void factor_vm::print_datastack() { - print_string("==== DATA STACK:\n"); + std::cout << "==== DATA STACK:\n"; print_objects((cell *)ds_bot,(cell *)ds); } void factor_vm::print_retainstack() { - print_string("==== RETAIN STACK:\n"); + std::cout << "==== RETAIN STACK:\n"; print_objects((cell *)rs_bot,(cell *)rs); } @@ -171,34 +161,48 @@ struct stack_frame_printer { void operator()(stack_frame *frame) { parent->print_obj(parent->frame_executing(frame)); - print_string("\n"); + std::cout << std::endl; parent->print_obj(parent->frame_scan(frame)); - print_string("\n"); - print_string("word/quot addr: "); - print_cell_hex((cell)parent->frame_executing(frame)); - print_string("\n"); - print_string("word/quot xt: "); - print_cell_hex((cell)frame->xt); - print_string("\n"); - print_string("return address: "); - print_cell_hex((cell)FRAME_RETURN_ADDRESS(frame,parent)); - print_string("\n"); + std::cout << std::endl; + std::cout << "word/quot addr: "; + std::cout << std::hex << (cell)parent->frame_executing(frame) << std::dec; + std::cout << std::endl; + std::cout << "word/quot xt: "; + std::cout << std::hex << (cell)frame->xt << std::dec; + std::cout << std::endl; + std::cout << "return address: "; + std::cout << std::hex << (cell)FRAME_RETURN_ADDRESS(frame,parent) << std::dec; + std::cout << std::endl; } }; void factor_vm::print_callstack() { - print_string("==== CALL STACK:\n"); + std::cout << "==== CALL STACK:\n"; stack_frame_printer printer(this); iterate_callstack(ctx,printer); } +struct padded_address { + cell value; + + explicit padded_address(cell value_) : value(value_) {} +}; + +std::ostream &operator<<(std::ostream &out, const padded_address &value) +{ + char prev = out.fill('0'); + out.width(sizeof(cell) * 2); + out << std::hex << value.value << std::dec; + out.fill(prev); + return out; +} + void factor_vm::dump_cell(cell x) { - print_cell_hex_pad(x); print_string(": "); + std::cout << padded_address(x) << ": "; x = *(cell *)x; - print_cell_hex_pad(x); print_string(" tag "); print_cell(TAG(x)); - nl(); + std::cout << padded_address(x) << " tag " << TAG(x) << std::endl; } void factor_vm::dump_memory(cell from, cell to) @@ -209,147 +213,160 @@ void factor_vm::dump_memory(cell from, cell to) dump_cell(from); } -void factor_vm::dump_zone(const char *name, zone *z) +template +void factor_vm::dump_generation(const char *name, Generation *gen) { - print_string(name); print_string(": "); - print_string("Start="); print_cell(z->start); - print_string(", size="); print_cell(z->size); - print_string(", here="); print_cell(z->here - z->start); nl(); + std::cout << name << ": "; + std::cout << "Start=" << gen->start; + std::cout << ", size=" << gen->size; + std::cout << ", end=" << gen->end; + std::cout << std::endl; } void factor_vm::dump_generations() { - dump_zone("Nursery",&nursery); - dump_zone("Aging",data->aging); - dump_zone("Tenured",data->tenured); + dump_generation("Nursery",&nursery); + dump_generation("Aging",data->aging); + dump_generation("Tenured",data->tenured); - print_string("Cards: base="); - print_cell((cell)data->cards); - print_string(", size="); - print_cell((cell)(data->cards_end - data->cards)); - nl(); + std::cout << "Cards:"; + std::cout << "base=" << (cell)data->cards << ", "; + std::cout << "size=" << (cell)(data->cards_end - data->cards) << std::endl; } +struct object_dumper { + factor_vm *parent; + cell type; + + explicit object_dumper(factor_vm *parent_, cell type_) : + parent(parent_), type(type_) {} + + void operator()(cell obj) + { + if(type == TYPE_COUNT || tagged(obj).type_p(type)) + { + std::cout << padded_address(obj) << " "; + parent->print_nested_obj(obj,2); + std::cout << std::endl; + } + } +}; + void factor_vm::dump_objects(cell type) { primitive_full_gc(); - begin_scan(); - - cell obj; - while(to_boolean(obj = next_object())) - { - if(type == TYPE_COUNT || tagged(obj).type_p(type)) - { - print_cell_hex_pad(obj); - print_string(" "); - print_nested_obj(obj,2); - nl(); - } - } - - end_scan(); + object_dumper dumper(this,type); + each_object(dumper); } -struct data_references_finder { +struct data_reference_slot_visitor { cell look_for, obj; factor_vm *parent; - explicit data_references_finder(cell look_for_, cell obj_, factor_vm *parent_) - : look_for(look_for_), obj(obj_), parent(parent_) { } + explicit data_reference_slot_visitor(cell look_for_, cell obj_, factor_vm *parent_) : + look_for(look_for_), obj(obj_), parent(parent_) { } void operator()(cell *scan) { if(look_for == *scan) { - print_cell_hex_pad(obj); - print_string(" "); + std::cout << padded_address(obj) << " "; parent->print_nested_obj(obj,2); - nl(); + std::cout << std::endl; } } }; +struct data_reference_object_visitor { + cell look_for; + factor_vm *parent; + + explicit data_reference_object_visitor(cell look_for_, factor_vm *parent_) : + look_for(look_for_), parent(parent_) {} + + void operator()(cell obj) + { + data_reference_slot_visitor visitor(look_for,obj,parent); + parent->do_slots(UNTAG(obj),visitor); + } +}; + void factor_vm::find_data_references(cell look_for) { - begin_scan(); - - cell obj; - - while(to_boolean(obj = next_object())) - { - data_references_finder finder(look_for,obj,this); - do_slots(UNTAG(obj),finder); - } - - end_scan(); + data_reference_object_visitor visitor(look_for,this); + each_object(visitor); } -/* Dump all code blocks for debugging */ -void factor_vm::dump_code_heap() -{ - cell reloc_size = 0, literal_size = 0; +struct code_block_printer { + factor_vm *parent; + cell reloc_size, literal_size; - heap_block *scan = code->first_block(); + explicit code_block_printer(factor_vm *parent_) : + parent(parent_), reloc_size(0), literal_size(0) {} - while(scan) + void operator()(code_block *scan, cell size) { const char *status; - if(scan->type() == FREE_BLOCK_TYPE) + if(scan->free_p()) status = "free"; - else if(code->state->is_marked_p(scan)) + else if(parent->code->marked_p(scan)) { - reloc_size += object_size(((code_block *)scan)->relocation); - literal_size += object_size(((code_block *)scan)->literals); + reloc_size += parent->object_size(scan->relocation); + literal_size += parent->object_size(scan->literals); status = "marked"; } else { - reloc_size += object_size(((code_block *)scan)->relocation); - literal_size += object_size(((code_block *)scan)->literals); + reloc_size += parent->object_size(scan->relocation); + literal_size += parent->object_size(scan->literals); status = "allocated"; } - print_cell_hex((cell)scan); print_string(" "); - print_cell_hex(scan->size()); print_string(" "); - print_string(status); print_string("\n"); - - scan = code->next_block(scan); + std::cout << std::hex << (cell)scan << std::dec << " "; + std::cout << std::hex << size << std::dec << " "; + std::cout << status << std::endl; } - - print_cell(reloc_size); print_string(" bytes of relocation data\n"); - print_cell(literal_size); print_string(" bytes of literal data\n"); +}; + +/* Dump all code blocks for debugging */ +void factor_vm::dump_code_heap() +{ + code_block_printer printer(this); + code->allocator->iterate(printer); + std::cout << printer.reloc_size << " bytes of relocation data\n"; + std::cout << printer.literal_size << " bytes of literal data\n"; } void factor_vm::factorbug() { if(fep_disabled) { - print_string("Low level debugger disabled\n"); + std::cout << "Low level debugger disabled\n"; exit(1); } /* open_console(); */ - print_string("Starting low level debugger...\n"); - print_string(" Basic commands:\n"); - print_string("q -- continue executing Factor - NOT SAFE\n"); - print_string("im -- save image to fep.image\n"); - print_string("x -- exit Factor\n"); - print_string(" Advanced commands:\n"); - print_string("d -- dump memory\n"); - print_string("u -- dump object at tagged \n"); - print_string(". -- print object at tagged \n"); - print_string("t -- toggle output trimming\n"); - print_string("s r -- dump data, retain stacks\n"); - print_string(".s .r .c -- print data, retain, call stacks\n"); - print_string("e -- dump environment\n"); - print_string("g -- dump generations\n"); - print_string("data -- data heap dump\n"); - print_string("words -- words dump\n"); - print_string("tuples -- tuples dump\n"); - print_string("refs -- find data heap references to object\n"); - print_string("push -- push object on data stack - NOT SAFE\n"); - print_string("code -- code heap dump\n"); + std::cout << "Starting low level debugger...\n"; + std::cout << " Basic commands:\n"; + std::cout << "q -- continue executing Factor - NOT SAFE\n"; + std::cout << "im -- save image to fep.image\n"; + std::cout << "x -- exit Factor\n"; + std::cout << " Advanced commands:\n"; + std::cout << "d -- dump memory\n"; + std::cout << "u -- dump object at tagged \n"; + std::cout << ". -- print object at tagged \n"; + std::cout << "t -- toggle output trimming\n"; + std::cout << "s r -- dump data, retain stacks\n"; + std::cout << ".s .r .c -- print data, retain, call stacks\n"; + std::cout << "e -- dump environment\n"; + std::cout << "g -- dump generations\n"; + std::cout << "data -- data heap dump\n"; + std::cout << "words -- words dump\n"; + std::cout << "tuples -- tuples dump\n"; + std::cout << "refs -- find data heap references to object\n"; + std::cout << "push -- push object on data stack - NOT SAFE\n"; + std::cout << "code -- code heap dump\n"; bool seen_command = false; @@ -357,7 +374,7 @@ void factor_vm::factorbug() { char cmd[1024]; - print_string("READY\n"); + std::cout << "READY\n"; fflush(stdout); if(scanf("%1000s",cmd) <= 0) @@ -397,7 +414,7 @@ void factor_vm::factorbug() { cell addr = read_cell_hex(); print_obj(addr); - print_string("\n"); + std::cout << std::endl; } else if(strcmp(cmd,"t") == 0) full_output = !full_output; @@ -413,9 +430,8 @@ void factor_vm::factorbug() print_callstack(); else if(strcmp(cmd,"e") == 0) { - int i; - for(i = 0; i < USER_ENV; i++) - dump_cell((cell)&userenv[i]); + for(cell i = 0; i < special_object_count; i++) + dump_cell((cell)&special_objects[i]); } else if(strcmp(cmd,"g") == 0) dump_generations(); @@ -430,9 +446,9 @@ void factor_vm::factorbug() else if(strcmp(cmd,"refs") == 0) { cell addr = read_cell_hex(); - print_string("Data heap references:\n"); + std::cout << "Data heap references:\n"; find_data_references(addr); - nl(); + std::cout << std::endl; } else if(strcmp(cmd,"words") == 0) dump_objects(WORD_TYPE); @@ -446,14 +462,14 @@ void factor_vm::factorbug() else if(strcmp(cmd,"code") == 0) dump_code_heap(); else - print_string("unknown command\n"); + std::cout << "unknown command\n"; } } void factor_vm::primitive_die() { - print_string("The die word was called by the library. Unless you called it yourself,\n"); - print_string("you have triggered a bug in Factor. Please report.\n"); + std::cout << "The die word was called by the library. Unless you called it yourself,\n"; + std::cout << "you have triggered a bug in Factor. Please report.\n"; factorbug(); } diff --git a/vm/dispatch.cpp b/vm/dispatch.cpp index 6bd9d6a13e..3eba483fe6 100755 --- a/vm/dispatch.cpp +++ b/vm/dispatch.cpp @@ -70,16 +70,6 @@ cell factor_vm::lookup_tuple_method(cell obj, cell methods) return false_object; } -cell factor_vm::lookup_hi_tag_method(cell obj, cell methods) -{ - array *hi_tag_methods = untag(methods); - cell tag = untag(obj)->h.hi_tag() - HEADER_TYPE; -#ifdef FACTOR_DEBUG - assert(tag < TYPE_COUNT - HEADER_TYPE); -#endif - return array_nth(hi_tag_methods,tag); -} - cell factor_vm::lookup_method(cell obj, cell methods) { cell tag = TAG(obj); @@ -92,13 +82,6 @@ cell factor_vm::lookup_method(cell obj, cell methods) else return method; } - else if(tag == OBJECT_TYPE) - { - if(TAG(method) == ARRAY_TYPE) - return lookup_hi_tag_method(obj,method); - else - return method; - } else return method; } @@ -112,15 +95,11 @@ void factor_vm::primitive_lookup_method() cell factor_vm::object_class(cell obj) { - switch(TAG(obj)) - { - case TUPLE_TYPE: + cell tag = TAG(obj); + if(tag == TUPLE_TYPE) return untag(obj)->layout; - case OBJECT_TYPE: - return untag(obj)->h.value; - default: - return tag_fixnum(TAG(obj)); - } + else + return tag_fixnum(tag); } cell factor_vm::method_cache_hashcode(cell klass, array *array) @@ -139,7 +118,7 @@ void factor_vm::update_method_cache(cell cache, cell klass, cell method) void factor_vm::primitive_mega_cache_miss() { - megamorphic_cache_misses++; + dispatch_stats.megamorphic_cache_misses++; cell cache = dpop(); fixnum index = untag_fixnum(dpop()); @@ -156,42 +135,38 @@ void factor_vm::primitive_mega_cache_miss() void factor_vm::primitive_reset_dispatch_stats() { - megamorphic_cache_hits = megamorphic_cache_misses = 0; + memset(&dispatch_stats,0,sizeof(dispatch_statistics)); } void factor_vm::primitive_dispatch_stats() { - growable_array stats(this); - stats.add(allot_cell(megamorphic_cache_hits)); - stats.add(allot_cell(megamorphic_cache_misses)); - stats.trim(); - dpush(stats.elements.value()); + dpush(tag(byte_array_from_value(&dispatch_stats))); } void quotation_jit::emit_mega_cache_lookup(cell methods_, fixnum index, cell cache_) { - gc_root methods(methods_,parent); - gc_root cache(cache_,parent); + data_root methods(methods_,parent); + data_root cache(cache_,parent); /* Generate machine code to determine the object's class. */ - emit_class_lookup(index,PIC_HI_TAG_TUPLE); + emit_class_lookup(index,PIC_TUPLE); /* Do a cache lookup. */ - emit_with(parent->userenv[MEGA_LOOKUP],cache.value()); + emit_with(parent->special_objects[MEGA_LOOKUP],cache.value()); /* If we end up here, the cache missed. */ - emit(parent->userenv[JIT_PROLOG]); + emit(parent->special_objects[JIT_PROLOG]); /* Push index, method table and cache on the stack. */ push(methods.value()); push(tag_fixnum(index)); push(cache.value()); - word_call(parent->userenv[MEGA_MISS_WORD]); + word_call(parent->special_objects[MEGA_MISS_WORD]); /* Now the new method has been stored into the cache, and its on the stack. */ - emit(parent->userenv[JIT_EPILOG]); - emit(parent->userenv[JIT_EXECUTE_JUMP]); + emit(parent->special_objects[JIT_EPILOG]); + emit(parent->special_objects[JIT_EXECUTE_JUMP]); } } diff --git a/vm/dispatch.hpp b/vm/dispatch.hpp index 412ef35bb4..87d08e2760 100644 --- a/vm/dispatch.hpp +++ b/vm/dispatch.hpp @@ -1,4 +1,16 @@ namespace factor { +struct dispatch_statistics { + cell megamorphic_cache_hits; + cell megamorphic_cache_misses; + + cell cold_call_to_ic_transitions; + cell ic_to_pic_transitions; + cell pic_to_mega_transitions; + + cell pic_tag_count; + cell pic_tuple_count; +}; + } diff --git a/vm/errors.cpp b/vm/errors.cpp index c587fa723a..7d7b1f0080 100755 --- a/vm/errors.cpp +++ b/vm/errors.cpp @@ -5,22 +5,24 @@ namespace factor void fatal_error(const char *msg, cell tagged) { - print_string("fatal_error: "); print_string(msg); - print_string(": "); print_cell_hex(tagged); nl(); + std::cout << "fatal_error: " << msg; + std::cout << ": " << std::hex << tagged << std::dec; + std::cout << std::endl; exit(1); } void critical_error(const char *msg, cell tagged) { - print_string("You have triggered a bug in Factor. Please report.\n"); - print_string("critical_error: "); print_string(msg); - print_string(": "); print_cell_hex(tagged); nl(); + std::cout << "You have triggered a bug in Factor. Please report.\n"; + std::cout << "critical_error: " << msg; + std::cout << ": " << std::hex << tagged << std::dec; + std::cout << std::endl; tls_vm()->factorbug(); } void out_of_memory() { - print_string("Out of memory\n\n"); + std::cout << "Out of memory\n\n"; tls_vm()->dump_generations(); exit(1); } @@ -29,14 +31,15 @@ void factor_vm::throw_error(cell error, stack_frame *callstack_top) { /* If the error handler is set, we rewind any C stack frames and pass the error to user-space. */ - if(!current_gc && to_boolean(userenv[BREAK_ENV])) + if(!current_gc && to_boolean(special_objects[OBJ_BREAK])) { /* If error was thrown during heap scan, we re-enable the GC */ gc_off = false; /* Reset local roots */ - gc_locals.clear(); - gc_bignums.clear(); + data_roots.clear(); + bignum_roots.clear(); + code_roots.clear(); /* If we had an underflow or overflow, stack pointers might be out of bounds */ @@ -53,23 +56,23 @@ void factor_vm::throw_error(cell error, stack_frame *callstack_top) else callstack_top = ctx->callstack_top; - throw_impl(userenv[BREAK_ENV],callstack_top,this); + throw_impl(special_objects[OBJ_BREAK],callstack_top,this); } /* Error was thrown in early startup before error handler is set, just crash. */ else { - print_string("You have triggered a bug in Factor. Please report.\n"); - print_string("early_error: "); + std::cout << "You have triggered a bug in Factor. Please report.\n"; + std::cout << "early_error: "; print_obj(error); - nl(); + std::cout << std::endl; factorbug(); } } void factor_vm::general_error(vm_error_type error, cell arg1, cell arg2, stack_frame *callstack_top) { - throw_error(allot_array_4(userenv[ERROR_ENV], + throw_error(allot_array_4(special_objects[OBJ_ERROR], tag_fixnum(error),arg1,arg2),callstack_top); } diff --git a/vm/factor.cpp b/vm/factor.cpp index 5548ebd610..d382745da8 100755 --- a/vm/factor.cpp +++ b/vm/factor.cpp @@ -4,7 +4,7 @@ namespace factor { factor_vm *vm; -unordered_map thread_vms; +std::map thread_vms; void init_globals() { @@ -15,29 +15,16 @@ void factor_vm::default_parameters(vm_parameters *p) { p->image_path = NULL; - /* We make a wild guess here that if we're running on ARM, we don't - have a lot of memory. */ -#ifdef FACTOR_ARM - p->ds_size = 8 * sizeof(cell); - p->rs_size = 8 * sizeof(cell); - - p->code_size = 4; - p->young_size = 1; - p->aging_size = 1; - p->tenured_size = 6; -#else p->ds_size = 32 * sizeof(cell); p->rs_size = 32 * sizeof(cell); p->code_size = 8 * sizeof(cell); p->young_size = sizeof(cell) / 4; p->aging_size = sizeof(cell) / 2; - p->tenured_size = 4 * sizeof(cell); -#endif + p->tenured_size = 24 * sizeof(cell); p->max_pic_size = 3; - p->secure_gc = false; p->fep = false; p->signals = true; @@ -85,7 +72,6 @@ void factor_vm::init_parameters_from_args(vm_parameters *p, int argc, vm_char ** else if(factor_arg(arg,STRING_LITERAL("-codeheap=%d"),&p->code_size)); else if(factor_arg(arg,STRING_LITERAL("-pic=%d"),&p->max_pic_size)); else if(factor_arg(arg,STRING_LITERAL("-callbacks=%d"),&p->callback_size)); - else if(STRCMP(arg,STRING_LITERAL("-securegc")) == 0) p->secure_gc = true; else if(STRCMP(arg,STRING_LITERAL("-fep")) == 0) p->fep = true; else if(STRCMP(arg,STRING_LITERAL("-nosignals")) == 0) p->signals = false; else if(STRNCMP(arg,STRING_LITERAL("-i="),3) == 0) p->image_path = arg + 3; @@ -96,14 +82,13 @@ void factor_vm::init_parameters_from_args(vm_parameters *p, int argc, vm_char ** /* Do some initialization that we do once only */ void factor_vm::do_stage1_init() { - print_string("*** Stage 2 early init... "); + std::cout << "*** Stage 2 early init... "; fflush(stdout); compile_all_words(); - userenv[STAGE2_ENV] = true_object; + special_objects[OBJ_STAGE2] = true_object; - print_string("done\n"); - fflush(stdout); + std::cout << "done\n"; } void factor_vm::init_factor(vm_parameters *p) @@ -148,17 +133,17 @@ void factor_vm::init_factor(vm_parameters *p) init_profiler(); - userenv[CPU_ENV] = allot_alien(false_object,(cell)FACTOR_CPU_STRING); - userenv[OS_ENV] = allot_alien(false_object,(cell)FACTOR_OS_STRING); - userenv[CELL_SIZE_ENV] = tag_fixnum(sizeof(cell)); - userenv[EXECUTABLE_ENV] = allot_alien(false_object,(cell)p->executable_path); - userenv[ARGS_ENV] = false_object; - userenv[EMBEDDED_ENV] = false_object; + special_objects[OBJ_CPU] = allot_alien(false_object,(cell)FACTOR_CPU_STRING); + special_objects[OBJ_OS] = allot_alien(false_object,(cell)FACTOR_OS_STRING); + special_objects[OBJ_CELL_SIZE] = tag_fixnum(sizeof(cell)); + special_objects[OBJ_EXECUTABLE] = allot_alien(false_object,(cell)p->executable_path); + special_objects[OBJ_ARGS] = false_object; + special_objects[OBJ_EMBEDDED] = false_object; /* We can GC now */ gc_off = false; - if(!to_boolean(userenv[STAGE2_ENV])) + if(!to_boolean(special_objects[OBJ_STAGE2])) do_stage1_init(); } @@ -173,7 +158,7 @@ void factor_vm::pass_args_to_factor(int argc, vm_char **argv) } args.trim(); - userenv[ARGS_ENV] = args.elements.value(); + special_objects[OBJ_ARGS] = args.elements.value(); } void factor_vm::start_factor(vm_parameters *p) @@ -181,13 +166,13 @@ void factor_vm::start_factor(vm_parameters *p) if(p->fep) factorbug(); nest_stacks(NULL); - c_to_factor_toplevel(userenv[BOOT_ENV]); + c_to_factor_toplevel(special_objects[OBJ_BOOT]); unnest_stacks(); } char *factor_vm::factor_eval_string(char *string) { - char *(*callback)(char *) = (char *(*)(char *))alien_offset(userenv[EVAL_CALLBACK_ENV]); + char *(*callback)(char *) = (char *(*)(char *))alien_offset(special_objects[OBJ_EVAL_CALLBACK]); return callback(string); } @@ -198,13 +183,13 @@ void factor_vm::factor_eval_free(char *result) void factor_vm::factor_yield() { - void (*callback)() = (void (*)())alien_offset(userenv[YIELD_CALLBACK_ENV]); + void (*callback)() = (void (*)())alien_offset(special_objects[OBJ_YIELD_CALLBACK]); callback(); } void factor_vm::factor_sleep(long us) { - void (*callback)(long) = (void (*)(long))alien_offset(userenv[SLEEP_CALLBACK_ENV]); + void (*callback)(long) = (void (*)(long))alien_offset(special_objects[OBJ_SLEEP_CALLBACK]); callback(us); } diff --git a/vm/free_list.cpp b/vm/free_list.cpp new file mode 100644 index 0000000000..cde6a2d781 --- /dev/null +++ b/vm/free_list.cpp @@ -0,0 +1,130 @@ +#include "master.hpp" + +namespace factor +{ + +void free_list::clear_free_list() +{ + for(cell i = 0; i < free_list_count; i++) + small_blocks[i].clear(); + large_blocks.clear(); + free_block_count = 0; + free_space = 0; +} + +void free_list::initial_free_list(cell start, cell end, cell occupied) +{ + clear_free_list(); + if(occupied != end - start) + { + free_heap_block *last_block = (free_heap_block *)(start + occupied); + last_block->make_free(end - (cell)last_block); + add_to_free_list(last_block); + } +} + +void free_list::add_to_free_list(free_heap_block *block) +{ + cell size = block->size(); + + free_block_count++; + free_space += size; + + if(size < free_list_count * block_granularity) + small_blocks[size / block_granularity].push_back(block); + else + large_blocks.insert(block); +} + +free_heap_block *free_list::find_free_block(cell size) +{ + /* Check small free lists */ + for(cell i = size / block_granularity; i < free_list_count; i++) + { + std::vector &blocks = small_blocks[i]; + if(blocks.size()) + { + free_heap_block *block = blocks.back(); + blocks.pop_back(); + + free_block_count--; + free_space -= block->size(); + + return block; + } + } + + /* Check large free lists */ + free_heap_block key; + key.make_free(size); + large_block_set::iterator iter = large_blocks.lower_bound(&key); + large_block_set::iterator end = large_blocks.end(); + + if(iter != end) + { + free_heap_block *block = *iter; + large_blocks.erase(iter); + + free_block_count--; + free_space -= block->size(); + + return block; + } + + return NULL; +} + +free_heap_block *free_list::split_free_block(free_heap_block *block, cell size) +{ + if(block->size() != size) + { + /* split the block in two */ + free_heap_block *split = (free_heap_block *)((cell)block + size); + split->make_free(block->size() - size); + block->make_free(size); + add_to_free_list(split); + } + + return block; +} + +bool free_list::can_allot_p(cell size) +{ + /* Check small free lists */ + for(cell i = size / block_granularity; i < free_list_count; i++) + { + if(small_blocks[i].size()) return true; + } + + /* Check large free lists */ + large_block_set::const_iterator iter = large_blocks.begin(); + large_block_set::const_iterator end = large_blocks.end(); + + for(; iter != end; iter++) + { + if((*iter)->size() >= size) return true; + } + + return false; +} + +cell free_list::largest_free_block() +{ + if(large_blocks.size()) + { + large_block_set::reverse_iterator last = large_blocks.rbegin(); + return (*last)->size(); + } + else + { + for(int i = free_list_count - 1; i >= 0; i--) + { + if(small_blocks[i].size()) + return small_blocks[i].back()->size(); + } + + return 0; + } +} + +} diff --git a/vm/free_list.hpp b/vm/free_list.hpp new file mode 100644 index 0000000000..305de0ac58 --- /dev/null +++ b/vm/free_list.hpp @@ -0,0 +1,50 @@ +namespace factor +{ + +static const cell free_list_count = 32; + +struct free_heap_block +{ + cell header; + + bool free_p() const + { + return header & 1 == 1; + } + + cell size() const + { + return header >> 3; + } + + void make_free(cell size) + { + header = (size << 3) | 1; + } +}; + +struct block_size_compare { + bool operator()(free_heap_block *a, free_heap_block *b) + { + return a->size() < b->size(); + } +}; + +typedef std::multiset large_block_set; + +struct free_list { + std::vector small_blocks[free_list_count]; + large_block_set large_blocks; + cell free_block_count; + cell free_space; + + void clear_free_list(); + void initial_free_list(cell start, cell end, cell occupied); + void add_to_free_list(free_heap_block *block); + free_heap_block *find_free_block(cell size); + free_heap_block *split_free_block(free_heap_block *block, cell size); + bool can_allot_p(cell size); + cell largest_free_block(); +}; + +} diff --git a/vm/free_list_allocator.hpp b/vm/free_list_allocator.hpp new file mode 100644 index 0000000000..a4801daa72 --- /dev/null +++ b/vm/free_list_allocator.hpp @@ -0,0 +1,272 @@ +namespace factor +{ + +template struct free_list_allocator { + cell size; + cell start; + cell end; + free_list free_blocks; + mark_bits state; + + explicit free_list_allocator(cell size, cell start); + void initial_free_list(cell occupied); + bool contains_p(Block *block); + Block *first_block(); + Block *last_block(); + Block *next_block_after(Block *block); + Block *next_allocated_block_after(Block *block); + bool can_allot_p(cell size); + Block *allot(cell size); + void free(Block *block); + cell occupied_space(); + cell free_space(); + cell largest_free_block(); + cell free_block_count(); + void sweep(); + template void sweep(Iterator &iter); + template void compact(Iterator &iter, Sizer &sizer); + template void iterate(Iterator &iter, Sizer &sizer); + template void iterate(Iterator &iter); +}; + +template +free_list_allocator::free_list_allocator(cell size_, cell start_) : + size(size_), + start(start_), + end(start_ + size_), + state(mark_bits(size_,start_)) +{ + initial_free_list(0); +} + +template void free_list_allocator::initial_free_list(cell occupied) +{ + free_blocks.initial_free_list(start,end,occupied); +} + +template bool free_list_allocator::contains_p(Block *block) +{ + return ((cell)block - start) < size; +} + +template Block *free_list_allocator::first_block() +{ + return (Block *)start; +} + +template Block *free_list_allocator::last_block() +{ + return (Block *)end; +} + +template Block *free_list_allocator::next_block_after(Block *block) +{ + return (Block *)((cell)block + block->size()); +} + +template Block *free_list_allocator::next_allocated_block_after(Block *block) +{ + while(block != this->last_block() && block->free_p()) + { + free_heap_block *free_block = (free_heap_block *)block; + block = (object *)((cell)free_block + free_block->size()); + } + + if(block == this->last_block()) + return NULL; + else + return block; +} + +template bool free_list_allocator::can_allot_p(cell size) +{ + return free_blocks.can_allot_p(size); +} + +template Block *free_list_allocator::allot(cell size) +{ + size = align(size,block_granularity); + + free_heap_block *block = free_blocks.find_free_block(size); + if(block) + { + block = free_blocks.split_free_block(block,size); + return (Block *)block; + } + else + return NULL; +} + +template void free_list_allocator::free(Block *block) +{ + free_heap_block *free_block = (free_heap_block *)block; + free_block->make_free(block->size()); + free_blocks.add_to_free_list(free_block); +} + +template cell free_list_allocator::free_space() +{ + return free_blocks.free_space; +} + +template cell free_list_allocator::occupied_space() +{ + return size - free_blocks.free_space; +} + +template cell free_list_allocator::largest_free_block() +{ + return free_blocks.largest_free_block(); +} + +template cell free_list_allocator::free_block_count() +{ + return free_blocks.free_block_count; +} + +template +void free_list_allocator::sweep() +{ + free_blocks.clear_free_list(); + + Block *start = this->first_block(); + Block *end = this->last_block(); + + while(start != end) + { + /* find next unmarked block */ + start = state.next_unmarked_block_after(start); + + if(start != end) + { + /* find size */ + cell size = state.unmarked_block_size(start); + assert(size > 0); + + free_heap_block *free_block = (free_heap_block *)start; + free_block->make_free(size); + free_blocks.add_to_free_list(free_block); + + start = (Block *)((char *)start + size); + } + } +} + +template +template +void free_list_allocator::sweep(Iterator &iter) +{ + free_blocks.clear_free_list(); + + Block *prev = NULL; + Block *scan = this->first_block(); + Block *end = this->last_block(); + + while(scan != end) + { + cell size = scan->size(); + + if(scan->free_p()) + { + if(prev && prev->free_p()) + { + free_heap_block *free_prev = (free_heap_block *)prev; + free_prev->make_free(free_prev->size() + size); + } + else + prev = scan; + } + else if(this->state.marked_p(scan)) + { + if(prev && prev->free_p()) + free_blocks.add_to_free_list((free_heap_block *)prev); + prev = scan; + iter(scan,size); + } + else + { + if(prev && prev->free_p()) + { + free_heap_block *free_prev = (free_heap_block *)prev; + free_prev->make_free(free_prev->size() + size); + } + else + { + free_heap_block *free_block = (free_heap_block *)scan; + free_block->make_free(size); + prev = scan; + } + } + + scan = (Block *)((cell)scan + size); + } + + if(prev && prev->free_p()) + free_blocks.add_to_free_list((free_heap_block *)prev); +} + +template struct heap_compactor { + mark_bits *state; + char *address; + Iterator &iter; + + explicit heap_compactor(mark_bits *state_, Block *address_, Iterator &iter_) : + state(state_), address((char *)address_), iter(iter_) {} + + void operator()(Block *block, cell size) + { + if(this->state->marked_p(block)) + { + iter(block,(Block *)address,size); + address += size; + } + } +}; + +/* The forwarding map must be computed first by calling +state.compute_forwarding(). */ +template +template +void free_list_allocator::compact(Iterator &iter, Sizer &sizer) +{ + heap_compactor compactor(&state,first_block(),iter); + iterate(compactor,sizer); + + /* Now update the free list; there will be a single free block at + the end */ + free_blocks.initial_free_list(start,end,(cell)compactor.address - start); +} + +/* During compaction we have to be careful and measure object sizes differently */ +template +template +void free_list_allocator::iterate(Iterator &iter, Sizer &sizer) +{ + Block *scan = first_block(); + Block *end = last_block(); + + while(scan != end) + { + cell size = sizer(scan); + Block *next = (Block *)((cell)scan + size); + if(!scan->free_p()) iter(scan,size); + scan = next; + } +} + +template struct standard_sizer { + cell operator()(Block *block) + { + return block->size(); + } +}; + +template +template +void free_list_allocator::iterate(Iterator &iter) +{ + standard_sizer sizer; + iterate(iter,sizer); +} + +} diff --git a/vm/full_collector.cpp b/vm/full_collector.cpp index f9db1c8653..3b92e2574e 100644 --- a/vm/full_collector.cpp +++ b/vm/full_collector.cpp @@ -4,197 +4,144 @@ namespace factor { full_collector::full_collector(factor_vm *parent_) : - copying_collector( + collector( parent_, - &parent_->gc_stats.full_stats, parent_->data->tenured, full_policy(parent_)) {} -struct stack_frame_marker { - factor_vm *parent; +/* After a sweep, invalidate any code heap roots which are not marked, +so that if a block makes a tail call to a generic word, and the PIC +compiler triggers a GC, and the caller block gets gets GCd as a result, +the PIC code won't try to overwrite the call site */ +void factor_vm::update_code_roots_for_sweep() +{ + std::vector::const_iterator iter = code_roots.begin(); + std::vector::const_iterator end = code_roots.end(); + + mark_bits *state = &code->allocator->state; + + for(; iter < end; iter++) + { + code_root *root = *iter; + code_block *block = (code_block *)(root->value & -block_granularity); + if(root->valid && !state->marked_p(block)) + root->valid = false; + } +} + +/* After a compaction, invalidate any code heap roots which are not +marked as above, and also slide the valid roots up so that call sites +can be updated correctly. */ +void factor_vm::update_code_roots_for_compaction() +{ + std::vector::const_iterator iter = code_roots.begin(); + std::vector::const_iterator end = code_roots.end(); + + mark_bits *state = &code->allocator->state; + + for(; iter < end; iter++) + { + code_root *root = *iter; + code_block *block = (code_block *)(root->value & -block_granularity); + + /* Offset of return address within 16-byte allocation line */ + cell offset = root->value - (cell)block; + + if(root->valid && state->marked_p((code_block *)root->value)) + { + block = state->forward_block(block); + root->value = (cell)block + offset; + } + else + root->valid = false; + } +} + +struct code_block_marker { + code_heap *code; full_collector *collector; - explicit stack_frame_marker(full_collector *collector_) : - parent(collector_->parent), collector(collector_) {} + explicit code_block_marker(code_heap *code_, full_collector *collector_) : + code(code_), collector(collector_) {} - void operator()(stack_frame *frame) + code_block *operator()(code_block *compiled) { - collector->mark_code_block(parent->frame_code(frame)); - } -}; - -/* Mark code blocks executing in currently active stack frames. */ -void full_collector::mark_active_blocks() -{ - stack_frame_marker marker(this); - parent->iterate_active_frames(marker); -} - -void full_collector::mark_object_code_block(object *obj) -{ - switch(obj->h.hi_tag()) - { - case WORD_TYPE: + if(!code->marked_p(compiled)) { - word *w = (word *)obj; - if(w->code) - mark_code_block(w->code); - if(w->profiling) - mark_code_block(w->profiling); - break; + code->set_marked_p(compiled); + collector->trace_literal_references(compiled); } - case QUOTATION_TYPE: - { - quotation *q = (quotation *)obj; - if(q->code) - mark_code_block(q->code); - break; - } - case CALLSTACK_TYPE: - { - callstack *stack = (callstack *)obj; - stack_frame_marker marker(this); - parent->iterate_callstack_object(stack,marker); - break; - } - } -} -struct callback_tracer { - full_collector *collector; - - callback_tracer(full_collector *collector_) : collector(collector_) {} - - void operator()(callback *stub) - { - collector->mark_code_block(stub->compiled); + return compiled; } }; -void full_collector::trace_callbacks() -{ - callback_tracer tracer(this); - parent->callbacks->iterate(tracer); -} - -/* Trace all literals referenced from a code block. Only for aging and nursery collections */ -void full_collector::trace_literal_references(code_block *compiled) -{ - this->trace_handle(&compiled->owner); - this->trace_handle(&compiled->literals); - this->trace_handle(&compiled->relocation); -} - -/* Mark all literals referenced from a word XT. Only for tenured -collections */ -void full_collector::mark_code_block(code_block *compiled) -{ - this->code->mark_block(compiled); - trace_literal_references(compiled); -} - -void full_collector::cheneys_algorithm() -{ - while(scan && scan < target->here) - { - object *obj = (object *)scan; - this->trace_slots(obj); - this->mark_object_code_block(obj); - scan = target->next_object_after(this->parent,scan); - } -} - -/* After growing the heap, we have to perform a full relocation to update -references to card and deck arrays. */ -struct big_code_heap_updater { - factor_vm *parent; - - big_code_heap_updater(factor_vm *parent_) : parent(parent_) {} - - void operator()(heap_block *block) - { - parent->relocate_code_block((code_block *)block); - } -}; - -/* After a full GC that did not grow the heap, we have to update references -to literals and other words. */ -struct small_code_heap_updater { - factor_vm *parent; - - small_code_heap_updater(factor_vm *parent_) : parent(parent_) {} - - void operator()(heap_block *block) - { - parent->update_code_block_for_full_gc((code_block *)block); - } -}; - -void factor_vm::collect_full_impl(bool trace_contexts_p) +void factor_vm::collect_mark_impl(bool trace_contexts_p) { full_collector collector(this); - code->state->clear_mark_bits(); + code->clear_mark_bits(); + data->tenured->clear_mark_bits(); + data->tenured->clear_mark_stack(); + + code_block_visitor code_marker(this,code_block_marker(code,&collector)); collector.trace_roots(); if(trace_contexts_p) { collector.trace_contexts(); - collector.mark_active_blocks(); - collector.trace_callbacks(); + code_marker.visit_context_code_blocks(); + code_marker.visit_callback_code_blocks(); } - collector.cheneys_algorithm(); + std::vector *mark_stack = &data->tenured->mark_stack; - reset_generation(data->aging); - nursery.here = nursery.start; + while(!mark_stack->empty()) + { + object *obj = mark_stack->back(); + mark_stack->pop_back(); + collector.trace_object(obj); + code_marker.visit_object_code_block(obj); + } + + data->reset_generation(data->tenured); + data->reset_generation(data->aging); + data->reset_generation(&nursery); + code->clear_remembered_set(); } -void factor_vm::collect_growing_heap(cell requested_bytes, - bool trace_contexts_p, - bool compact_code_heap_p) +void factor_vm::collect_sweep_impl() +{ + current_gc->event->started_data_sweep(); + data->tenured->sweep(); + update_code_roots_for_sweep(); + current_gc->event->ended_data_sweep(); +} + +void factor_vm::collect_full(bool trace_contexts_p) +{ + collect_mark_impl(trace_contexts_p); + collect_sweep_impl(); + if(data->tenured->largest_free_block() <= data->nursery->size + data->aging->size) + collect_compact_impl(trace_contexts_p); + else + update_code_heap_words_and_literals(); +} + +void factor_vm::collect_compact(bool trace_contexts_p) +{ + collect_mark_impl(trace_contexts_p); + collect_compact_impl(trace_contexts_p); +} + +void factor_vm::collect_growing_heap(cell requested_bytes, bool trace_contexts_p) { /* Grow the data heap and copy all live objects to the new heap. */ data_heap *old = data; set_data_heap(data->grow(requested_bytes)); - collect_full_impl(trace_contexts_p); + collect_mark_impl(trace_contexts_p); + collect_compact_code_impl(trace_contexts_p); delete old; - - if(compact_code_heap_p) - { - compact_code_heap(trace_contexts_p); - big_code_heap_updater updater(this); - iterate_code_heap(updater); - } - else - { - big_code_heap_updater updater(this); - code->free_unmarked(updater); - } - - code->clear_remembered_set(); -} - -void factor_vm::collect_full(bool trace_contexts_p, bool compact_code_heap_p) -{ - /* Copy all live objects to the tenured semispace. */ - std::swap(data->tenured,data->tenured_semispace); - reset_generation(data->tenured); - collect_full_impl(trace_contexts_p); - - if(compact_code_heap_p) - { - compact_code_heap(trace_contexts_p); - big_code_heap_updater updater(this); - iterate_code_heap(updater); - } - else - { - small_code_heap_updater updater(this); - code->free_unmarked(updater); - } - - code->clear_remembered_set(); } } diff --git a/vm/full_collector.hpp b/vm/full_collector.hpp index 8cc37f782d..eb125b7429 100644 --- a/vm/full_collector.hpp +++ b/vm/full_collector.hpp @@ -3,26 +3,31 @@ namespace factor struct full_policy { factor_vm *parent; - zone *tenured; + tenured_space *tenured; - full_policy(factor_vm *parent_) : parent(parent_), tenured(parent->data->tenured) {} + explicit full_policy(factor_vm *parent_) : parent(parent_), tenured(parent->data->tenured) {} bool should_copy_p(object *untagged) { return !tenured->contains_p(untagged); } + + void promoted_object(object *obj) + { + tenured->mark_and_push(obj); + } + + void visited_object(object *obj) + { + if(!tenured->marked_p(obj)) + tenured->mark_and_push(obj); + } }; -struct full_collector : copying_collector { +struct full_collector : collector { bool trace_contexts_p; - full_collector(factor_vm *parent_); - void mark_active_blocks(); - void mark_object_code_block(object *object); - void trace_callbacks(); - void trace_literal_references(code_block *compiled); - void mark_code_block(code_block *compiled); - void cheneys_algorithm(); + explicit full_collector(factor_vm *parent_); }; } diff --git a/vm/gc.cpp b/vm/gc.cpp index 9e361a37e8..de8a2886f7 100755 --- a/vm/gc.cpp +++ b/vm/gc.cpp @@ -3,9 +3,128 @@ namespace factor { -gc_state::gc_state(gc_op op_) : op(op_), start_time(current_micros()) {} +gc_event::gc_event(gc_op op_, factor_vm *parent) : + op(op_), + cards_scanned(0), + decks_scanned(0), + code_blocks_scanned(0), + start_time(current_micros()), + card_scan_time(0), + code_scan_time(0), + data_sweep_time(0), + code_sweep_time(0), + compaction_time(0) +{ + data_heap_before = parent->data_room(); + code_heap_before = parent->code_room(); + start_time = current_micros(); +} -gc_state::~gc_state() {} +void gc_event::started_card_scan() +{ + temp_time = current_micros(); +} + +void gc_event::ended_card_scan(cell cards_scanned_, cell decks_scanned_) +{ + cards_scanned += cards_scanned_; + decks_scanned += decks_scanned_; + card_scan_time = (current_micros() - temp_time); +} + +void gc_event::started_code_scan() +{ + temp_time = current_micros(); +} + +void gc_event::ended_code_scan(cell code_blocks_scanned_) +{ + code_blocks_scanned += code_blocks_scanned_; + code_scan_time = (current_micros() - temp_time); +} + +void gc_event::started_data_sweep() +{ + temp_time = current_micros(); +} + +void gc_event::ended_data_sweep() +{ + data_sweep_time = (current_micros() - temp_time); +} + +void gc_event::started_code_sweep() +{ + temp_time = current_micros(); +} + +void gc_event::ended_code_sweep() +{ + code_sweep_time = (current_micros() - temp_time); +} + +void gc_event::started_compaction() +{ + temp_time = current_micros(); +} + +void gc_event::ended_compaction() +{ + compaction_time = (current_micros() - temp_time); +} + +void gc_event::ended_gc(factor_vm *parent) +{ + data_heap_after = parent->data_room(); + code_heap_after = parent->code_room(); + total_time = current_micros() - start_time; +} + +gc_state::gc_state(gc_op op_, factor_vm *parent) : op(op_), start_time(current_micros()) +{ + event = new gc_event(op,parent); +} + +gc_state::~gc_state() +{ + delete event; + event = NULL; +} + +void factor_vm::end_gc() +{ + current_gc->event->ended_gc(this); + if(gc_events) gc_events->push_back(*current_gc->event); + delete current_gc->event; + current_gc->event = NULL; +} + +void factor_vm::start_gc_again() +{ + end_gc(); + + switch(current_gc->op) + { + case collect_nursery_op: + current_gc->op = collect_aging_op; + break; + case collect_aging_op: + current_gc->op = collect_to_tenured_op; + break; + case collect_to_tenured_op: + current_gc->op = collect_full_op; + break; + case collect_full_op: + case collect_compact_op: + current_gc->op = collect_growing_heap_op; + break; + default: + critical_error("Bad GC op",current_gc->op); + break; + } + + current_gc->event = new gc_event(current_gc->op,this); +} void factor_vm::update_code_heap_for_minor_gc(std::set *remembered_set) { @@ -16,79 +135,64 @@ void factor_vm::update_code_heap_for_minor_gc(std::set *remembered for(; iter != end; iter++) update_literal_references(*iter); } -void factor_vm::record_gc_stats(generation_statistics *stats) -{ - cell gc_elapsed = (current_micros() - current_gc->start_time); - stats->collections++; - stats->gc_time += gc_elapsed; - if(stats->max_gc_time < gc_elapsed) - stats->max_gc_time = gc_elapsed; -} - -void factor_vm::gc(gc_op op, - cell requested_bytes, - bool trace_contexts_p, - bool compact_code_heap_p) +void factor_vm::gc(gc_op op, cell requested_bytes, bool trace_contexts_p) { assert(!gc_off); assert(!current_gc); save_stacks(); - current_gc = new gc_state(op); + current_gc = new gc_state(op,this); /* Keep trying to GC higher and higher generations until we don't run out of space */ if(setjmp(current_gc->gc_unwind)) { /* We come back here if a generation is full */ - switch(current_gc->op) - { - case collect_nursery_op: - current_gc->op = collect_aging_op; - break; - case collect_aging_op: - current_gc->op = collect_to_tenured_op; - break; - case collect_to_tenured_op: - current_gc->op = collect_full_op; - break; - case collect_full_op: - current_gc->op = collect_growing_heap_op; - break; - default: - critical_error("Bad GC op\n",op); - break; - } + start_gc_again(); } + current_gc->event->op = current_gc->op; + switch(current_gc->op) { case collect_nursery_op: collect_nursery(); - record_gc_stats(&gc_stats.nursery_stats); break; case collect_aging_op: collect_aging(); - record_gc_stats(&gc_stats.aging_stats); + if(data->low_memory_p()) + { + current_gc->op = collect_full_op; + current_gc->event->op = collect_full_op; + collect_full(trace_contexts_p); + } break; case collect_to_tenured_op: collect_to_tenured(); - record_gc_stats(&gc_stats.aging_stats); + if(data->low_memory_p()) + { + current_gc->op = collect_full_op; + current_gc->event->op = collect_full_op; + collect_full(trace_contexts_p); + } break; case collect_full_op: - collect_full(trace_contexts_p,compact_code_heap_p); - record_gc_stats(&gc_stats.full_stats); + collect_full(trace_contexts_p); + break; + case collect_compact_op: + collect_compact(trace_contexts_p); break; case collect_growing_heap_op: - collect_growing_heap(requested_bytes,trace_contexts_p,compact_code_heap_p); - record_gc_stats(&gc_stats.full_stats); + collect_growing_heap(requested_bytes,trace_contexts_p); break; default: - critical_error("Bad GC op\n",op); + critical_error("Bad GC op\n",current_gc->op); break; } + end_gc(); + delete current_gc; current_gc = NULL; } @@ -97,67 +201,21 @@ void factor_vm::primitive_minor_gc() { gc(collect_nursery_op, 0, /* requested size */ - true, /* trace contexts? */ - false /* compact code heap? */); + true /* trace contexts? */); } void factor_vm::primitive_full_gc() { gc(collect_full_op, 0, /* requested size */ - true, /* trace contexts? */ - false /* compact code heap? */); + true /* trace contexts? */); } void factor_vm::primitive_compact_gc() { - gc(collect_full_op, + gc(collect_compact_op, 0, /* requested size */ - true, /* trace contexts? */ - true /* compact code heap? */); -} - -void factor_vm::add_gc_stats(generation_statistics *stats, growable_array *result) -{ - result->add(allot_cell(stats->collections)); - result->add(tag(long_long_to_bignum(stats->gc_time))); - result->add(tag(long_long_to_bignum(stats->max_gc_time))); - result->add(allot_cell(stats->collections == 0 ? 0 : stats->gc_time / stats->collections)); - result->add(allot_cell(stats->object_count)); - result->add(tag(long_long_to_bignum(stats->bytes_copied))); -} - -void factor_vm::primitive_gc_stats() -{ - growable_array result(this); - - add_gc_stats(&gc_stats.nursery_stats,&result); - add_gc_stats(&gc_stats.aging_stats,&result); - add_gc_stats(&gc_stats.full_stats,&result); - - u64 total_gc_time = - gc_stats.nursery_stats.gc_time + - gc_stats.aging_stats.gc_time + - gc_stats.full_stats.gc_time; - - result.add(tag(ulong_long_to_bignum(total_gc_time))); - result.add(tag(ulong_long_to_bignum(gc_stats.cards_scanned))); - result.add(tag(ulong_long_to_bignum(gc_stats.decks_scanned))); - result.add(tag(ulong_long_to_bignum(gc_stats.card_scan_time))); - result.add(allot_cell(gc_stats.code_blocks_scanned)); - - result.trim(); - dpush(result.elements.value()); -} - -void factor_vm::clear_gc_stats() -{ - memset(&gc_stats,0,sizeof(gc_statistics)); -} - -void factor_vm::primitive_clear_gc_stats() -{ - clear_gc_stats(); + true /* trace contexts? */); } /* classes.tuple uses this to reshape tuples; tools.deploy.shaker uses this @@ -191,20 +249,20 @@ void factor_vm::primitive_become() compile_all_words(); } -void factor_vm::inline_gc(cell *gc_roots_base, cell gc_roots_size) +void factor_vm::inline_gc(cell *data_roots_base, cell data_roots_size) { - for(cell i = 0; i < gc_roots_size; i++) - gc_locals.push_back((cell)&gc_roots_base[i]); + for(cell i = 0; i < data_roots_size; i++) + data_roots.push_back((cell)&data_roots_base[i]); primitive_minor_gc(); - for(cell i = 0; i < gc_roots_size; i++) - gc_locals.pop_back(); + for(cell i = 0; i < data_roots_size; i++) + data_roots.pop_back(); } -VM_C_API void inline_gc(cell *gc_roots_base, cell gc_roots_size, factor_vm *parent) +VM_C_API void inline_gc(cell *data_roots_base, cell data_roots_size, factor_vm *parent) { - parent->inline_gc(gc_roots_base,gc_roots_size); + parent->inline_gc(data_roots_base,data_roots_size); } /* @@ -213,17 +271,18 @@ VM_C_API void inline_gc(cell *gc_roots_base, cell gc_roots_size, factor_vm *pare */ object *factor_vm::allot_large_object(header header, cell size) { - /* If tenured space does not have enough room, collect */ - if(data->tenured->here + size > data->tenured->end) - primitive_full_gc(); - - /* If it still won't fit, grow the heap */ - if(data->tenured->here + size > data->tenured->end) + /* If tenured space does not have enough room, collect and compact */ + if(!data->tenured->can_allot_p(size)) { - gc(collect_growing_heap_op, - size, /* requested size */ - true, /* trace contexts? */ - false /* compact code heap? */); + primitive_compact_gc(); + + /* If it still won't fit, grow the heap */ + if(!data->tenured->can_allot_p(size)) + { + gc(collect_growing_heap_op, + size, /* requested size */ + true /* trace contexts? */); + } } object *obj = data->tenured->allot(size); @@ -239,4 +298,23 @@ object *factor_vm::allot_large_object(header header, cell size) return obj; } +void factor_vm::primitive_enable_gc_events() +{ + gc_events = new std::vector(); +} + +void factor_vm::primitive_disable_gc_events() +{ + if(gc_events) + { + byte_array *data = byte_array_from_values(&gc_events->front(),gc_events->size()); + dpush(tag(data)); + + delete gc_events; + gc_events = NULL; + } + else + dpush(false_object); +} + } diff --git a/vm/gc.hpp b/vm/gc.hpp index 18b926ed8c..a9250eddb2 100755 --- a/vm/gc.hpp +++ b/vm/gc.hpp @@ -6,37 +6,53 @@ enum gc_op { collect_aging_op, collect_to_tenured_op, collect_full_op, + collect_compact_op, collect_growing_heap_op }; -/* statistics */ -struct generation_statistics { - cell collections; - u64 gc_time; - u64 max_gc_time; - cell object_count; - u64 bytes_copied; -}; +struct gc_event { + gc_op op; + data_heap_room data_heap_before; + code_heap_room code_heap_before; + data_heap_room data_heap_after; + code_heap_room code_heap_after; + cell cards_scanned; + cell decks_scanned; + cell code_blocks_scanned; + u64 start_time; + cell total_time; + cell card_scan_time; + cell code_scan_time; + cell data_sweep_time; + cell code_sweep_time; + cell compaction_time; + cell temp_time; -struct gc_statistics { - generation_statistics nursery_stats; - generation_statistics aging_stats; - generation_statistics full_stats; - u64 cards_scanned; - u64 decks_scanned; - u64 card_scan_time; - u64 code_blocks_scanned; + explicit gc_event(gc_op op_, factor_vm *parent); + void started_card_scan(); + void ended_card_scan(cell cards_scanned_, cell decks_scanned_); + void started_code_scan(); + void ended_code_scan(cell code_blocks_scanned_); + void started_data_sweep(); + void ended_data_sweep(); + void started_code_sweep(); + void ended_code_sweep(); + void started_compaction(); + void ended_compaction(); + void ended_gc(factor_vm *parent); }; struct gc_state { gc_op op; u64 start_time; jmp_buf gc_unwind; + gc_event *event; - explicit gc_state(gc_op op_); + explicit gc_state(gc_op op_, factor_vm *parent); ~gc_state(); + void start_again(gc_op op_, factor_vm *parent); }; -VM_C_API void inline_gc(cell *gc_roots_base, cell gc_roots_size, factor_vm *parent); +VM_C_API void inline_gc(cell *data_roots_base, cell data_roots_size, factor_vm *parent); } diff --git a/vm/generic_arrays.hpp b/vm/generic_arrays.hpp index 0ba6d11da2..f45785f9ef 100755 --- a/vm/generic_arrays.hpp +++ b/vm/generic_arrays.hpp @@ -1,7 +1,7 @@ namespace factor { -template cell array_capacity(Array *array) +template cell array_capacity(const Array *array) { #ifdef FACTOR_DEBUG assert(array->h.hi_tag() == Array::type_number); @@ -19,7 +19,7 @@ template cell array_size(Array *array) return array_size(array_capacity(array)); } -template Array *factor_vm::allot_array_internal(cell capacity) +template Array *factor_vm::allot_uninitialized_array(cell capacity) { Array *array = allot(array_size(capacity)); array->capacity = tag_fixnum(capacity); @@ -33,7 +33,7 @@ template bool factor_vm::reallot_array_in_place_p(Array *array, template Array *factor_vm::reallot_array(Array *array_, cell capacity) { - gc_root array(array_,this); + data_root array(array_,this); if(reallot_array_in_place_p(array.untagged(),capacity)) { @@ -46,7 +46,7 @@ template Array *factor_vm::reallot_array(Array *array_, cell cap if(capacity < to_copy) to_copy = capacity; - Array *new_array = allot_array_internal(capacity); + Array *new_array = allot_uninitialized_array(capacity); memcpy(new_array + 1,array.untagged() + 1,to_copy * Array::element_size); memset((char *)(new_array + 1) + to_copy * Array::element_size, diff --git a/vm/heap.cpp b/vm/heap.cpp deleted file mode 100644 index 71aac62704..0000000000 --- a/vm/heap.cpp +++ /dev/null @@ -1,235 +0,0 @@ -#include "master.hpp" - -/* This malloc-style heap code is reasonably generic. Maybe in the future, it -will be used for the data heap too, if we ever get mark/sweep/compact GC. */ - -namespace factor -{ - -void heap::clear_free_list() -{ - memset(&free,0,sizeof(heap_free_list)); -} - -heap::heap(bool secure_gc_, cell size, bool executable_p) : secure_gc(secure_gc_) -{ - if(size > (1L << (sizeof(cell) * 8 - 6))) fatal_error("Heap too large",size); - seg = new segment(align_page(size),executable_p); - if(!seg) fatal_error("Out of memory in heap allocator",size); - state = new mark_bits(seg->start,size); - clear_free_list(); -} - -heap::~heap() -{ - delete seg; - seg = NULL; - delete state; - state = NULL; -} - -void heap::add_to_free_list(free_heap_block *block) -{ - if(block->size() < free_list_count * block_size_increment) - { - int index = block->size() / block_size_increment; - block->next_free = free.small_blocks[index]; - free.small_blocks[index] = block; - } - else - { - block->next_free = free.large_blocks; - free.large_blocks = block; - } -} - -/* Called after reading the code heap from the image file, and after code heap -compaction. Makes a free list consisting of one free block, at the very end. */ -void heap::build_free_list(cell size) -{ - clear_free_list(); - free_heap_block *end = (free_heap_block *)(seg->start + size); - end->set_type(FREE_BLOCK_TYPE); - end->set_size(seg->end - (cell)end); - add_to_free_list(end); -} - -void heap::assert_free_block(free_heap_block *block) -{ - if(block->type() != FREE_BLOCK_TYPE) - critical_error("Invalid block in free list",(cell)block); -} - -free_heap_block *heap::find_free_block(cell size) -{ - cell attempt = size; - - while(attempt < free_list_count * block_size_increment) - { - int index = attempt / block_size_increment; - free_heap_block *block = free.small_blocks[index]; - if(block) - { - assert_free_block(block); - free.small_blocks[index] = block->next_free; - return block; - } - - attempt *= 2; - } - - free_heap_block *prev = NULL; - free_heap_block *block = free.large_blocks; - - while(block) - { - assert_free_block(block); - if(block->size() >= size) - { - if(prev) - prev->next_free = block->next_free; - else - free.large_blocks = block->next_free; - return block; - } - - prev = block; - block = block->next_free; - } - - return NULL; -} - -free_heap_block *heap::split_free_block(free_heap_block *block, cell size) -{ - if(block->size() != size ) - { - /* split the block in two */ - free_heap_block *split = (free_heap_block *)((cell)block + size); - split->set_type(FREE_BLOCK_TYPE); - split->set_size(block->size() - size); - split->next_free = block->next_free; - block->set_size(size); - add_to_free_list(split); - } - - return block; -} - -/* Allocate a block of memory from the mark and sweep GC heap */ -heap_block *heap::heap_allot(cell size, cell type) -{ - size = (size + block_size_increment - 1) & ~(block_size_increment - 1); - - free_heap_block *block = find_free_block(size); - if(block) - { - block = split_free_block(block,size); - block->set_type(type); - return block; - } - else - return NULL; -} - -/* Deallocates a block manually */ -void heap::heap_free(heap_block *block) -{ - block->set_type(FREE_BLOCK_TYPE); - add_to_free_list((free_heap_block *)block); -} - -void heap::mark_block(heap_block *block) -{ - state->set_marked_p(block,true); -} - -/* Compute total sum of sizes of free blocks, and size of largest free block */ -void heap::heap_usage(cell *used, cell *total_free, cell *max_free) -{ - *used = 0; - *total_free = 0; - *max_free = 0; - - heap_block *scan = first_block(); - - while(scan) - { - cell size = scan->size(); - - if(scan->type() == FREE_BLOCK_TYPE) - { - *total_free += size; - if(size > *max_free) - *max_free = size; - } - else - *used += size; - - scan = next_block(scan); - } -} - -/* The size of the heap after compaction */ -cell heap::heap_size() -{ - heap_block *scan = first_block(); - - while(scan) - { - if(scan->type() == FREE_BLOCK_TYPE) break; - else scan = next_block(scan); - } - - assert(scan->type() == FREE_BLOCK_TYPE); - assert((cell)scan + scan->size() == seg->end); - - return (cell)scan - (cell)first_block(); -} - -void heap::compact_heap() -{ - forwarding.clear(); - - heap_block *scan = first_block(); - char *address = (char *)scan; - - /* Slide blocks up while building the forwarding hashtable. */ - while(scan) - { - heap_block *next = next_block(scan); - - if(state->is_marked_p(scan)) - { - cell size = scan->size(); - memmove(address,scan,size); - forwarding[scan] = address; - address += size; - } - - scan = next; - } - - /* Now update the free list; there will be a single free block at - the end */ - build_free_list((cell)address - seg->start); -} - -heap_block *heap::free_allocated(heap_block *prev, heap_block *scan) -{ - if(secure_gc) - memset(scan + 1,0,scan->size() - sizeof(heap_block)); - - if(prev && prev->type() == FREE_BLOCK_TYPE) - { - prev->set_size(prev->size() + scan->size()); - return prev; - } - else - { - scan->set_type(FREE_BLOCK_TYPE); - return scan; - } -} - -} diff --git a/vm/heap.hpp b/vm/heap.hpp deleted file mode 100644 index a3c057138b..0000000000 --- a/vm/heap.hpp +++ /dev/null @@ -1,93 +0,0 @@ -namespace factor -{ - -static const cell free_list_count = 32; -static const cell block_size_increment = 16; - -struct heap_free_list { - free_heap_block *small_blocks[free_list_count]; - free_heap_block *large_blocks; -}; - -struct heap { - bool secure_gc; - segment *seg; - heap_free_list free; - mark_bits *state; - unordered_map forwarding; - - explicit heap(bool secure_gc_, cell size, bool executable_p); - ~heap(); - - inline heap_block *next_block(heap_block *block) - { - cell next = ((cell)block + block->size()); - if(next == seg->end) - return NULL; - else - return (heap_block *)next; - } - - inline heap_block *first_block() - { - return (heap_block *)seg->start; - } - - inline heap_block *last_block() - { - return (heap_block *)seg->end; - } - - void clear_free_list(); - void new_heap(cell size); - void add_to_free_list(free_heap_block *block); - void build_free_list(cell size); - void assert_free_block(free_heap_block *block); - free_heap_block *find_free_block(cell size); - free_heap_block *split_free_block(free_heap_block *block, cell size); - heap_block *heap_allot(cell size, cell type); - void heap_free(heap_block *block); - void mark_block(heap_block *block); - void heap_usage(cell *used, cell *total_free, cell *max_free); - cell heap_size(); - void compact_heap(); - - heap_block *free_allocated(heap_block *prev, heap_block *scan); - - /* After code GC, all referenced code blocks have status set to B_MARKED, so any - which are allocated and not marked can be reclaimed. */ - template void free_unmarked(Iterator &iter) - { - clear_free_list(); - - heap_block *prev = NULL; - heap_block *scan = first_block(); - - while(scan) - { - if(scan->type() == FREE_BLOCK_TYPE) - { - if(prev && prev->type() == FREE_BLOCK_TYPE) - prev->set_size(prev->size() + scan->size()); - else - prev = scan; - } - else if(state->is_marked_p(scan)) - { - if(prev && prev->type() == FREE_BLOCK_TYPE) - add_to_free_list((free_heap_block *)prev); - prev = scan; - iter(scan); - } - else - prev = free_allocated(prev,scan); - - scan = next_block(scan); - } - - if(prev && prev->type() == FREE_BLOCK_TYPE) - add_to_free_list((free_heap_block *)prev); - } -}; - -} diff --git a/vm/image.cpp b/vm/image.cpp index c6d1ad7aca..b3a9eae7a5 100755 --- a/vm/image.cpp +++ b/vm/image.cpp @@ -6,7 +6,7 @@ namespace factor /* Certain special objects in the image are known to the runtime */ void factor_vm::init_objects(image_header *h) { - memcpy(userenv,h->userenv,sizeof(userenv)); + memcpy(special_objects,h->special_objects,sizeof(special_objects)); true_object = h->true_object; bignum_zero = h->bignum_zero; @@ -16,31 +16,22 @@ void factor_vm::init_objects(image_header *h) void factor_vm::load_data_heap(FILE *file, image_header *h, vm_parameters *p) { - cell good_size = h->data_size + (1 << 20); - - if(good_size > p->tenured_size) - p->tenured_size = good_size; + p->tenured_size = std::max((h->data_size * 3) / 2,p->tenured_size); init_data_heap(p->young_size, p->aging_size, - p->tenured_size, - p->secure_gc); - - clear_gc_stats(); + p->tenured_size); fixnum bytes_read = fread((void*)data->tenured->start,1,h->data_size,file); if((cell)bytes_read != h->data_size) { - print_string("truncated image: "); - print_fixnum(bytes_read); - print_string(" bytes read, "); - print_cell(h->data_size); - print_string(" bytes expected\n"); + std::cout << "truncated image: " << bytes_read << " bytes read, "; + std::cout << h->data_size << " bytes expected\n"; fatal_error("load_data_heap failed",0); } - data->tenured->here = data->tenured->start + h->data_size; + data->tenured->initial_free_list(h->data_size); } void factor_vm::load_code_heap(FILE *file, image_header *h, vm_parameters *p) @@ -52,19 +43,16 @@ void factor_vm::load_code_heap(FILE *file, image_header *h, vm_parameters *p) if(h->code_size != 0) { - size_t bytes_read = fread(code->first_block(),1,h->code_size,file); + size_t bytes_read = fread(code->allocator->first_block(),1,h->code_size,file); if(bytes_read != h->code_size) { - print_string("truncated image: "); - print_fixnum(bytes_read); - print_string(" bytes read, "); - print_cell(h->code_size); - print_string(" bytes expected\n"); + std::cout << "truncated image: " << bytes_read << " bytes read, "; + std::cout << h->code_size << " bytes expected\n"; fatal_error("load_code_heap failed",0); } } - code->build_free_list(h->code_size); + code->allocator->initial_free_list(h->code_size); } void factor_vm::data_fixup(cell *handle, cell data_relocation_base) @@ -102,9 +90,12 @@ void factor_vm::fixup_quotation(quotation *quot, cell code_relocation_base) quot->xt = (void *)lazy_jit_compile; } -void factor_vm::fixup_alien(alien *d) +void factor_vm::fixup_alien(alien *ptr) { - if(!to_boolean(d->base)) d->expired = true_object; + if(!to_boolean(ptr->base)) + ptr->expired = true_object; + else + ptr->update_address(); } struct stack_frame_fixupper { @@ -155,7 +146,7 @@ void factor_vm::relocate_object(object *object, data_fixup(&t->layout,data_relocation_base); cell *scan = t->data(); - cell *end = (cell *)((cell)object + untagged_object_size(object)); + cell *end = (cell *)((cell)object + object->size()); for(; scan < end; scan++) data_fixup(scan,data_relocation_base); @@ -190,8 +181,8 @@ void factor_vm::relocate_object(object *object, where it is loaded, we need to fix up pointers in the image. */ void factor_vm::relocate_data(cell data_relocation_base, cell code_relocation_base) { - for(cell i = 0; i < USER_ENV; i++) - data_fixup(&userenv[i],data_relocation_base); + for(cell i = 0; i < special_object_count; i++) + data_fixup(&special_objects[i],data_relocation_base); data_fixup(&true_object,data_relocation_base); data_fixup(&bignum_zero,data_relocation_base); @@ -203,8 +194,8 @@ void factor_vm::relocate_data(cell data_relocation_base, cell code_relocation_ba while(obj) { relocate_object((object *)obj,data_relocation_base,code_relocation_base); - data->tenured->record_object_start_offset((object *)obj); - obj = data->tenured->next_object_after(this,obj); + data->tenured->starts.record_object_start_offset((object *)obj); + obj = data->tenured->next_object_after(obj); } } @@ -222,10 +213,10 @@ struct code_block_fixupper { factor_vm *parent; cell data_relocation_base; - code_block_fixupper(factor_vm *parent_, cell data_relocation_base_) : + explicit code_block_fixupper(factor_vm *parent_, cell data_relocation_base_) : parent(parent_), data_relocation_base(data_relocation_base_) { } - void operator()(code_block *compiled) + void operator()(code_block *compiled, cell size) { parent->fixup_code_block(compiled,data_relocation_base); } @@ -244,8 +235,8 @@ void factor_vm::load_image(vm_parameters *p) FILE *file = OPEN_READ(p->image_path); if(file == NULL) { - print_string("Cannot open image file: "); print_native_string(p->image_path); nl(); - print_string(strerror(errno)); nl(); + std::cout << "Cannot open image file: " << p->image_path << std::endl; + std::cout << strerror(errno) << std::endl; exit(1); } @@ -270,7 +261,7 @@ void factor_vm::load_image(vm_parameters *p) relocate_code(h.data_relocation_base); /* Store image path name */ - userenv[IMAGE_ENV] = allot_alien(false_object,(cell)p->image_path); + special_objects[OBJ_IMAGE] = allot_alien(false_object,(cell)p->image_path); } /* Save the current image to disk */ @@ -282,37 +273,35 @@ bool factor_vm::save_image(const vm_char *filename) file = OPEN_WRITE(filename); if(file == NULL) { - print_string("Cannot open image file: "); print_native_string(filename); nl(); - print_string(strerror(errno)); nl(); + std::cout << "Cannot open image file: " << filename << std::endl; + std::cout << strerror(errno) << std::endl; return false; } h.magic = image_magic; h.version = image_version; h.data_relocation_base = data->tenured->start; - h.data_size = data->tenured->here - data->tenured->start; + h.data_size = data->tenured->occupied_space(); h.code_relocation_base = code->seg->start; - h.code_size = code->heap_size(); + h.code_size = code->allocator->occupied_space(); h.true_object = true_object; h.bignum_zero = bignum_zero; h.bignum_pos_one = bignum_pos_one; h.bignum_neg_one = bignum_neg_one; - for(cell i = 0; i < USER_ENV; i++) - h.userenv[i] = (save_env_p(i) ? userenv[i] : false_object); + for(cell i = 0; i < special_object_count; i++) + h.special_objects[i] = (save_env_p(i) ? special_objects[i] : false_object); bool ok = true; if(fwrite(&h,sizeof(image_header),1,file) != 1) ok = false; if(fwrite((void*)data->tenured->start,h.data_size,1,file) != 1) ok = false; - if(fwrite(code->first_block(),h.code_size,1,file) != 1) ok = false; + if(fwrite(code->allocator->first_block(),h.code_size,1,file) != 1) ok = false; if(fclose(file)) ok = false; if(!ok) - { - print_string("save-image failed: "); print_string(strerror(errno)); nl(); - } + std::cout << "save-image failed: " << strerror(errno) << std::endl; return ok; } @@ -322,7 +311,7 @@ void factor_vm::primitive_save_image() /* do a full GC to push everything into tenured space */ primitive_compact_gc(); - gc_root path(dpop(),this); + data_root path(dpop(),this); path.untag_check(this); save_image((vm_char *)(path.untagged() + 1)); } @@ -332,17 +321,16 @@ void factor_vm::primitive_save_image_and_exit() /* We unbox this before doing anything else. This is the only point where we might throw an error, so we have to throw an error here since later steps destroy the current image. */ - gc_root path(dpop(),this); + data_root path(dpop(),this); path.untag_check(this); - /* strip out userenv data which is set on startup anyway */ - for(cell i = 0; i < USER_ENV; i++) - if(!save_env_p(i)) userenv[i] = false_object; + /* strip out special_objects data which is set on startup anyway */ + for(cell i = 0; i < special_object_count; i++) + if(!save_env_p(i)) special_objects[i] = false_object; - gc(collect_full_op, + gc(collect_compact_op, 0, /* requested size */ - false, /* discard objects only reachable from stacks */ - true /* compact the code heap */); + false /* discard objects only reachable from stacks */); /* Save the image */ if(save_image((vm_char *)(path.untagged() + 1))) diff --git a/vm/image.hpp b/vm/image.hpp index 8a7080110c..cca0e9e378 100755 --- a/vm/image.hpp +++ b/vm/image.hpp @@ -25,7 +25,7 @@ struct image_header { /* tagged pointer to bignum -1 */ cell bignum_neg_one; /* Initial user environment */ - cell userenv[USER_ENV]; + cell special_objects[special_object_count]; }; struct vm_parameters { @@ -34,7 +34,6 @@ struct vm_parameters { cell ds_size, rs_size; cell young_size, aging_size, tenured_size; cell code_size; - bool secure_gc; bool fep; bool console; bool signals; diff --git a/vm/inline_cache.cpp b/vm/inline_cache.cpp index f6e756f758..469bb8bf2e 100755 --- a/vm/inline_cache.cpp +++ b/vm/inline_cache.cpp @@ -6,10 +6,6 @@ namespace factor void factor_vm::init_inline_caching(int max_size) { max_pic_size = max_size; - cold_call_to_ic_transitions = 0; - ic_to_pic_transitions = 0; - pic_to_mega_transitions = 0; - for(int i = 0; i < 4; i++) pic_counts[i] = 0; } void factor_vm::deallocate_inline_cache(cell return_address) @@ -19,15 +15,9 @@ void factor_vm::deallocate_inline_cache(cell return_address) check_code_pointer((cell)old_xt); code_block *old_block = (code_block *)old_xt - 1; - cell old_type = old_block->type(); -#ifdef FACTOR_DEBUG - /* The call target was either another PIC, - or a compiled quotation (megamorphic stub) */ - assert(old_type == PIC_TYPE || old_type == QUOTATION_TYPE); -#endif - - if(old_type == PIC_TYPE) + /* Free the old PIC since we know its unreachable */ + if(old_block->pic_p()) code->code_heap_free(old_block); } @@ -35,50 +25,34 @@ void factor_vm::deallocate_inline_cache(cell return_address) it contains */ cell factor_vm::determine_inline_cache_type(array *cache_entries) { - bool seen_hi_tag = false, seen_tuple = false; + bool seen_tuple = false; cell i; for(i = 0; i < array_capacity(cache_entries); i += 2) { - cell klass = array_nth(cache_entries,i); - /* Is it a tuple layout? */ - switch(TAG(klass)) + if(TAG(array_nth(cache_entries,i)) == ARRAY_TYPE) { - case FIXNUM_TYPE: - { - fixnum type = untag_fixnum(klass); - if(type >= HEADER_TYPE) - seen_hi_tag = true; - } - break; - case ARRAY_TYPE: seen_tuple = true; break; - default: - critical_error("Expected a fixnum or array",klass); - break; } } - if(seen_hi_tag && seen_tuple) return PIC_HI_TAG_TUPLE; - if(seen_hi_tag && !seen_tuple) return PIC_HI_TAG; - if(!seen_hi_tag && seen_tuple) return PIC_TUPLE; - if(!seen_hi_tag && !seen_tuple) return PIC_TAG; - - critical_error("Oops",0); - return 0; + return seen_tuple ? PIC_TUPLE : PIC_TAG; } void factor_vm::update_pic_count(cell type) { - pic_counts[type - PIC_TAG]++; + if(type == PIC_TAG) + dispatch_stats.pic_tag_count++; + else + dispatch_stats.pic_tuple_count++; } struct inline_cache_jit : public jit { fixnum index; - explicit inline_cache_jit(cell generic_word_,factor_vm *vm) : jit(PIC_TYPE,generic_word_,vm) {}; + explicit inline_cache_jit(cell generic_word_,factor_vm *vm) : jit(code_block_pic,generic_word_,vm) {}; void emit_check(cell klass); void compile_inline_cache(fixnum index, @@ -91,10 +65,10 @@ struct inline_cache_jit : public jit { void inline_cache_jit::emit_check(cell klass) { cell code_template; - if(TAG(klass) == FIXNUM_TYPE && untag_fixnum(klass) < HEADER_TYPE) - code_template = parent->userenv[PIC_CHECK_TAG]; + if(TAG(klass) == FIXNUM_TYPE) + code_template = parent->special_objects[PIC_CHECK_TAG]; else - code_template = parent->userenv[PIC_CHECK]; + code_template = parent->special_objects[PIC_CHECK_TUPLE]; emit_with(code_template,klass); } @@ -107,9 +81,9 @@ void inline_cache_jit::compile_inline_cache(fixnum index, cell cache_entries_, bool tail_call_p) { - gc_root generic_word(generic_word_,parent); - gc_root methods(methods_,parent); - gc_root cache_entries(cache_entries_,parent); + data_root generic_word(generic_word_,parent); + data_root methods(methods_,parent); + data_root cache_entries(cache_entries_,parent); cell inline_cache_type = parent->determine_inline_cache_type(cache_entries.untagged()); parent->update_pic_count(inline_cache_type); @@ -127,7 +101,7 @@ void inline_cache_jit::compile_inline_cache(fixnum index, /* Yes? Jump to method */ cell method = array_nth(cache_entries.untagged(),i + 1); - emit_with(parent->userenv[PIC_HIT],method); + emit_with(parent->special_objects[PIC_HIT],method); } /* Generate machine code to handle a cache miss, which ultimately results in @@ -139,14 +113,18 @@ void inline_cache_jit::compile_inline_cache(fixnum index, push(methods.value()); push(tag_fixnum(index)); push(cache_entries.value()); - word_special(parent->userenv[tail_call_p ? PIC_MISS_TAIL_WORD : PIC_MISS_WORD]); + word_special(parent->special_objects[tail_call_p ? PIC_MISS_TAIL_WORD : PIC_MISS_WORD]); } -code_block *factor_vm::compile_inline_cache(fixnum index,cell generic_word_,cell methods_,cell cache_entries_,bool tail_call_p) +code_block *factor_vm::compile_inline_cache(fixnum index, + cell generic_word_, + cell methods_, + cell cache_entries_, + bool tail_call_p) { - gc_root generic_word(generic_word_,this); - gc_root methods(methods_,this); - gc_root cache_entries(cache_entries_,this); + data_root generic_word(generic_word_,this); + data_root methods(methods_,this); + data_root cache_entries(cache_entries_,this); inline_cache_jit jit(generic_word.value(),this); jit.compile_inline_cache(index, @@ -159,7 +137,7 @@ code_block *factor_vm::compile_inline_cache(fixnum index,cell generic_word_,cell return code; } -/* A generic word's definition performs general method lookup. Allocates memory */ +/* A generic word's definition performs general method lookup. */ void *factor_vm::megamorphic_call_stub(cell generic_word) { return untag(generic_word)->xt; @@ -173,12 +151,12 @@ cell factor_vm::inline_cache_size(cell cache_entries) /* Allocates memory */ cell factor_vm::add_inline_cache_entry(cell cache_entries_, cell klass_, cell method_) { - gc_root cache_entries(cache_entries_,this); - gc_root klass(klass_,this); - gc_root method(method_,this); + data_root cache_entries(cache_entries_,this); + data_root klass(klass_,this); + data_root method(method_,this); cell pic_size = array_capacity(cache_entries.untagged()); - gc_root new_cache_entries(reallot_array(cache_entries.untagged(),pic_size + 2),this); + data_root new_cache_entries(reallot_array(cache_entries.untagged(),pic_size + 2),this); set_array_nth(new_cache_entries.untagged(),pic_size,klass.value()); set_array_nth(new_cache_entries.untagged(),pic_size + 1,method.value()); return new_cache_entries.value(); @@ -187,29 +165,34 @@ cell factor_vm::add_inline_cache_entry(cell cache_entries_, cell klass_, cell me void factor_vm::update_pic_transitions(cell pic_size) { if(pic_size == max_pic_size) - pic_to_mega_transitions++; + dispatch_stats.pic_to_mega_transitions++; else if(pic_size == 0) - cold_call_to_ic_transitions++; + dispatch_stats.cold_call_to_ic_transitions++; else if(pic_size == 1) - ic_to_pic_transitions++; + dispatch_stats.ic_to_pic_transitions++; } -/* The cache_entries parameter is either f (on cold call site) or an array (on cache miss). -Called from assembly with the actual return address */ -void *factor_vm::inline_cache_miss(cell return_address) +/* The cache_entries parameter is empty (on cold call site) or has entries +(on cache miss). Called from assembly with the actual return address. +Compilation of the inline cache may trigger a GC, which may trigger a compaction; +also, the block containing the return address may now be dead. Use a code_root +to take care of the details. */ +void *factor_vm::inline_cache_miss(cell return_address_) { - check_code_pointer(return_address); + code_root return_address(return_address_,this); + + check_code_pointer(return_address.value); /* Since each PIC is only referenced from a single call site, if the old call target was a PIC, we can deallocate it immediately, instead of leaving dead PICs around until the next GC. */ - deallocate_inline_cache(return_address); + deallocate_inline_cache(return_address.value); - gc_root cache_entries(dpop(),this); + data_root cache_entries(dpop(),this); fixnum index = untag_fixnum(dpop()); - gc_root methods(dpop(),this); - gc_root generic_word(dpop(),this); - gc_root object(((cell *)ds)[-index],this); + data_root methods(dpop(),this); + data_root generic_word(dpop(),this); + data_root object(((cell *)ds)[-index],this); void *xt; @@ -224,7 +207,7 @@ void *factor_vm::inline_cache_miss(cell return_address) cell klass = object_class(object.value()); cell method = lookup_method(object.value(),methods.value()); - gc_root new_cache_entries(add_inline_cache_entry( + data_root new_cache_entries(add_inline_cache_entry( cache_entries.value(), klass, method),this); @@ -232,18 +215,21 @@ void *factor_vm::inline_cache_miss(cell return_address) generic_word.value(), methods.value(), new_cache_entries.value(), - tail_call_site_p(return_address))->xt(); + tail_call_site_p(return_address.value))->xt(); } /* Install the new stub. */ - set_call_target(return_address,xt); + if(return_address.valid) + { + set_call_target(return_address.value,xt); #ifdef PIC_DEBUG - printf("Updated %s call site 0x%lx with 0x%lx\n", - tail_call_site_p(return_address) ? "tail" : "non-tail", - return_address, - (cell)xt); + std::cout << "Updated " + << (tail_call_site_p(return_address) ? "tail" : "non-tail") + << " call site 0x" << std::hex << return_address << std::dec + << " with " << std::hex << (cell)xt << std::dec; #endif + } return xt; } @@ -253,24 +239,4 @@ VM_C_API void *inline_cache_miss(cell return_address, factor_vm *parent) return parent->inline_cache_miss(return_address); } -void factor_vm::primitive_reset_inline_cache_stats() -{ - cold_call_to_ic_transitions = ic_to_pic_transitions = pic_to_mega_transitions = 0; - cell i; - for(i = 0; i < 4; i++) pic_counts[i] = 0; -} - -void factor_vm::primitive_inline_cache_stats() -{ - growable_array stats(this); - stats.add(allot_cell(cold_call_to_ic_transitions)); - stats.add(allot_cell(ic_to_pic_transitions)); - stats.add(allot_cell(pic_to_mega_transitions)); - cell i; - for(i = 0; i < 4; i++) - stats.add(allot_cell(pic_counts[i])); - stats.trim(); - dpush(stats.elements.value()); -} - } diff --git a/vm/io.cpp b/vm/io.cpp index d5cfc1745c..a8f9cb6897 100755 --- a/vm/io.cpp +++ b/vm/io.cpp @@ -16,9 +16,9 @@ normal operation. */ void factor_vm::init_c_io() { - userenv[STDIN_ENV] = allot_alien(false_object,(cell)stdin); - userenv[STDOUT_ENV] = allot_alien(false_object,(cell)stdout); - userenv[STDERR_ENV] = allot_alien(false_object,(cell)stderr); + special_objects[OBJ_STDIN] = allot_alien(false_object,(cell)stdin); + special_objects[OBJ_STDOUT] = allot_alien(false_object,(cell)stdout); + special_objects[OBJ_STDERR] = allot_alien(false_object,(cell)stderr); } void factor_vm::io_error() @@ -33,8 +33,8 @@ void factor_vm::io_error() void factor_vm::primitive_fopen() { - gc_root mode(dpop(),this); - gc_root path(dpop(),this); + data_root mode(dpop(),this); + data_root path(dpop(),this); mode.untag_check(this); path.untag_check(this); @@ -88,7 +88,7 @@ void factor_vm::primitive_fread() return; } - gc_root buf(allot_array_internal(size),this); + data_root buf(allot_uninitialized_array(size),this); for(;;) { diff --git a/vm/jit.cpp b/vm/jit.cpp index ced487e659..e72e88bfdf 100644 --- a/vm/jit.cpp +++ b/vm/jit.cpp @@ -10,7 +10,7 @@ namespace factor - polymorphic inline caches (inline_cache.cpp) */ /* Allocates memory */ -jit::jit(cell type_, cell owner_, factor_vm *vm) +jit::jit(code_block_type type_, cell owner_, factor_vm *vm) : type(type_), owner(owner_,vm), code(vm), @@ -24,7 +24,7 @@ jit::jit(cell type_, cell owner_, factor_vm *vm) void jit::emit_relocation(cell code_template_) { - gc_root code_template(code_template_,parent); + data_root code_template(code_template_,parent); cell capacity = array_capacity(code_template.untagged()); for(cell i = 1; i < capacity; i += 3) { @@ -43,11 +43,11 @@ void jit::emit_relocation(cell code_template_) /* Allocates memory */ void jit::emit(cell code_template_) { - gc_root code_template(code_template_,parent); + data_root code_template(code_template_,parent); emit_relocation(code_template.value()); - gc_root insns(array_nth(code_template.untagged(),0),parent); + data_root insns(array_nth(code_template.untagged(),0),parent); if(computing_offset_p) { @@ -71,16 +71,16 @@ void jit::emit(cell code_template_) } void jit::emit_with(cell code_template_, cell argument_) { - gc_root code_template(code_template_,parent); - gc_root argument(argument_,parent); + data_root code_template(code_template_,parent); + data_root argument(argument_,parent); literal(argument.value()); emit(code_template.value()); } void jit::emit_class_lookup(fixnum index, cell type) { - emit_with(parent->userenv[PIC_LOAD],tag_fixnum(-index * sizeof(cell))); - emit(parent->userenv[type]); + emit_with(parent->special_objects[PIC_LOAD],tag_fixnum(-index * sizeof(cell))); + emit(parent->special_objects[type]); } /* Facility to convert compiled code offsets to quotation offsets. diff --git a/vm/jit.hpp b/vm/jit.hpp index d69f44d05d..b5a2457d57 100644 --- a/vm/jit.hpp +++ b/vm/jit.hpp @@ -2,8 +2,8 @@ namespace factor { struct jit { - cell type; - gc_root owner; + code_block_type type; + data_root owner; growable_byte_array code; growable_byte_array relocation; growable_array literals; @@ -12,7 +12,7 @@ struct jit { cell offset; factor_vm *parent; - explicit jit(cell jit_type, cell owner, factor_vm *vm); + explicit jit(code_block_type type, cell owner, factor_vm *parent); void compute_position(cell offset); void emit_relocation(cell code_template); @@ -21,35 +21,41 @@ struct jit { void literal(cell literal) { literals.add(literal); } void emit_with(cell code_template_, cell literal_); - void push(cell literal) { - emit_with(parent->userenv[JIT_PUSH_IMMEDIATE],literal); + void push(cell literal) + { + emit_with(parent->special_objects[JIT_PUSH_IMMEDIATE],literal); } - void word_jump(cell word_) { - gc_root word(word_,parent); + void word_jump(cell word_) + { + data_root word(word_,parent); literal(tag_fixnum(xt_tail_pic_offset)); literal(word.value()); - emit(parent->userenv[JIT_WORD_JUMP]); + emit(parent->special_objects[JIT_WORD_JUMP]); } - void word_call(cell word) { - emit_with(parent->userenv[JIT_WORD_CALL],word); + void word_call(cell word) + { + emit_with(parent->special_objects[JIT_WORD_CALL],word); } - void word_special(cell word) { - emit_with(parent->userenv[JIT_WORD_SPECIAL],word); + void word_special(cell word) + { + emit_with(parent->special_objects[JIT_WORD_SPECIAL],word); } - void emit_subprimitive(cell word_) { - gc_root word(word_,parent); - gc_root code_pair(word->subprimitive,parent); - literals.append(parent->untag(array_nth(code_pair.untagged(),0))); + void emit_subprimitive(cell word_) + { + data_root word(word_,parent); + data_root code_pair(word->subprimitive,parent); + literals.append(untag(array_nth(code_pair.untagged(),0))); emit(array_nth(code_pair.untagged(),1)); } void emit_class_lookup(fixnum index, cell type); - fixnum get_position() { + fixnum get_position() + { if(computing_offset_p) { /* If this is still on, emit() didn't clear it, @@ -60,7 +66,8 @@ struct jit { return position; } - void set_position(fixnum position_) { + void set_position(fixnum position_) + { if(computing_offset_p) position = position_; } diff --git a/vm/layouts.hpp b/vm/layouts.hpp index 34dbe163f9..5f3f3e9310 100644 --- a/vm/layouts.hpp +++ b/vm/layouts.hpp @@ -23,47 +23,43 @@ inline static cell align(cell a, cell b) return (a + (b-1)) & ~(b-1); } -inline static cell align8(cell a) -{ - return align(a,8); -} +static const cell data_alignment = 16; #define WORD_SIZE (signed)(sizeof(cell)*8) -#define TAG_MASK 7 -#define TAG_BITS 3 +#define TAG_MASK 15 +#define TAG_BITS 4 #define TAG(x) ((cell)(x) & TAG_MASK) #define UNTAG(x) ((cell)(x) & ~TAG_MASK) #define RETAG(x,tag) (UNTAG(x) | (tag)) /*** Tags ***/ #define FIXNUM_TYPE 0 -#define BIGNUM_TYPE 1 +#define F_TYPE 1 #define ARRAY_TYPE 2 #define FLOAT_TYPE 3 #define QUOTATION_TYPE 4 -#define F_TYPE 5 -#define OBJECT_TYPE 6 +#define BIGNUM_TYPE 5 +#define ALIEN_TYPE 6 #define TUPLE_TYPE 7 - -#define HEADER_TYPE 8 /* anything less than this is a tag */ - -#define GC_COLLECTED 5 /* can be anything other than FIXNUM_TYPE */ - -/*** Header types ***/ #define WRAPPER_TYPE 8 #define BYTE_ARRAY_TYPE 9 #define CALLSTACK_TYPE 10 #define STRING_TYPE 11 #define WORD_TYPE 12 #define DLL_TYPE 13 -#define ALIEN_TYPE 14 -#define TYPE_COUNT 15 +#define TYPE_COUNT 14 -/* Not real types, but code_block's type can be set to this */ -#define PIC_TYPE 16 -#define FREE_BLOCK_TYPE 17 +#define FORWARDING_POINTER 5 /* can be anything other than FIXNUM_TYPE */ + +enum code_block_type +{ + code_block_unoptimized, + code_block_optimized, + code_block_profiling, + code_block_pic +}; /* Constants used when floating-point trap exceptions are thrown */ enum @@ -80,7 +76,8 @@ static const cell false_object = F_TYPE; inline static bool immediate_p(cell obj) { - return (obj == false_object || TAG(obj) == FIXNUM_TYPE); + /* We assume that fixnums have tag 0 and false_object has tag 1 */ + return TAG(obj) <= F_TYPE; } inline static fixnum untag_fixnum(cell tagged) @@ -96,11 +93,6 @@ inline static cell tag_fixnum(fixnum untagged) return RETAG(untagged << TAG_BITS,FIXNUM_TYPE); } -inline static cell tag_for(cell type) -{ - return type < HEADER_TYPE ? type : OBJECT_TYPE; -} - struct object; struct header { @@ -111,27 +103,32 @@ struct header { explicit header(cell value_) : value(value_ << TAG_BITS) {} - void check_header() { + void check_header() const + { #ifdef FACTOR_DEBUG assert(TAG(value) == FIXNUM_TYPE && untag_fixnum(value) < TYPE_COUNT); #endif } - cell hi_tag() { + cell hi_tag() const + { check_header(); return value >> TAG_BITS; } - bool forwarding_pointer_p() { - return TAG(value) == GC_COLLECTED; + bool forwarding_pointer_p() const + { + return TAG(value) == FORWARDING_POINTER; } - object *forwarding_pointer() { + object *forwarding_pointer() const + { return (object *)UNTAG(value); } - void forward_to(object *pointer) { - value = RETAG(pointer,GC_COLLECTED); + void forward_to(object *pointer) + { + value = RETAG(pointer,FORWARDING_POINTER); } }; @@ -140,7 +137,18 @@ struct header { struct object { NO_TYPE_CHECK; header h; - cell *slots() { return (cell *)this; } + + cell size() const; + cell binary_payload_start() const; + + cell *slots() const { return (cell *)this; } + + /* Only valid for objects in tenured space; must fast to free_heap_block + to do anything with it if its free */ + bool free_p() const + { + return h.value & 1 == 1; + } }; /* Assembly code makes assumptions about the layout of this struct */ @@ -150,7 +158,7 @@ struct array : public object { /* tagged */ cell capacity; - cell *data() { return (cell *)(this + 1); } + cell *data() const { return (cell *)(this + 1); } }; /* These are really just arrays, but certain elements have special @@ -171,7 +179,7 @@ struct bignum : public object { /* tagged */ cell capacity; - cell *data() { return (cell *)(this + 1); } + cell *data() const { return (cell *)(this + 1); } }; struct byte_array : public object { @@ -180,7 +188,12 @@ struct byte_array : public object { /* tagged */ cell capacity; - template Scalar *data() { return (Scalar *)(this + 1); } +#ifndef FACTOR_64 + cell padding0; + cell padding1; +#endif + + template Scalar *data() const { return (Scalar *)(this + 1); } }; /* Assembly code makes assumptions about the layout of this struct */ @@ -193,39 +206,53 @@ struct string : public object { /* tagged */ cell hashcode; - u8 *data() { return (u8 *)(this + 1); } + u8 *data() const { return (u8 *)(this + 1); } + + cell nth(cell i) const; }; /* The compiled code heap is structured into blocks. */ -struct heap_block +struct code_block { cell header; - - cell type() { return (header >> 1) & 0x1f; } - void set_type(cell type) - { - header = ((header & ~(0x1f << 1)) | (type << 1)); - } - - cell size() { return (header >> 6); } - void set_size(cell size) - { - header = (header & 0x2f) | (size << 6); - } -}; - -struct free_heap_block : public heap_block -{ - free_heap_block *next_free; -}; - -struct code_block : public heap_block -{ cell owner; /* tagged pointer to word, quotation or f */ cell literals; /* tagged pointer to array or f */ cell relocation; /* tagged pointer to byte-array or f */ - void *xt() { return (void *)(this + 1); } + bool free_p() const + { + return header & 1 == 1; + } + + code_block_type type() const + { + return (code_block_type)((header >> 1) & 0x3); + } + + void set_type(code_block_type type) + { + header = ((header & ~0x7) | (type << 1)); + } + + bool pic_p() const + { + return type() == code_block_pic; + } + + bool optimized_p() const + { + return type() == code_block_optimized; + } + + cell size() const + { + return header >> 3; + } + + void *xt() const + { + return (void *)(this + 1); + } }; /* Assembly code makes assumptions about the layout of this struct */ @@ -298,6 +325,16 @@ struct alien : public object { cell expired; /* untagged */ cell displacement; + /* untagged */ + cell address; + + void update_address() + { + if(base == false_object) + address = displacement; + else + address = UNTAG(base) + sizeof(byte_array) + displacement; + } }; struct dll : public object { @@ -308,8 +345,7 @@ struct dll : public object { void *dll; }; -struct stack_frame -{ +struct stack_frame { void *xt; /* Frame size in bytes */ cell size; @@ -320,13 +356,13 @@ struct callstack : public object { /* tagged */ cell length; - stack_frame *frame_at(cell offset) + stack_frame *frame_at(cell offset) const { return (stack_frame *)((char *)(this + 1) + offset); } - stack_frame *top() { return (stack_frame *)(this + 1); } - stack_frame *bottom() { return (stack_frame *)((cell)(this + 1) + untag_fixnum(length)); } + stack_frame *top() const { return (stack_frame *)(this + 1); } + stack_frame *bottom() const { return (stack_frame *)((cell)(this + 1) + untag_fixnum(length)); } }; struct tuple : public object { @@ -334,7 +370,7 @@ struct tuple : public object { /* tagged layout */ cell layout; - cell *data() { return (cell *)(this + 1); } + cell *data() const { return (cell *)(this + 1); } }; } diff --git a/vm/local_roots.hpp b/vm/local_roots.hpp deleted file mode 100644 index 6ae059f4c4..0000000000 --- a/vm/local_roots.hpp +++ /dev/null @@ -1,46 +0,0 @@ -namespace factor -{ - -template -struct gc_root : public tagged -{ - factor_vm *parent; - - void push() { parent->check_tagged_pointer(tagged::value()); parent->gc_locals.push_back((cell)this); } - - explicit gc_root(cell value_,factor_vm *vm) : tagged(value_),parent(vm) { push(); } - explicit gc_root(Type *value_, factor_vm *vm) : tagged(value_),parent(vm) { push(); } - - const gc_root& operator=(const Type *x) { tagged::operator=(x); return *this; } - const gc_root& operator=(const cell &x) { tagged::operator=(x); return *this; } - - ~gc_root() { -#ifdef FACTOR_DEBUG - assert(parent->gc_locals.back() == (cell)this); -#endif - parent->gc_locals.pop_back(); - } -}; - -/* A similar hack for the bignum implementation */ -struct gc_bignum -{ - bignum **addr; - factor_vm *parent; - gc_bignum(bignum **addr_, factor_vm *vm) : addr(addr_), parent(vm) { - if(*addr_) - parent->check_data_pointer(*addr_); - parent->gc_bignums.push_back((cell)addr); - } - - ~gc_bignum() { -#ifdef FACTOR_DEBUG - assert(parent->gc_bignums.back() == (cell)addr); -#endif - parent->gc_bignums.pop_back(); - } -}; - -#define GC_BIGNUM(x) gc_bignum x##__gc_root(&x,this) - -} diff --git a/vm/mach_signal.cpp b/vm/mach_signal.cpp index d05942ff7e..3fa7dcbf07 100644 --- a/vm/mach_signal.cpp +++ b/vm/mach_signal.cpp @@ -84,7 +84,7 @@ static void call_fault_handler( { THREADHANDLE thread_id = pthread_from_mach_thread_np(thread); assert(thread_id); - unordered_map::const_iterator vm = thread_vms.find(thread_id); + std::map::const_iterator vm = thread_vms.find(thread_id); if (vm != thread_vms.end()) vm->second->call_fault_handler(exception,code,exc_state,thread_state,float_state); } diff --git a/vm/mark_bits.hpp b/vm/mark_bits.hpp index 7945be1850..b54a2c9d46 100644 --- a/vm/mark_bits.hpp +++ b/vm/mark_bits.hpp @@ -1,43 +1,34 @@ namespace factor { -const int forwarding_granularity = 128; +const int block_granularity = 16; +const int forwarding_granularity = 64; -template struct mark_bits { - cell start; +template struct mark_bits { cell size; + cell start; cell bits_size; - unsigned int *marked; - unsigned int *freed; - cell forwarding_size; + u64 *marked; cell *forwarding; void clear_mark_bits() { - memset(marked,0,bits_size * sizeof(unsigned int)); - } - - void clear_free_bits() - { - memset(freed,0,bits_size * sizeof(unsigned int)); + memset(marked,0,bits_size * sizeof(u64)); } void clear_forwarding() { - memset(forwarding,0,forwarding_size * sizeof(cell)); + memset(forwarding,0,bits_size * sizeof(cell)); } - explicit mark_bits(cell start_, cell size_) : - start(start_), + explicit mark_bits(cell size_, cell start_) : size(size_), - bits_size(size / Granularity / 32), - marked(new unsigned int[bits_size]), - freed(new unsigned int[bits_size]), - forwarding_size(size / Granularity / forwarding_granularity), - forwarding(new cell[forwarding_size]) + start(start_), + bits_size(size / block_granularity / forwarding_granularity), + marked(new u64[bits_size]), + forwarding(new cell[bits_size]) { clear_mark_bits(); - clear_free_bits(); clear_forwarding(); } @@ -45,58 +36,168 @@ template struct mark_bits { { delete[] marked; marked = NULL; - delete[] freed; - freed = NULL; delete[] forwarding; forwarding = NULL; } + cell block_line(Block *address) + { + return (((cell)address - start) / block_granularity); + } + + Block *line_block(cell line) + { + return (Block *)(line * block_granularity + start); + } + std::pair bitmap_deref(Block *address) { - cell word_number = (((cell)address - start) / Granularity); - cell word_index = (word_number >> 5); - cell word_shift = (word_number & 31); - -#ifdef FACTOR_DEBUG - assert(word_index < bits_size); -#endif - + cell line_number = block_line(address); + cell word_index = (line_number >> 6); + cell word_shift = (line_number & 63); return std::make_pair(word_index,word_shift); } - bool bitmap_elt(unsigned int *bits, Block *address) + bool bitmap_elt(u64 *bits, Block *address) { - std::pair pair = bitmap_deref(address); - return (bits[pair.first] & (1 << pair.second)) != 0; + std::pair position = bitmap_deref(address); + return (bits[position.first] & ((u64)1 << position.second)) != 0; } - void set_bitmap_elt(unsigned int *bits, Block *address, bool flag) + Block *next_block_after(Block *block) { - std::pair pair = bitmap_deref(address); - if(flag) - bits[pair.first] |= (1 << pair.second); + return (Block *)((cell)block + block->size()); + } + + void set_bitmap_range(u64 *bits, Block *address) + { + std::pair start = bitmap_deref(address); + std::pair end = bitmap_deref(next_block_after(address)); + + u64 start_mask = ((u64)1 << start.second) - 1; + u64 end_mask = ((u64)1 << end.second) - 1; + + if(start.first == end.first) + bits[start.first] |= start_mask ^ end_mask; else - bits[pair.first] &= ~(1 << pair.second); + { +#ifdef FACTOR_DEBUG + assert(start.first < bits_size); +#endif + bits[start.first] |= ~start_mask; + + for(cell index = start.first + 1; index < end.first; index++) + bits[index] = (u64)-1; + + if(end_mask != 0) + { +#ifdef FACTOR_DEBUG + assert(end.first < bits_size); +#endif + bits[end.first] |= end_mask; + } + } } - bool is_marked_p(Block *address) + bool marked_p(Block *address) { return bitmap_elt(marked,address); } - void set_marked_p(Block *address, bool marked_p) + void set_marked_p(Block *address) { - set_bitmap_elt(marked,address,marked_p); + set_bitmap_range(marked,address); } - bool is_free_p(Block *address) + /* The eventual destination of a block after compaction is just the number + of marked blocks before it. Live blocks must be marked on entry. */ + void compute_forwarding() { - return bitmap_elt(freed,address); + cell accum = 0; + for(cell index = 0; index < bits_size; index++) + { + forwarding[index] = accum; + accum += popcount(marked[index]); + } } - void set_free_p(Block *address, bool free_p) + /* We have the popcount for every 64 entries; look up and compute the rest */ + Block *forward_block(Block *original) { - set_bitmap_elt(freed,address,free_p); +#ifdef FACTOR_DEBUG + assert(marked_p(original)); +#endif + std::pair position = bitmap_deref(original); + + cell approx_popcount = forwarding[position.first]; + u64 mask = ((u64)1 << position.second) - 1; + + cell new_line_number = approx_popcount + popcount(marked[position.first] & mask); + Block *new_block = line_block(new_line_number); +#ifdef FACTOR_DEBUG + assert(new_block <= original); +#endif + return new_block; + } + + Block *next_unmarked_block_after(Block *original) + { + std::pair position = bitmap_deref(original); + cell bit_index = position.second; + + for(cell index = position.first; index < bits_size; index++) + { + u64 mask = ((s64)marked[index] >> bit_index); + if(~mask) + { + /* Found an unmarked block on this page. + Stop, it's hammer time */ + cell clear_bit = rightmost_clear_bit(mask); + return line_block(index * 64 + bit_index + clear_bit); + } + else + { + /* No unmarked blocks on this page. + Keep looking */ + bit_index = 0; + } + } + + /* No unmarked blocks were found */ + return (Block *)(this->start + this->size); + } + + Block *next_marked_block_after(Block *original) + { + std::pair position = bitmap_deref(original); + cell bit_index = position.second; + + for(cell index = position.first; index < bits_size; index++) + { + u64 mask = (marked[index] >> bit_index); + if(mask) + { + /* Found an marked block on this page. + Stop, it's hammer time */ + cell set_bit = rightmost_set_bit(mask); + return line_block(index * 64 + bit_index + set_bit); + } + else + { + /* No marked blocks on this page. + Keep looking */ + bit_index = 0; + } + } + + /* No marked blocks were found */ + return (Block *)(this->start + this->size); + } + + cell unmarked_block_size(Block *original) + { + Block *next_marked = next_marked_block_after(original); + return ((char *)next_marked - (char *)original); } }; diff --git a/vm/master.hpp b/vm/master.hpp index 7c841d94f6..39242a36af 100755 --- a/vm/master.hpp +++ b/vm/master.hpp @@ -25,26 +25,10 @@ /* C++ headers */ #include +#include #include #include - -#if __GNUC__ == 4 - #include - - namespace factor - { - using std::tr1::unordered_map; - } -#elif __GNUC__ == 3 - #include - - namespace factor - { - using boost::unordered_map; - } -#else - #error Factor requires GCC 3.x or later -#endif +#include /* Forward-declare this since it comes up in function prototypes */ namespace factor @@ -65,12 +49,18 @@ namespace factor #include "bignumint.hpp" #include "bignum.hpp" #include "code_block.hpp" -#include "zone.hpp" +#include "bump_allocator.hpp" +#include "bitwise_hacks.hpp" +#include "mark_bits.hpp" +#include "free_list.hpp" +#include "free_list_allocator.hpp" #include "write_barrier.hpp" -#include "old_space.hpp" +#include "object_start_map.hpp" +#include "nursery_space.hpp" #include "aging_space.hpp" #include "tenured_space.hpp" #include "data_heap.hpp" +#include "code_heap.hpp" #include "gc.hpp" #include "debug.hpp" #include "strings.hpp" @@ -78,21 +68,23 @@ namespace factor #include "words.hpp" #include "float_bits.hpp" #include "io.hpp" -#include "mark_bits.hpp" -#include "heap.hpp" #include "image.hpp" #include "alien.hpp" -#include "code_heap.hpp" #include "callbacks.hpp" +#include "dispatch.hpp" #include "vm.hpp" #include "allot.hpp" #include "tagged.hpp" -#include "local_roots.hpp" +#include "data_roots.hpp" +#include "code_roots.hpp" +#include "slot_visitor.hpp" #include "collector.hpp" #include "copying_collector.hpp" #include "nursery_collector.hpp" #include "aging_collector.hpp" #include "to_tenured_collector.hpp" +#include "code_block_visitor.hpp" +#include "compaction.hpp" #include "full_collector.hpp" #include "callstack.hpp" #include "generic_arrays.hpp" @@ -102,7 +94,6 @@ namespace factor #include "byte_arrays.hpp" #include "jit.hpp" #include "quotations.hpp" -#include "dispatch.hpp" #include "inline_cache.hpp" #include "factor.hpp" #include "utilities.hpp" diff --git a/vm/math.cpp b/vm/math.cpp index d5d4e89837..4266edc09c 100755 --- a/vm/math.cpp +++ b/vm/math.cpp @@ -379,7 +379,7 @@ fixnum factor_vm::to_fixnum(cell tagged) } } -VM_C_API fixnum to_fixnum(cell tagged,factor_vm *parent) +VM_C_API fixnum to_fixnum(cell tagged, factor_vm *parent) { return parent->to_fixnum(tagged); } @@ -399,7 +399,7 @@ void factor_vm::box_signed_1(s8 n) dpush(tag_fixnum(n)); } -VM_C_API void box_signed_1(s8 n,factor_vm *parent) +VM_C_API void box_signed_1(s8 n, factor_vm *parent) { return parent->box_signed_1(n); } @@ -409,7 +409,7 @@ void factor_vm::box_unsigned_1(u8 n) dpush(tag_fixnum(n)); } -VM_C_API void box_unsigned_1(u8 n,factor_vm *parent) +VM_C_API void box_unsigned_1(u8 n, factor_vm *parent) { return parent->box_unsigned_1(n); } @@ -419,7 +419,7 @@ void factor_vm::box_signed_2(s16 n) dpush(tag_fixnum(n)); } -VM_C_API void box_signed_2(s16 n,factor_vm *parent) +VM_C_API void box_signed_2(s16 n, factor_vm *parent) { return parent->box_signed_2(n); } @@ -429,7 +429,7 @@ void factor_vm::box_unsigned_2(u16 n) dpush(tag_fixnum(n)); } -VM_C_API void box_unsigned_2(u16 n,factor_vm *parent) +VM_C_API void box_unsigned_2(u16 n, factor_vm *parent) { return parent->box_unsigned_2(n); } @@ -439,7 +439,7 @@ void factor_vm::box_signed_4(s32 n) dpush(allot_integer(n)); } -VM_C_API void box_signed_4(s32 n,factor_vm *parent) +VM_C_API void box_signed_4(s32 n, factor_vm *parent) { return parent->box_signed_4(n); } @@ -449,7 +449,7 @@ void factor_vm::box_unsigned_4(u32 n) dpush(allot_cell(n)); } -VM_C_API void box_unsigned_4(u32 n,factor_vm *parent) +VM_C_API void box_unsigned_4(u32 n, factor_vm *parent) { return parent->box_unsigned_4(n); } @@ -459,7 +459,7 @@ void factor_vm::box_signed_cell(fixnum integer) dpush(allot_integer(integer)); } -VM_C_API void box_signed_cell(fixnum integer,factor_vm *parent) +VM_C_API void box_signed_cell(fixnum integer, factor_vm *parent) { return parent->box_signed_cell(integer); } @@ -469,7 +469,7 @@ void factor_vm::box_unsigned_cell(cell cell) dpush(allot_cell(cell)); } -VM_C_API void box_unsigned_cell(cell cell,factor_vm *parent) +VM_C_API void box_unsigned_cell(cell cell, factor_vm *parent) { return parent->box_unsigned_cell(cell); } @@ -482,7 +482,7 @@ void factor_vm::box_signed_8(s64 n) dpush(tag_fixnum(n)); } -VM_C_API void box_signed_8(s64 n,factor_vm *parent) +VM_C_API void box_signed_8(s64 n, factor_vm *parent) { return parent->box_signed_8(n); } @@ -501,7 +501,7 @@ s64 factor_vm::to_signed_8(cell obj) } } -VM_C_API s64 to_signed_8(cell obj,factor_vm *parent) +VM_C_API s64 to_signed_8(cell obj, factor_vm *parent) { return parent->to_signed_8(obj); } @@ -514,7 +514,7 @@ void factor_vm::box_unsigned_8(u64 n) dpush(tag_fixnum(n)); } -VM_C_API void box_unsigned_8(u64 n,factor_vm *parent) +VM_C_API void box_unsigned_8(u64 n, factor_vm *parent) { return parent->box_unsigned_8(n); } @@ -533,7 +533,7 @@ u64 factor_vm::to_unsigned_8(cell obj) } } -VM_C_API u64 to_unsigned_8(cell obj,factor_vm *parent) +VM_C_API u64 to_unsigned_8(cell obj, factor_vm *parent) { return parent->to_unsigned_8(obj); } @@ -553,7 +553,7 @@ float factor_vm::to_float(cell value) return untag_float_check(value); } -VM_C_API float to_float(cell value,factor_vm *parent) +VM_C_API float to_float(cell value, factor_vm *parent) { return parent->to_float(value); } @@ -563,7 +563,7 @@ void factor_vm::box_double(double flo) dpush(allot_float(flo)); } -VM_C_API void box_double(double flo,factor_vm *parent) +VM_C_API void box_double(double flo, factor_vm *parent) { return parent->box_double(flo); } @@ -573,7 +573,7 @@ double factor_vm::to_double(cell value) return untag_float_check(value); } -VM_C_API double to_double(cell value,factor_vm *parent) +VM_C_API double to_double(cell value, factor_vm *parent) { return parent->to_double(value); } diff --git a/vm/nursery_collector.cpp b/vm/nursery_collector.cpp index 909cde02f8..155da243d4 100644 --- a/vm/nursery_collector.cpp +++ b/vm/nursery_collector.cpp @@ -6,7 +6,6 @@ namespace factor nursery_collector::nursery_collector(factor_vm *parent_) : copying_collector( parent_, - &parent_->gc_stats.nursery_stats, parent_->data->aging, nursery_policy(parent_)) {} @@ -18,17 +17,27 @@ void factor_vm::collect_nursery() collector.trace_roots(); collector.trace_contexts(); + + current_gc->event->started_card_scan(); collector.trace_cards(data->tenured, card_points_to_nursery, simple_unmarker(card_points_to_nursery)); collector.trace_cards(data->aging, card_points_to_nursery, - simple_unmarker(card_mark_mask)); - collector.trace_code_heap_roots(&code->points_to_nursery); - collector.cheneys_algorithm(); - update_code_heap_for_minor_gc(&code->points_to_nursery); + full_unmarker()); + current_gc->event->ended_card_scan(collector.cards_scanned,collector.decks_scanned); - nursery.here = nursery.start; + current_gc->event->started_code_scan(); + collector.trace_code_heap_roots(&code->points_to_nursery); + current_gc->event->ended_code_scan(collector.code_blocks_scanned); + + collector.cheneys_algorithm(); + + current_gc->event->started_code_sweep(); + update_code_heap_for_minor_gc(&code->points_to_nursery); + current_gc->event->ended_code_sweep(); + + data->reset_generation(&nursery); code->points_to_nursery.clear(); } diff --git a/vm/nursery_collector.hpp b/vm/nursery_collector.hpp index f9d2172929..de9b38d283 100644 --- a/vm/nursery_collector.hpp +++ b/vm/nursery_collector.hpp @@ -4,16 +4,20 @@ namespace factor struct nursery_policy { factor_vm *parent; - nursery_policy(factor_vm *parent_) : parent(parent_) {} + explicit nursery_policy(factor_vm *parent_) : parent(parent_) {} - bool should_copy_p(object *untagged) + bool should_copy_p(object *obj) { - return parent->nursery.contains_p(untagged); + return parent->nursery.contains_p(obj); } + + void promoted_object(object *obj) {} + + void visited_object(object *obj) {} }; struct nursery_collector : copying_collector { - nursery_collector(factor_vm *parent_); + explicit nursery_collector(factor_vm *parent_); }; } diff --git a/vm/nursery_space.hpp b/vm/nursery_space.hpp new file mode 100644 index 0000000000..c44d2a8e44 --- /dev/null +++ b/vm/nursery_space.hpp @@ -0,0 +1,9 @@ +namespace factor +{ + +struct nursery_space : bump_allocator +{ + explicit nursery_space(cell size, cell start) : bump_allocator(size,start) {} +}; + +} diff --git a/vm/object_start_map.cpp b/vm/object_start_map.cpp new file mode 100644 index 0000000000..724f365e79 --- /dev/null +++ b/vm/object_start_map.cpp @@ -0,0 +1,90 @@ +#include "master.hpp" + +namespace factor +{ + +object_start_map::object_start_map(cell size_, cell start_) : + size(size_), start(start_) +{ + object_start_offsets = new card[addr_to_card(size_)]; + object_start_offsets_end = object_start_offsets + addr_to_card(size_); + clear_object_start_offsets(); +} + +object_start_map::~object_start_map() +{ + delete[] object_start_offsets; +} + +cell object_start_map::first_object_in_card(cell card_index) +{ + return object_start_offsets[card_index]; +} + +cell object_start_map::find_object_containing_card(cell card_index) +{ + if(card_index == 0) + return start; + else + { + card_index--; + + while(first_object_in_card(card_index) == card_starts_inside_object) + { +#ifdef FACTOR_DEBUG + /* First card should start with an object */ + assert(card_index > 0); +#endif + card_index--; + } + + return start + (card_index << card_bits) + first_object_in_card(card_index); + } +} + +/* we need to remember the first object allocated in the card */ +void object_start_map::record_object_start_offset(object *obj) +{ + cell idx = addr_to_card((cell)obj - start); + card obj_start = ((cell)obj & addr_card_mask); + object_start_offsets[idx] = std::min(object_start_offsets[idx],obj_start); +} + +void object_start_map::clear_object_start_offsets() +{ + memset(object_start_offsets,card_starts_inside_object,addr_to_card(size)); +} + +void object_start_map::update_card_for_sweep(cell index, u16 mask) +{ + cell offset = object_start_offsets[index]; + if(offset != card_starts_inside_object) + { + mask >>= (offset / block_granularity); + + if(mask == 0) + { + /* The rest of the block after the old object start is free */ + object_start_offsets[index] = card_starts_inside_object; + } + else + { + /* Move the object start forward if necessary */ + object_start_offsets[index] = offset + (rightmost_set_bit(mask) * block_granularity); + } + } +} + +void object_start_map::update_for_sweep(mark_bits *state) +{ + for(cell index = 0; index < state->bits_size; index++) + { + u64 mask = state->marked[index]; + update_card_for_sweep(index * 4, mask & 0xffff); + update_card_for_sweep(index * 4 + 1, (mask >> 16) & 0xffff); + update_card_for_sweep(index * 4 + 2, (mask >> 32) & 0xffff); + update_card_for_sweep(index * 4 + 3, (mask >> 48) & 0xffff); + } +} + +} diff --git a/vm/old_space.hpp b/vm/object_start_map.hpp similarity index 59% rename from vm/old_space.hpp rename to vm/object_start_map.hpp index d037a039ae..a2e24eeda6 100644 --- a/vm/old_space.hpp +++ b/vm/object_start_map.hpp @@ -3,19 +3,20 @@ namespace factor static const cell card_starts_inside_object = 0xff; -struct old_space : zone { +struct object_start_map { + cell size, start; card *object_start_offsets; card *object_start_offsets_end; - old_space(cell size_, cell start_); - ~old_space(); + explicit object_start_map(cell size_, cell start_); + ~object_start_map(); cell first_object_in_card(cell card_index); cell find_object_containing_card(cell card_index); void record_object_start_offset(object *obj); - object *allot(cell size); void clear_object_start_offsets(); - cell next_object_after(factor_vm *parent, cell scan); + void update_card_for_sweep(cell index, u16 mask); + void update_for_sweep(mark_bits *state); }; } diff --git a/vm/old_space.cpp b/vm/old_space.cpp deleted file mode 100644 index 5fd78a7cf4..0000000000 --- a/vm/old_space.cpp +++ /dev/null @@ -1,74 +0,0 @@ -#include "master.hpp" - -namespace factor -{ - -old_space::old_space(cell size_, cell start_) : zone(size_,start_) -{ - object_start_offsets = new card[addr_to_card(size_)]; - object_start_offsets_end = object_start_offsets + addr_to_card(size_); -} - -old_space::~old_space() -{ - delete[] object_start_offsets; -} - -cell old_space::first_object_in_card(cell card_index) -{ - return object_start_offsets[card_index]; -} - -cell old_space::find_object_containing_card(cell card_index) -{ - if(card_index == 0) - return start; - else - { - card_index--; - - while(first_object_in_card(card_index) == card_starts_inside_object) - { -#ifdef FACTOR_DEBUG - /* First card should start with an object */ - assert(card_index > 0); -#endif - card_index--; - } - - return start + (card_index << card_bits) + first_object_in_card(card_index); - } -} - -/* we need to remember the first object allocated in the card */ -void old_space::record_object_start_offset(object *obj) -{ - cell idx = addr_to_card((cell)obj - start); - if(object_start_offsets[idx] == card_starts_inside_object) - object_start_offsets[idx] = ((cell)obj & addr_card_mask); -} - -object *old_space::allot(cell size) -{ - if(here + size > end) return NULL; - - object *obj = zone::allot(size); - record_object_start_offset(obj); - return obj; -} - -void old_space::clear_object_start_offsets() -{ - memset(object_start_offsets,card_starts_inside_object,addr_to_card(size)); -} - -cell old_space::next_object_after(factor_vm *parent, cell scan) -{ - cell size = parent->untagged_object_size((object *)scan); - if(scan + size < here) - return scan + size; - else - return 0; -} - -} diff --git a/vm/os-macosx.mm b/vm/os-macosx.mm index 96f169bbcf..438957bd04 100644 --- a/vm/os-macosx.mm +++ b/vm/os-macosx.mm @@ -14,7 +14,7 @@ NS_DURING NS_VOIDRETURN; NS_HANDLER dpush(allot_alien(false_object,(cell)localException)); - quot = userenv[COCOA_EXCEPTION_ENV]; + quot = special_objects[OBJ_COCOA_EXCEPTION]; if(!tagged(quot).type_p(QUOTATION_TYPE)) { /* No Cocoa exception handler was registered, so diff --git a/vm/os-windows.hpp b/vm/os-windows.hpp index b12ebd0610..403842b2cb 100644 --- a/vm/os-windows.hpp +++ b/vm/os-windows.hpp @@ -37,8 +37,6 @@ typedef wchar_t vm_char; #define OPEN_READ(path) _wfopen(path,L"rb") #define OPEN_WRITE(path) _wfopen(path,L"wb") -#define print_native_string(string) wprintf(L"%s",string) - /* Difference between Jan 1 00:00:00 1601 and Jan 1 00:00:00 1970 */ #define EPOCH_OFFSET 0x019db1ded53e8000LL diff --git a/vm/primitives.cpp b/vm/primitives.cpp index ea02545148..957e6128ed 100644 --- a/vm/primitives.cpp +++ b/vm/primitives.cpp @@ -55,7 +55,6 @@ PRIMITIVE_FORWARD(existsp) PRIMITIVE_FORWARD(minor_gc) PRIMITIVE_FORWARD(full_gc) PRIMITIVE_FORWARD(compact_gc) -PRIMITIVE_FORWARD(gc_stats) PRIMITIVE_FORWARD(save_image) PRIMITIVE_FORWARD(save_image_and_exit) PRIMITIVE_FORWARD(datastack) @@ -115,7 +114,6 @@ PRIMITIVE_FORWARD(call_clear) PRIMITIVE_FORWARD(resize_byte_array) PRIMITIVE_FORWARD(dll_validp) PRIMITIVE_FORWARD(unimplemented) -PRIMITIVE_FORWARD(clear_gc_stats) PRIMITIVE_FORWARD(jit_compile) PRIMITIVE_FORWARD(load_locals) PRIMITIVE_FORWARD(check_datastack) @@ -123,13 +121,13 @@ PRIMITIVE_FORWARD(mega_cache_miss) PRIMITIVE_FORWARD(lookup_method) PRIMITIVE_FORWARD(reset_dispatch_stats) PRIMITIVE_FORWARD(dispatch_stats) -PRIMITIVE_FORWARD(reset_inline_cache_stats) -PRIMITIVE_FORWARD(inline_cache_stats) PRIMITIVE_FORWARD(optimized_p) PRIMITIVE_FORWARD(quot_compiled_p) PRIMITIVE_FORWARD(vm_ptr) PRIMITIVE_FORWARD(strip_stack_traces) PRIMITIVE_FORWARD(callback) +PRIMITIVE_FORWARD(enable_gc_events) +PRIMITIVE_FORWARD(disable_gc_events) const primitive_type primitives[] = { primitive_bignum_to_fixnum, @@ -193,7 +191,6 @@ const primitive_type primitives[] = { primitive_minor_gc, primitive_full_gc, primitive_compact_gc, - primitive_gc_stats, primitive_save_image, primitive_save_image_and_exit, primitive_datastack, @@ -279,7 +276,6 @@ const primitive_type primitives[] = { primitive_resize_byte_array, primitive_dll_validp, primitive_unimplemented, - primitive_clear_gc_stats, primitive_jit_compile, primitive_load_locals, primitive_check_datastack, @@ -289,13 +285,13 @@ const primitive_type primitives[] = { primitive_lookup_method, primitive_reset_dispatch_stats, primitive_dispatch_stats, - primitive_reset_inline_cache_stats, - primitive_inline_cache_stats, primitive_optimized_p, primitive_quot_compiled_p, primitive_vm_ptr, primitive_strip_stack_traces, primitive_callback, + primitive_enable_gc_events, + primitive_disable_gc_events, }; } diff --git a/vm/profiler.cpp b/vm/profiler.cpp index 4674b726b1..50e88cc57a 100755 --- a/vm/profiler.cpp +++ b/vm/profiler.cpp @@ -11,10 +11,10 @@ void factor_vm::init_profiler() /* Allocates memory */ code_block *factor_vm::compile_profiling_stub(cell word_) { - gc_root word(word_,this); + data_root word(word_,this); - jit jit(WORD_TYPE,word.value(),this); - jit.emit_with(userenv[JIT_PROFILING],word.value()); + jit jit(code_block_profiling,word.value(),this); + jit.emit_with(special_objects[JIT_PROFILING],word.value()); return jit.to_code_block(); } @@ -31,7 +31,7 @@ void factor_vm::set_profiling(bool profiling) and allocate profiling blocks if necessary */ primitive_full_gc(); - gc_root words(find_all_words(),this); + data_root words(find_all_words(),this); cell i; cell length = array_capacity(words.untagged()); @@ -40,7 +40,7 @@ void factor_vm::set_profiling(bool profiling) tagged word(array_nth(words.untagged(),i)); if(profiling) word->counter = tag_fixnum(0); - update_word_xt(word.value()); + update_word_xt(word.untagged()); } update_code_heap_words(); diff --git a/vm/quotations.cpp b/vm/quotations.cpp index 9c2c85215d..fc19266cee 100755 --- a/vm/quotations.cpp +++ b/vm/quotations.cpp @@ -6,11 +6,11 @@ namespace factor /* Simple non-optimizing compiler. This is one of the two compilers implementing Factor; the second one is written -in Factor and performs advanced optimizations. See core/compiler/compiler.factor. +in Factor and performs advanced optimizations. See basis/compiler/compiler.factor. The non-optimizing compiler compiles a quotation at a time by concatenating machine code chunks; prolog, epilog, call word, jump to word, etc. These machine -code chunks are generated from Factor code in core/cpu/.../bootstrap.factor. +code chunks are generated from Factor code in basis/cpu/.../bootstrap.factor. Calls to words and constant quotations (referenced by conditionals and dips) are direct jumps to machine code blocks. Literals are also referenced directly @@ -38,29 +38,29 @@ so this results in a big speedup for relatively little effort. */ bool quotation_jit::primitive_call_p(cell i, cell length) { - return (i + 2) == length && array_nth(elements.untagged(),i + 1) == parent->userenv[JIT_PRIMITIVE_WORD]; + return (i + 2) == length && array_nth(elements.untagged(),i + 1) == parent->special_objects[JIT_PRIMITIVE_WORD]; } bool quotation_jit::fast_if_p(cell i, cell length) { return (i + 3) == length && tagged(array_nth(elements.untagged(),i + 1)).type_p(QUOTATION_TYPE) - && array_nth(elements.untagged(),i + 2) == parent->userenv[JIT_IF_WORD]; + && array_nth(elements.untagged(),i + 2) == parent->special_objects[JIT_IF_WORD]; } bool quotation_jit::fast_dip_p(cell i, cell length) { - return (i + 2) <= length && array_nth(elements.untagged(),i + 1) == parent->userenv[JIT_DIP_WORD]; + return (i + 2) <= length && array_nth(elements.untagged(),i + 1) == parent->special_objects[JIT_DIP_WORD]; } bool quotation_jit::fast_2dip_p(cell i, cell length) { - return (i + 2) <= length && array_nth(elements.untagged(),i + 1) == parent->userenv[JIT_2DIP_WORD]; + return (i + 2) <= length && array_nth(elements.untagged(),i + 1) == parent->special_objects[JIT_2DIP_WORD]; } bool quotation_jit::fast_3dip_p(cell i, cell length) { - return (i + 2) <= length && array_nth(elements.untagged(),i + 1) == parent->userenv[JIT_3DIP_WORD]; + return (i + 2) <= length && array_nth(elements.untagged(),i + 1) == parent->special_objects[JIT_3DIP_WORD]; } bool quotation_jit::mega_lookup_p(cell i, cell length) @@ -68,13 +68,13 @@ bool quotation_jit::mega_lookup_p(cell i, cell length) return (i + 4) <= length && tagged(array_nth(elements.untagged(),i + 1)).type_p(FIXNUM_TYPE) && tagged(array_nth(elements.untagged(),i + 2)).type_p(ARRAY_TYPE) - && array_nth(elements.untagged(),i + 3) == parent->userenv[MEGA_LOOKUP_WORD]; + && array_nth(elements.untagged(),i + 3) == parent->special_objects[MEGA_LOOKUP_WORD]; } bool quotation_jit::declare_p(cell i, cell length) { return (i + 2) <= length - && array_nth(elements.untagged(),i + 1) == parent->userenv[JIT_DECLARE_WORD]; + && array_nth(elements.untagged(),i + 1) == parent->special_objects[JIT_DECLARE_WORD]; } bool quotation_jit::stack_frame_p() @@ -88,7 +88,7 @@ bool quotation_jit::stack_frame_p() switch(tagged(obj).type()) { case WORD_TYPE: - if(!parent->to_boolean(parent->untag(obj)->subprimitive)) + if(!parent->to_boolean(untag(obj)->subprimitive)) return true; break; case QUOTATION_TYPE: @@ -110,9 +110,9 @@ bool quotation_jit::trivial_quotation_p(array *elements) void quotation_jit::emit_quot(cell quot_) { - gc_root quot(quot_,parent); + data_root quot(quot_,parent); - array *elements = parent->untag(quot->array); + array *elements = untag(quot->array); /* If the quotation consists of a single word, compile a direct call to the word. */ @@ -133,7 +133,7 @@ void quotation_jit::iterate_quotation() set_position(0); if(stack_frame) - emit(parent->userenv[JIT_PROLOG]); + emit(parent->special_objects[JIT_PROLOG]); cell i; cell length = array_capacity(elements.untagged()); @@ -143,7 +143,7 @@ void quotation_jit::iterate_quotation() { set_position(i); - gc_root obj(array_nth(elements.untagged(),i),parent); + data_root obj(array_nth(elements.untagged(),i),parent); switch(obj.type()) { @@ -152,23 +152,23 @@ void quotation_jit::iterate_quotation() if(parent->to_boolean(obj.as()->subprimitive)) emit_subprimitive(obj.value()); /* The (execute) primitive is special-cased */ - else if(obj.value() == parent->userenv[JIT_EXECUTE_WORD]) + else if(obj.value() == parent->special_objects[JIT_EXECUTE_WORD]) { if(i == length - 1) { - if(stack_frame) emit(parent->userenv[JIT_EPILOG]); + if(stack_frame) emit(parent->special_objects[JIT_EPILOG]); tail_call = true; - emit(parent->userenv[JIT_EXECUTE_JUMP]); + emit(parent->special_objects[JIT_EXECUTE_JUMP]); } else - emit(parent->userenv[JIT_EXECUTE_CALL]); + emit(parent->special_objects[JIT_EXECUTE_CALL]); } /* Everything else */ else { if(i == length - 1) { - if(stack_frame) emit(parent->userenv[JIT_EPILOG]); + if(stack_frame) emit(parent->special_objects[JIT_EPILOG]); tail_call = true; /* Inline cache misses are special-cased. The calling convention for tail @@ -178,8 +178,8 @@ void quotation_jit::iterate_quotation() the inline cache miss primitive, and we don't want to clobber the saved address. */ - if(obj.value() == parent->userenv[PIC_MISS_WORD] - || obj.value() == parent->userenv[PIC_MISS_TAIL_WORD]) + if(obj.value() == parent->special_objects[PIC_MISS_WORD] + || obj.value() == parent->special_objects[PIC_MISS_TAIL_WORD]) { word_special(obj.value()); } @@ -201,7 +201,7 @@ void quotation_jit::iterate_quotation() { literal(tag_fixnum(0)); literal(obj.value()); - emit(parent->userenv[JIT_PRIMITIVE]); + emit(parent->special_objects[JIT_PRIMITIVE]); i++; @@ -215,12 +215,12 @@ void quotation_jit::iterate_quotation() mutually recursive in the library, but both still work) */ if(fast_if_p(i,length)) { - if(stack_frame) emit(parent->userenv[JIT_EPILOG]); + if(stack_frame) emit(parent->special_objects[JIT_EPILOG]); tail_call = true; emit_quot(array_nth(elements.untagged(),i)); emit_quot(array_nth(elements.untagged(),i + 1)); - emit(parent->userenv[JIT_IF]); + emit(parent->special_objects[JIT_IF]); i += 2; } @@ -228,21 +228,21 @@ void quotation_jit::iterate_quotation() else if(fast_dip_p(i,length)) { emit_quot(obj.value()); - emit(parent->userenv[JIT_DIP]); + emit(parent->special_objects[JIT_DIP]); i++; } /* 2dip */ else if(fast_2dip_p(i,length)) { emit_quot(obj.value()); - emit(parent->userenv[JIT_2DIP]); + emit(parent->special_objects[JIT_2DIP]); i++; } /* 3dip */ else if(fast_3dip_p(i,length)) { emit_quot(obj.value()); - emit(parent->userenv[JIT_3DIP]); + emit(parent->special_objects[JIT_3DIP]); i++; } else @@ -276,14 +276,13 @@ void quotation_jit::iterate_quotation() set_position(length); if(stack_frame) - emit(parent->userenv[JIT_EPILOG]); - emit(parent->userenv[JIT_RETURN]); + emit(parent->special_objects[JIT_EPILOG]); + emit(parent->special_objects[JIT_RETURN]); } } void factor_vm::set_quot_xt(quotation *quot, code_block *code) { - assert(code->type() == QUOTATION_TYPE); quot->code = code; quot->xt = code->xt(); } @@ -291,7 +290,7 @@ void factor_vm::set_quot_xt(quotation *quot, code_block *code) /* Allocates memory */ void factor_vm::jit_compile(cell quot_, bool relocating) { - gc_root quot(quot_,this); + data_root quot(quot_,this); if(quot->code) return; quotation_jit compiler(quot.value(),true,relocating,this); @@ -328,18 +327,18 @@ void factor_vm::primitive_quotation_xt() void factor_vm::compile_all_words() { - gc_root words(find_all_words(),this); + data_root words(find_all_words(),this); cell i; cell length = array_capacity(words.untagged()); for(i = 0; i < length; i++) { - gc_root word(array_nth(words.untagged(),i),this); + data_root word(array_nth(words.untagged(),i),this); - if(!word->code || !word_optimized_p(word.untagged())) + if(!word->code || !word->code->optimized_p()) jit_compile_word(word.value(),word->def,false); - update_word_xt(word.value()); + update_word_xt(word.untagged()); } @@ -349,8 +348,8 @@ void factor_vm::compile_all_words() /* Allocates memory */ fixnum factor_vm::quot_code_offset_to_scan(cell quot_, cell offset) { - gc_root quot(quot_,this); - gc_root array(quot->array,this); + data_root quot(quot_,this); + data_root array(quot->array,this); quotation_jit compiler(quot.value(),false,false,this); compiler.compute_position(offset); @@ -361,7 +360,7 @@ fixnum factor_vm::quot_code_offset_to_scan(cell quot_, cell offset) cell factor_vm::lazy_jit_compile_impl(cell quot_, stack_frame *stack) { - gc_root quot(quot_,this); + data_root quot(quot_,this); ctx->callstack_top = stack; jit_compile(quot.value(),true); return quot.value(); diff --git a/vm/quotations.hpp b/vm/quotations.hpp index feb2af1ce4..6d04d80de3 100755 --- a/vm/quotations.hpp +++ b/vm/quotations.hpp @@ -2,11 +2,11 @@ namespace factor { struct quotation_jit : public jit { - gc_root elements; + data_root elements; bool compiling, relocate; explicit quotation_jit(cell quot, bool compiling_, bool relocate_, factor_vm *vm) - : jit(QUOTATION_TYPE,quot,vm), + : jit(code_block_unoptimized,quot,vm), elements(owner.as().untagged()->array,vm), compiling(compiling_), relocate(relocate_){}; diff --git a/vm/run.cpp b/vm/run.cpp index 79aca937ca..6d3e9f7374 100755 --- a/vm/run.cpp +++ b/vm/run.cpp @@ -6,14 +6,14 @@ namespace factor void factor_vm::primitive_getenv() { fixnum e = untag_fixnum(dpeek()); - drepl(userenv[e]); + drepl(special_objects[e]); } void factor_vm::primitive_setenv() { fixnum e = untag_fixnum(dpop()); cell value = dpop(); - userenv[e] = value; + special_objects[e] = value; } void factor_vm::primitive_exit() @@ -52,7 +52,7 @@ void factor_vm::primitive_load_locals() cell factor_vm::clone_object(cell obj_) { - gc_root obj(obj_,this); + data_root obj(obj_,this); if(immediate_p(obj.value())) return obj.value(); diff --git a/vm/run.hpp b/vm/run.hpp index 9a23979066..6ca2e50464 100755 --- a/vm/run.hpp +++ b/vm/run.hpp @@ -1,39 +1,39 @@ namespace factor { -#define USER_ENV 70 +static const cell special_object_count = 70; enum special_object { - NAMESTACK_ENV, /* used by library only */ - CATCHSTACK_ENV, /* used by library only, per-callback */ + OBJ_NAMESTACK, /* used by library only */ + OBJ_CATCHSTACK, /* used by library only, per-callback */ - CURRENT_CALLBACK_ENV = 2, /* used by library only, per-callback */ - WALKER_HOOK_ENV, /* non-local exit hook, used by library only */ - CALLCC_1_ENV, /* used to pass the value in callcc1 */ + OBJ_CURRENT_CALLBACK = 2, /* used by library only, per-callback */ + OBJ_WALKER_HOOK, /* non-local exit hook, used by library only */ + OBJ_CALLCC_1, /* used to pass the value in callcc1 */ - BREAK_ENV = 5, /* quotation called by throw primitive */ - ERROR_ENV, /* a marker consed onto kernel errors */ + OBJ_BREAK = 5, /* quotation called by throw primitive */ + OBJ_ERROR, /* a marker consed onto kernel errors */ - CELL_SIZE_ENV = 7, /* sizeof(cell) */ - CPU_ENV, /* CPU architecture */ - OS_ENV, /* operating system name */ + OBJ_CELL_SIZE = 7, /* sizeof(cell) */ + OBJ_CPU, /* CPU architecture */ + OBJ_OS, /* operating system name */ - ARGS_ENV = 10, /* command line arguments */ - STDIN_ENV, /* stdin FILE* handle */ - STDOUT_ENV, /* stdout FILE* handle */ + OBJ_ARGS = 10, /* command line arguments */ + OBJ_STDIN, /* stdin FILE* handle */ + OBJ_STDOUT, /* stdout FILE* handle */ - IMAGE_ENV = 13, /* image path name */ - EXECUTABLE_ENV, /* runtime executable path name */ + OBJ_IMAGE = 13, /* image path name */ + OBJ_EXECUTABLE, /* runtime executable path name */ - EMBEDDED_ENV = 15, /* are we embedded in another app? */ - EVAL_CALLBACK_ENV, /* used when Factor is embedded in a C app */ - YIELD_CALLBACK_ENV, /* used when Factor is embedded in a C app */ - SLEEP_CALLBACK_ENV, /* used when Factor is embedded in a C app */ + OBJ_EMBEDDED = 15, /* are we embedded in another app? */ + OBJ_EVAL_CALLBACK, /* used when Factor is embedded in a C app */ + OBJ_YIELD_CALLBACK, /* used when Factor is embedded in a C app */ + OBJ_SLEEP_CALLBACK, /* used when Factor is embedded in a C app */ - COCOA_EXCEPTION_ENV = 19, /* Cocoa exception handler quotation */ + OBJ_COCOA_EXCEPTION = 19, /* Cocoa exception handler quotation */ - BOOT_ENV = 20, /* boot quotation */ - GLOBAL_ENV, /* global namespace */ + OBJ_BOOT = 20, /* boot quotation */ + OBJ_GLOBAL, /* global namespace */ /* Quotation compilation in quotations.c */ JIT_PROLOG = 23, @@ -65,11 +65,9 @@ enum special_object { /* Polymorphic inline cache generation in inline_cache.c */ PIC_LOAD = 47, PIC_TAG, - PIC_HI_TAG, PIC_TUPLE, - PIC_HI_TAG_TUPLE, PIC_CHECK_TAG, - PIC_CHECK, + PIC_CHECK_TUPLE, PIC_HIT, PIC_MISS_WORD, PIC_MISS_TAIL_WORD, @@ -77,27 +75,27 @@ enum special_object { /* Megamorphic cache generation in dispatch.c */ MEGA_LOOKUP = 57, MEGA_LOOKUP_WORD, - MEGA_MISS_WORD, + MEGA_MISS_WORD, - UNDEFINED_ENV = 60, /* default quotation for undefined words */ + OBJ_UNDEFINED = 60, /* default quotation for undefined words */ - STDERR_ENV = 61, /* stderr FILE* handle */ + OBJ_STDERR = 61, /* stderr FILE* handle */ - STAGE2_ENV = 62, /* have we bootstrapped? */ + OBJ_STAGE2 = 62, /* have we bootstrapped? */ - CURRENT_THREAD_ENV = 63, + OBJ_CURRENT_THREAD = 63, - THREADS_ENV = 64, - RUN_QUEUE_ENV = 65, - SLEEP_QUEUE_ENV = 66, + OBJ_THREADS = 64, + OBJ_RUN_QUEUE = 65, + OBJ_SLEEP_QUEUE = 66, }; -#define FIRST_SAVE_ENV BOOT_ENV -#define LAST_SAVE_ENV STAGE2_ENV +#define OBJ_FIRST_SAVE OBJ_BOOT +#define OBJ_LAST_SAVE OBJ_STAGE2 inline static bool save_env_p(cell i) { - return (i >= FIRST_SAVE_ENV && i <= LAST_SAVE_ENV); + return (i >= OBJ_FIRST_SAVE && i <= OBJ_LAST_SAVE); } } diff --git a/vm/slot_visitor.hpp b/vm/slot_visitor.hpp new file mode 100644 index 0000000000..e777347622 --- /dev/null +++ b/vm/slot_visitor.hpp @@ -0,0 +1,106 @@ +namespace factor +{ + +template struct slot_visitor { + factor_vm *parent; + Visitor visitor; + + explicit slot_visitor(factor_vm *parent_, Visitor visitor_) : + parent(parent_), visitor(visitor_) {} + + void visit_handle(cell *handle) + { + cell pointer = *handle; + + if(immediate_p(pointer)) return; + + object *untagged = untag(pointer); + untagged = visitor(untagged); + *handle = RETAG(untagged,TAG(pointer)); + } + + void visit_slots(object *ptr, cell payload_start) + { + cell *slot = (cell *)ptr; + cell *end = (cell *)((cell)ptr + payload_start); + + if(slot != end) + { + slot++; + for(; slot < end; slot++) visit_handle(slot); + } + } + + void visit_slots(object *ptr) + { + visit_slots(ptr,ptr->binary_payload_start()); + } + + void visit_stack_elements(segment *region, cell *top) + { + for(cell *ptr = (cell *)region->start; ptr <= top; ptr++) + visit_handle(ptr); + } + + void visit_data_roots() + { + std::vector::const_iterator iter = parent->data_roots.begin(); + std::vector::const_iterator end = parent->data_roots.end(); + + for(; iter < end; iter++) + visit_handle((cell *)(*iter)); + } + + void visit_bignum_roots() + { + std::vector::const_iterator iter = parent->bignum_roots.begin(); + std::vector::const_iterator end = parent->bignum_roots.end(); + + for(; iter < end; iter++) + { + cell *handle = (cell *)(*iter); + + if(*handle) + *handle = (cell)visitor(*(object **)handle); + } + } + + void visit_roots() + { + visit_handle(&parent->true_object); + visit_handle(&parent->bignum_zero); + visit_handle(&parent->bignum_pos_one); + visit_handle(&parent->bignum_neg_one); + + visit_data_roots(); + visit_bignum_roots(); + + for(cell i = 0; i < special_object_count; i++) + visit_handle(&parent->special_objects[i]); + } + + void visit_contexts() + { + context *ctx = parent->ctx; + + while(ctx) + { + visit_stack_elements(ctx->datastack_region,(cell *)ctx->datastack); + visit_stack_elements(ctx->retainstack_region,(cell *)ctx->retainstack); + + visit_handle(&ctx->catchstack_save); + visit_handle(&ctx->current_callback_save); + + ctx = ctx->next; + } + } + + void visit_literal_references(code_block *compiled) + { + visit_handle(&compiled->owner); + visit_handle(&compiled->literals); + visit_handle(&compiled->relocation); + } +}; + +} diff --git a/vm/strings.cpp b/vm/strings.cpp index d7434fe660..9e135e6779 100644 --- a/vm/strings.cpp +++ b/vm/strings.cpp @@ -3,20 +3,20 @@ namespace factor { -cell factor_vm::string_nth(string* str, cell index) +cell string::nth(cell index) const { /* If high bit is set, the most significant 16 bits of the char come from the aux vector. The least significant bit of the corresponding aux vector entry is negated, so that we can XOR the two components together and get the original code point back. */ - cell lo_bits = str->data()[index]; + cell lo_bits = data()[index]; if((lo_bits & 0x80) == 0) return lo_bits; else { - byte_array *aux = untag(str->aux); + byte_array *aux = untag(this->aux); cell hi_bits = aux->data()[index]; return (hi_bits << 7) ^ lo_bits; } @@ -29,7 +29,7 @@ void factor_vm::set_string_nth_fast(string *str, cell index, cell ch) void factor_vm::set_string_nth_slow(string *str_, cell index, cell ch) { - gc_root str(str_,this); + data_root str(str_,this); byte_array *aux; @@ -45,7 +45,7 @@ void factor_vm::set_string_nth_slow(string *str_, cell index, cell ch) if the most significant bit of a character is set. Initially all of the bits are clear. */ - aux = allot_array_internal(untag_fixnum(str->length) * sizeof(u16)); + aux = allot_uninitialized_array(untag_fixnum(str->length) * sizeof(u16)); str->aux = tag(aux); write_barrier(&str->aux); @@ -78,7 +78,7 @@ string *factor_vm::allot_string_internal(cell capacity) /* Allocates memory */ void factor_vm::fill_string(string *str_, cell start, cell capacity, cell fill) { - gc_root str(str_,this); + data_root str(str_,this); if(fill <= 0x7f) memset(&str->data()[start],fill,capacity - start); @@ -94,7 +94,7 @@ void factor_vm::fill_string(string *str_, cell start, cell capacity, cell fill) /* Allocates memory */ string *factor_vm::allot_string(cell capacity, cell fill) { - gc_root str(allot_string_internal(capacity),this); + data_root str(allot_string_internal(capacity),this); fill_string(str.untagged(),0,capacity,fill); return str.untagged(); } @@ -115,7 +115,7 @@ bool factor_vm::reallot_string_in_place_p(string *str, cell capacity) string* factor_vm::reallot_string(string *str_, cell capacity) { - gc_root str(str_,this); + data_root str(str_,this); if(reallot_string_in_place_p(str.untagged(),capacity)) { @@ -135,7 +135,7 @@ string* factor_vm::reallot_string(string *str_, cell capacity) if(capacity < to_copy) to_copy = capacity; - gc_root new_str(allot_string_internal(capacity),this); + data_root new_str(allot_string_internal(capacity),this); memcpy(new_str->data(),str->data(),to_copy); @@ -166,7 +166,7 @@ void factor_vm::primitive_string_nth() { string *str = untag(dpop()); cell index = untag_fixnum(dpop()); - dpush(tag_fixnum(string_nth(str,index))); + dpush(tag_fixnum(str->nth(index))); } void factor_vm::primitive_set_string_nth_fast() diff --git a/vm/strings.hpp b/vm/strings.hpp index 727ca8516e..54ff981d99 100644 --- a/vm/strings.hpp +++ b/vm/strings.hpp @@ -1,7 +1,7 @@ namespace factor { -inline static cell string_capacity(string *str) +inline static cell string_capacity(const string *str) { return untag_fixnum(str->length); } diff --git a/vm/tagged.hpp b/vm/tagged.hpp index a61c599aeb..e520e326fa 100755 --- a/vm/tagged.hpp +++ b/vm/tagged.hpp @@ -3,12 +3,12 @@ namespace factor template cell tag(Type *value) { - return RETAG(value,tag_for(Type::type_number)); + return RETAG(value,Type::type_number); } inline static cell tag_dynamic(object *value) { - return RETAG(value,tag_for(value->h.hi_tag())); + return RETAG(value,value->h.hi_tag()); } template @@ -16,36 +16,49 @@ struct tagged { cell value_; - cell value() const { return value_; } - Type *untagged() const { return (Type *)(UNTAG(value_)); } - - cell type() const { - cell tag = TAG(value_); - if(tag == OBJECT_TYPE) - return untagged()->h.hi_tag(); - else - return tag; + cell type() const + { + return TAG(value_); } - bool type_p(cell type_) const { return type() == type_; } + bool type_p(cell type_) const + { + return type() == type_; + } - Type *untag_check(factor_vm *parent) const { - if(Type::type_number != TYPE_COUNT && !type_p(Type::type_number)) + bool type_p() const + { + if(Type::type_number == TYPE_COUNT) + return true; + else + return type_p(Type::type_number); + } + + cell value() const + { +#ifdef FACTOR_DEBUG + assert(type_p()); +#endif + return value_; + } + + Type *untagged() const + { +#ifdef FACTOR_DEBUG + assert(type_p()); +#endif + return (Type *)(UNTAG(value_)); + } + + Type *untag_check(factor_vm *parent) const + { + if(!type_p()) parent->type_error(Type::type_number,value_); return untagged(); } - explicit tagged(cell tagged) : value_(tagged) { -#ifdef FACTOR_DEBUG - untag_check(tls_vm()); -#endif - } - - explicit tagged(Type *untagged) : value_(factor::tag(untagged)) { -#ifdef FACTOR_DEBUG - untag_check(tls_vm()); -#endif - } + explicit tagged(cell tagged) : value_(tagged) {} + explicit tagged(Type *untagged) : value_(factor::tag(untagged)) {} Type *operator->() const { return untagged(); } cell *operator&() const { return &value_; } @@ -64,7 +77,7 @@ template Type *factor_vm::untag_check(cell value) return tagged(value).untag_check(this); } -template Type *factor_vm::untag(cell value) +template Type *untag(cell value) { return tagged(value).untagged(); } diff --git a/vm/tenured_space.hpp b/vm/tenured_space.hpp index f9f584b200..baab47e383 100644 --- a/vm/tenured_space.hpp +++ b/vm/tenured_space.hpp @@ -1,8 +1,63 @@ namespace factor { -struct tenured_space : old_space { - tenured_space(cell size, cell start) : old_space(size,start) {} +struct tenured_space : free_list_allocator { + object_start_map starts; + std::vector mark_stack; + + explicit tenured_space(cell size, cell start) : + free_list_allocator(size,start), starts(size,start) {} + + object *allot(cell size) + { + object *obj = free_list_allocator::allot(size); + if(obj) + { + starts.record_object_start_offset(obj); + return obj; + } + else + return NULL; + } + + cell first_object() + { + return (cell)next_allocated_block_after(this->first_block()); + } + + cell next_object_after(cell scan) + { + cell size = ((object *)scan)->size(); + object *next = (object *)(scan + size); + return (cell)next_allocated_block_after(next); + } + + void clear_mark_bits() + { + state.clear_mark_bits(); + } + + void clear_mark_stack() + { + mark_stack.clear(); + } + + bool marked_p(object *obj) + { + return this->state.marked_p(obj); + } + + void mark_and_push(object *obj) + { + this->state.set_marked_p(obj); + this->mark_stack.push_back(obj); + } + + void sweep() + { + free_list_allocator::sweep(); + starts.update_for_sweep(&this->state); + } }; } diff --git a/vm/to_tenured_collector.cpp b/vm/to_tenured_collector.cpp index b5d4793ceb..0cee748205 100644 --- a/vm/to_tenured_collector.cpp +++ b/vm/to_tenured_collector.cpp @@ -4,30 +4,51 @@ namespace factor { to_tenured_collector::to_tenured_collector(factor_vm *myvm_) : - copying_collector( + collector( myvm_, - &myvm_->gc_stats.aging_stats, myvm_->data->tenured, to_tenured_policy(myvm_)) {} +void to_tenured_collector::tenure_reachable_objects() +{ + std::vector *mark_stack = &this->target->mark_stack; + while(!mark_stack->empty()) + { + object *obj = mark_stack->back(); + mark_stack->pop_back(); + this->trace_object(obj); + } +} + void factor_vm::collect_to_tenured() { /* Copy live objects from aging space to tenured space. */ to_tenured_collector collector(this); + data->tenured->clear_mark_stack(); + collector.trace_roots(); collector.trace_contexts(); + + current_gc->event->started_card_scan(); collector.trace_cards(data->tenured, card_points_to_aging, - dummy_unmarker()); + full_unmarker()); + current_gc->event->ended_card_scan(collector.cards_scanned,collector.decks_scanned); + + current_gc->event->started_code_scan(); collector.trace_code_heap_roots(&code->points_to_aging); - collector.cheneys_algorithm(); + current_gc->event->ended_code_scan(collector.code_blocks_scanned); + + collector.tenure_reachable_objects(); + + current_gc->event->started_code_sweep(); update_code_heap_for_minor_gc(&code->points_to_aging); + current_gc->event->ended_code_sweep(); - nursery.here = nursery.start; - reset_generation(data->aging); - code->points_to_nursery.clear(); - code->points_to_aging.clear(); + data->reset_generation(&nursery); + data->reset_generation(data->aging); + code->clear_remembered_set(); } } diff --git a/vm/to_tenured_collector.hpp b/vm/to_tenured_collector.hpp index 64bd9aa04d..2f2717efd1 100644 --- a/vm/to_tenured_collector.hpp +++ b/vm/to_tenured_collector.hpp @@ -3,18 +3,26 @@ namespace factor struct to_tenured_policy { factor_vm *myvm; - zone *tenured; + tenured_space *tenured; - to_tenured_policy(factor_vm *myvm_) : myvm(myvm_), tenured(myvm->data->tenured) {} + explicit to_tenured_policy(factor_vm *myvm_) : myvm(myvm_), tenured(myvm->data->tenured) {} bool should_copy_p(object *untagged) { return !tenured->contains_p(untagged); } + + void promoted_object(object *obj) + { + tenured->mark_stack.push_back(obj); + } + + void visited_object(object *obj) {} }; -struct to_tenured_collector : copying_collector { - to_tenured_collector(factor_vm *myvm_); +struct to_tenured_collector : collector { + explicit to_tenured_collector(factor_vm *myvm_); + void tenure_reachable_objects(); }; } diff --git a/vm/tuples.cpp b/vm/tuples.cpp index 8c759b4884..eaac437753 100755 --- a/vm/tuples.cpp +++ b/vm/tuples.cpp @@ -6,7 +6,7 @@ namespace factor /* push a new tuple on the stack, filling its slots with f */ void factor_vm::primitive_tuple() { - gc_root layout(dpop(),this); + data_root layout(dpop(),this); tagged t(allot(tuple_size(layout.untagged()))); t->layout = layout.value(); @@ -18,7 +18,7 @@ void factor_vm::primitive_tuple() /* push a new tuple on the stack, filling its slots from the stack */ void factor_vm::primitive_tuple_boa() { - gc_root layout(dpop(),this); + data_root layout(dpop(),this); tagged t(allot(tuple_size(layout.untagged()))); t->layout = layout.value(); diff --git a/vm/tuples.hpp b/vm/tuples.hpp index 04b23b5857..bcd041fc65 100644 --- a/vm/tuples.hpp +++ b/vm/tuples.hpp @@ -1,7 +1,7 @@ namespace factor { -inline static cell tuple_size(tuple_layout *layout) +inline static cell tuple_size(const tuple_layout *layout) { cell size = untag_fixnum(layout->size); return sizeof(tuple) + size * sizeof(cell); diff --git a/vm/utilities.cpp b/vm/utilities.cpp index 0595430283..8f063a9ad4 100755 --- a/vm/utilities.cpp +++ b/vm/utilities.cpp @@ -11,38 +11,6 @@ vm_char *safe_strdup(const vm_char *str) return ptr; } -/* We don't use printf directly, because format directives are not portable. -Instead we define the common cases here. */ -void nl() -{ - fputs("\n",stdout); -} - -void print_string(const char *str) -{ - fputs(str,stdout); -} - -void print_cell(cell x) -{ - printf(CELL_FORMAT,x); -} - -void print_cell_hex(cell x) -{ - printf(CELL_HEX_FORMAT,x); -} - -void print_cell_hex_pad(cell x) -{ - printf(CELL_HEX_PAD_FORMAT,x); -} - -void print_fixnum(fixnum x) -{ - printf(FIXNUM_FORMAT,x); -} - cell read_cell_hex() { cell cell; diff --git a/vm/utilities.hpp b/vm/utilities.hpp index 3552733f6b..94b9de6f48 100755 --- a/vm/utilities.hpp +++ b/vm/utilities.hpp @@ -26,12 +26,6 @@ inline static void memset_cell(void *dst, cell pattern, size_t size) } vm_char *safe_strdup(const vm_char *str); -void print_string(const char *str); -void nl(); -void print_cell(cell x); -void print_cell_hex(cell x); -void print_cell_hex_pad(cell x); -void print_fixnum(fixnum x); cell read_cell_hex(); } diff --git a/vm/vm.cpp b/vm/vm.cpp index 50dc441086..72c63292fd 100755 --- a/vm/vm.cpp +++ b/vm/vm.cpp @@ -6,11 +6,13 @@ namespace factor factor_vm::factor_vm() : nursery(0,0), profiling_p(false), - secure_gc(false), gc_off(false), current_gc(NULL), + gc_events(NULL), fep_disabled(false), full_output(false) - { } +{ + primitive_reset_dispatch_stats(); +} } diff --git a/vm/vm.hpp b/vm/vm.hpp index d3686c743e..aa5a3051e6 100755 --- a/vm/vm.hpp +++ b/vm/vm.hpp @@ -2,6 +2,7 @@ namespace factor { struct growable_array; +struct code_root; struct factor_vm { @@ -11,14 +12,14 @@ struct factor_vm context *ctx; /* New objects are allocated here */ - zone nursery; + nursery_space nursery; /* Add this to a shifted address to compute write barrier offsets */ cell cards_offset; cell decks_offset; /* TAGGED user environment data; see getenv/setenv prims */ - cell userenv[USER_ENV]; + cell special_objects[special_object_count]; /* Data stack and retain stack sizes */ cell ds_size, rs_size; @@ -39,9 +40,6 @@ struct factor_vm unsigned int signal_fpu_status; stack_frame *signal_callstack_top; - /* Zeroes out deallocated memory; set by the -securegc command line argument */ - bool secure_gc; - /* A heap walk allows useful things to be done, like finding all references to an object for debugging purposes. */ cell heap_scan_ptr; @@ -61,14 +59,16 @@ struct factor_vm /* Only set if we're performing a GC */ gc_state *current_gc; - /* Statistics */ - gc_statistics gc_stats; + /* If not NULL, we push GC events here */ + std::vector *gc_events; /* If a runtime function needs to call another function which potentially - allocates memory, it must wrap any local variable references to Factor - objects in gc_root instances */ - std::vector gc_locals; - std::vector gc_bignums; + allocates memory, it must wrap any references to the data and code + heaps with data_root and code_root smart pointers, which register + themselves here. See data_roots.hpp and code_roots.hpp */ + std::vector data_roots; + std::vector bignum_roots; + std::vector code_roots; /* Debugger */ bool fep_disabled; @@ -80,14 +80,7 @@ struct factor_vm cell bignum_neg_one; /* Method dispatch statistics */ - cell megamorphic_cache_hits; - cell megamorphic_cache_misses; - - cell cold_call_to_ic_transitions; - cell ic_to_pic_transitions; - cell pic_to_mega_transitions; - /* Indexed by PIC_TAG, PIC_HI_TAG, PIC_TUPLE, PIC_HI_TAG_TUPLE */ - cell pic_counts[4]; + dispatch_statistics dispatch_stats; /* Number of entries in a polymorphic inline cache */ cell max_pic_size; @@ -220,15 +213,10 @@ struct factor_vm //data heap void init_card_decks(); - void clear_cards(old_space *gen); - void clear_decks(old_space *gen); - void reset_generation(old_space *gen); void set_data_heap(data_heap *data_); - void init_data_heap(cell young_size, cell aging_size, cell tenured_size, bool secure_gc_); - cell untagged_object_size(object *pointer); - cell unaligned_object_size(object *pointer); + void init_data_heap(cell young_size, cell aging_size, cell tenured_size); void primitive_size(); - cell binary_payload_start(object *pointer); + data_heap_room data_room(); void primitive_data_room(); void begin_scan(); void end_scan(); @@ -236,10 +224,18 @@ struct factor_vm cell next_object(); void primitive_next_object(); void primitive_end_scan(); - template void each_object(Iterator &iterator); cell find_all_words(); cell object_size(cell tagged); + template inline void each_object(Iterator &iterator) + { + begin_scan(); + cell obj; + while(to_boolean(obj = next_object())) + iterator(obj); + end_scan(); + } + /* the write barrier must be called any time we are potentially storing a pointer from an older generation to a younger one */ inline void write_barrier(cell *slot_ptr) @@ -249,26 +245,31 @@ struct factor_vm } // gc + void end_gc(); + void start_gc_again(); void update_code_heap_for_minor_gc(std::set *remembered_set); void collect_nursery(); void collect_aging(); void collect_to_tenured(); - void collect_full_impl(bool trace_contexts_p); - void collect_growing_heap(cell requested_bytes, bool trace_contexts_p, bool compact_code_heap_p); - void collect_full(bool trace_contexts_p, bool compact_code_heap_p); - void record_gc_stats(generation_statistics *stats); - void gc(gc_op op, cell requested_bytes, bool trace_contexts_p, bool compact_code_heap_p); + void update_code_roots_for_sweep(); + void update_code_roots_for_compaction(); + void collect_mark_impl(bool trace_contexts_p); + void collect_sweep_impl(); + void collect_full(bool trace_contexts_p); + void collect_compact_impl(bool trace_contexts_p); + void collect_compact_code_impl(bool trace_contexts_p); + void collect_compact(bool trace_contexts_p); + void collect_growing_heap(cell requested_bytes, bool trace_contexts_p); + void gc(gc_op op, cell requested_bytes, bool trace_contexts_p); void primitive_minor_gc(); void primitive_full_gc(); void primitive_compact_gc(); - void primitive_gc_stats(); - void clear_gc_stats(); void primitive_become(); - void inline_gc(cell *gc_roots_base, cell gc_roots_size); + void inline_gc(cell *data_roots_base, cell data_roots_size); + void primitive_enable_gc_events(); + void primitive_disable_gc_events(); object *allot_object(header header, cell size); object *allot_large_object(header header, cell size); - void add_gc_stats(generation_statistics *stats, growable_array *result); - void primitive_clear_gc_stats(); template Type *allot(cell size) { @@ -286,20 +287,8 @@ struct factor_vm #endif } - inline void check_tagged_pointer(cell tagged) - { - #ifdef FACTOR_DEBUG - if(!immediate_p(tagged)) - { - object *obj = untag(tagged); - check_data_pointer(obj); - obj->h.hi_tag(); - } - #endif - } - // generic arrays - template Array *allot_array_internal(cell capacity); + template Array *allot_uninitialized_array(cell capacity); template bool reallot_array_in_place_p(Array *array, cell capacity); template Array *reallot_array(Array *array_, cell capacity); @@ -317,7 +306,7 @@ struct factor_vm void print_callstack(); void dump_cell(cell x); void dump_memory(cell from, cell to); - void dump_zone(const char *name, zone *z); + template void dump_generation(const char *name, Generation *gen); void dump_generations(); void dump_objects(cell type); void find_data_references_step(cell *scan); @@ -336,7 +325,7 @@ struct factor_vm inline void set_array_nth(array *array, cell slot, cell value); //strings - cell string_nth(string* str, cell index); + cell string_nth(const string *str, cell index); void set_string_nth_fast(string *str, cell index, cell ch); void set_string_nth_slow(string *str_, cell index, cell ch); void set_string_nth(string *str, cell index, cell ch); @@ -362,6 +351,9 @@ struct factor_vm void primitive_uninitialized_byte_array(); void primitive_resize_byte_array(); + template byte_array *byte_array_from_value(Type *value); + template byte_array *byte_array_from_values(Type *values, cell len); + //tuples void primitive_tuple(); void primitive_tuple_boa(); @@ -370,7 +362,7 @@ struct factor_vm word *allot_word(cell name_, cell vocab_, cell hashcode_); void primitive_word(); void primitive_word_xt(); - void update_word_xt(cell w_); + void update_word_xt(word *w_); void primitive_optimized_p(); void primitive_wrapper(); @@ -459,8 +451,9 @@ struct factor_vm inline double untag_float_check(cell tagged); inline fixnum float_to_fixnum(cell tagged); inline double fixnum_to_float(cell tagged); + + // tagged template Type *untag_check(cell value); - template Type *untag(cell value); //io void init_c_io(); @@ -495,12 +488,12 @@ struct factor_vm void update_literal_references(code_block *compiled); void relocate_code_block_step(relocation_entry rel, cell index, code_block *compiled); void update_word_references(code_block *compiled); - void update_code_block_for_full_gc(code_block *compiled); + void update_code_block_words_and_literals(code_block *compiled); void check_code_address(cell address); void relocate_code_block(code_block *compiled); void fixup_labels(array *labels, code_block *compiled); - code_block *allot_code_block(cell size, cell type); - code_block *add_code_block(cell type, cell code_, cell labels_, cell owner_, cell relocation_, cell literals_); + code_block *allot_code_block(cell size, code_block_type type); + code_block *add_code_block(code_block_type type, cell code_, cell labels_, cell owner_, cell relocation_, cell literals_); //code heap inline void check_code_pointer(cell ptr) @@ -514,25 +507,17 @@ struct factor_vm bool in_code_heap_p(cell ptr); void jit_compile_word(cell word_, cell def_, bool relocate); void update_code_heap_words(); + void update_code_heap_words_and_literals(); + void relocate_code_heap(); void primitive_modify_code_heap(); + code_heap_room code_room(); void primitive_code_room(); - void forward_object_xts(); - void forward_context_xts(); - void forward_callback_xts(); - void compact_code_heap(bool trace_contexts_p); void primitive_strip_stack_traces(); /* Apply a function to every code block */ template void iterate_code_heap(Iterator &iter) { - heap_block *scan = code->first_block(); - - while(scan) - { - if(scan->type() != FREE_BLOCK_TYPE) - iter((code_block *)scan); - scan = code->next_block(scan); - } + code->allocator->iterate(iter); } //callbacks @@ -567,7 +552,7 @@ struct factor_vm void primitive_callstack(); void primitive_set_callstack(); code_block *frame_code(stack_frame *frame); - cell frame_type(stack_frame *frame); + code_block_type frame_type(stack_frame *frame); cell frame_executing(stack_frame *frame); stack_frame *frame_successor(stack_frame *frame); cell frame_scan(stack_frame *frame); @@ -586,7 +571,7 @@ struct factor_vm template void do_slots(cell obj, Iterator &iter) { cell scan = obj; - cell payload_start = binary_payload_start((object *)obj); + cell payload_start = ((object *)obj)->binary_payload_start(); cell end = obj + payload_start; scan += sizeof(cell); @@ -634,7 +619,6 @@ struct factor_vm cell nth_superclass(tuple_layout *layout, fixnum echelon); cell nth_hashcode(tuple_layout *layout, fixnum echelon); cell lookup_tuple_method(cell obj, cell methods); - cell lookup_hi_tag_method(cell obj, cell methods); cell lookup_method(cell obj, cell methods); void primitive_lookup_method(); cell object_class(cell obj); @@ -655,8 +639,6 @@ struct factor_vm cell add_inline_cache_entry(cell cache_entries_, cell klass_, cell method_); void update_pic_transitions(cell pic_size); void *inline_cache_miss(cell return_address); - void primitive_reset_inline_cache_stats(); - void primitive_inline_cache_stats(); //factor void default_parameters(vm_parameters *p); @@ -705,6 +687,6 @@ struct factor_vm }; -extern unordered_map thread_vms; +extern std::map thread_vms; } diff --git a/vm/words.cpp b/vm/words.cpp index 6193a5c93c..dfaeed2496 100644 --- a/vm/words.cpp +++ b/vm/words.cpp @@ -5,15 +5,15 @@ namespace factor word *factor_vm::allot_word(cell name_, cell vocab_, cell hashcode_) { - gc_root vocab(vocab_,this); - gc_root name(name_,this); + data_root vocab(vocab_,this); + data_root name(name_,this); - gc_root new_word(allot(sizeof(word)),this); + data_root new_word(allot(sizeof(word)),this); new_word->hashcode = hashcode_; new_word->vocabulary = vocab.value(); new_word->name = name.value(); - new_word->def = userenv[UNDEFINED_ENV]; + new_word->def = special_objects[OBJ_UNDEFINED]; new_word->props = false_object; new_word->counter = tag_fixnum(0); new_word->pic_def = false_object; @@ -23,7 +23,7 @@ word *factor_vm::allot_word(cell name_, cell vocab_, cell hashcode_) new_word->code = NULL; jit_compile_word(new_word.value(),new_word->def,true); - update_word_xt(new_word.value()); + update_word_xt(new_word.untagged()); if(profiling_p) relocate_code_block(new_word->profiling); @@ -43,7 +43,7 @@ void factor_vm::primitive_word() /* word-xt ( word -- start end ) */ void factor_vm::primitive_word_xt() { - gc_root w(dpop(),this); + data_root w(dpop(),this); w.untag_check(this); if(profiling_p) @@ -59,17 +59,17 @@ void factor_vm::primitive_word_xt() } /* Allocates memory */ -void factor_vm::update_word_xt(cell w_) +void factor_vm::update_word_xt(word *w_) { - gc_root w(w_,this); + data_root w(w_,this); if(profiling_p) { if(!w->profiling) { - /* Note: can't do w->profiling = ... since if LHS - evaluates before RHS, since in that case if RHS does a - GC, we will have an invalid pointer on the LHS */ + /* Note: can't do w->profiling = ... since LHS evaluates + before RHS, and if RHS does a GC, we will have an + invalid pointer on the LHS */ code_block *profiling = compile_profiling_stub(w.value()); w->profiling = profiling; } @@ -82,7 +82,8 @@ void factor_vm::update_word_xt(cell w_) void factor_vm::primitive_optimized_p() { - drepl(tag_boolean(word_optimized_p(untag_check(dpeek())))); + word *w = untag_check(dpeek()); + drepl(tag_boolean(w->code->optimized_p())); } void factor_vm::primitive_wrapper() diff --git a/vm/words.hpp b/vm/words.hpp index 1701def6dc..412ef35bb4 100644 --- a/vm/words.hpp +++ b/vm/words.hpp @@ -1,9 +1,4 @@ namespace factor { -inline bool word_optimized_p(word *word) -{ - return word->code->type() == WORD_TYPE; -} - } diff --git a/vm/zone.hpp b/vm/zone.hpp deleted file mode 100644 index 4fe4ae9b6b..0000000000 --- a/vm/zone.hpp +++ /dev/null @@ -1,26 +0,0 @@ -namespace factor -{ - -struct zone { - /* offset of 'here' and 'end' is hardcoded in compiler backends */ - cell here; - cell start; - cell end; - cell size; - - zone(cell size_, cell start_) : here(0), start(start_), end(start_ + size_), size(size_) {} - - inline bool contains_p(object *pointer) - { - return ((cell)pointer - start) < size; - } - - inline object *allot(cell size) - { - cell h = here; - here = h + align8(size); - return (object *)h; - } -}; - -}