new intrinsic generators, pt1
Joe Groff
parent
d655c3c9cc
commit
bd77633d5b
basis/compiler/cfg/intrinsics/simd
|
|
@ -10,78 +10,27 @@ compiler.cfg.stacks compiler.cfg.stacks.local compiler.cfg.hats
|
|||
compiler.cfg.instructions compiler.cfg.registers
|
||||
compiler.cfg.intrinsics.alien
|
||||
specialized-arrays ;
|
||||
FROM: alien.c-types => heap-size uchar ushort uint ulonglong float double ;
|
||||
SPECIALIZED-ARRAYS: uchar ushort uint ulonglong float double ;
|
||||
FROM: alien.c-types => heap-size char short int longlong float double ;
|
||||
SPECIALIZED-ARRAYS: char short int longlong float double ;
|
||||
IN: compiler.cfg.intrinsics.simd
|
||||
|
||||
MACRO: check-elements ( quots -- )
|
||||
[ length '[ _ firstn ] ]
|
||||
[ '[ _ spread ] ]
|
||||
[ length 1 - \ and <repetition> [ ] like ]
|
||||
tri 3append ;
|
||||
! compound vector ops
|
||||
|
||||
MACRO: if-literals-match ( quots -- )
|
||||
[ length ] [ ] [ length ] tri
|
||||
! n quots n
|
||||
'[
|
||||
! node quot
|
||||
[
|
||||
dup node-input-infos
|
||||
_ tail-slice* [ literal>> ] map
|
||||
dup _ check-elements
|
||||
] dip
|
||||
swap [
|
||||
! node literals quot
|
||||
[ _ firstn ] dip call
|
||||
drop
|
||||
] [ 2drop emit-primitive ] if
|
||||
] ;
|
||||
: ^load-neg-zero-vector ( rep -- dst )
|
||||
{
|
||||
{ float-4-rep [ float-array{ -0.0 -0.0 -0.0 -0.0 } underlying>> ^^load-constant ] }
|
||||
{ double-2-rep [ double-array{ -0.0 -0.0 } underlying>> ^^load-constant ] }
|
||||
} case ;
|
||||
|
||||
: emit-vector-op ( node quot: ( rep -- ) -- )
|
||||
{ [ representation? ] } if-literals-match ; inline
|
||||
|
||||
: [binary] ( quot -- quot' )
|
||||
'[ [ ds-drop 2inputs ] dip @ ds-push ] ; inline
|
||||
|
||||
: emit-binary-vector-op ( node quot -- )
|
||||
[binary] emit-vector-op ; inline
|
||||
|
||||
: [unary] ( quot -- quot' )
|
||||
'[ [ ds-drop ds-pop ] dip @ ds-push ] ; inline
|
||||
|
||||
: emit-unary-vector-op ( node quot -- )
|
||||
[unary] emit-vector-op ; inline
|
||||
|
||||
: [unary/param] ( quot -- quot' )
|
||||
'[ [ -2 inc-d ds-pop ] 2dip @ ds-push ] ; inline
|
||||
|
||||
: emit-shift-vector-imm-op ( node quot -- )
|
||||
[unary/param]
|
||||
{ [ integer? ] [ representation? ] } if-literals-match ; inline
|
||||
|
||||
:: emit-shift-vector-op ( node imm-quot var-quot -- )
|
||||
node node-input-infos 2 tail-slice* first literal>> integer?
|
||||
[ node imm-quot emit-shift-vector-imm-op ]
|
||||
[ node var-quot emit-binary-vector-op ] if ; inline
|
||||
|
||||
: emit-gather-vector-2 ( node -- )
|
||||
[ ^^gather-vector-2 ] emit-binary-vector-op ;
|
||||
|
||||
: emit-gather-vector-4 ( node -- )
|
||||
[
|
||||
ds-drop
|
||||
[
|
||||
D 3 peek-loc
|
||||
D 2 peek-loc
|
||||
D 1 peek-loc
|
||||
D 0 peek-loc
|
||||
-4 inc-d
|
||||
] dip
|
||||
^^gather-vector-4
|
||||
ds-push
|
||||
] emit-vector-op ;
|
||||
|
||||
: shuffle? ( obj -- ? ) { [ array? ] [ [ integer? ] all? ] } 1&& ;
|
||||
: ^load-add-sub-vector ( rep -- dst )
|
||||
unsign-rep {
|
||||
{ float-4-rep [ float-array{ -0.0 0.0 -0.0 0.0 } underlying>> ^^load-constant ] }
|
||||
{ double-2-rep [ double-array{ -0.0 0.0 } underlying>> ^^load-constant ] }
|
||||
{ char-16-rep [ char-array{ -1 0 -1 0 -1 0 -1 0 -1 0 -1 0 -1 0 -1 0 } underlying>> ^^load-constant ] }
|
||||
{ short-8-rep [ short-array{ -1 0 -1 0 -1 0 -1 0 } underlying>> ^^load-constant ] }
|
||||
{ int-4-rep [ int-array{ -1 0 -1 0 } underlying>> ^^load-constant ] }
|
||||
{ longlong-2-rep [ longlong-array{ -1 0 } underlying>> ^^load-constant ] }
|
||||
} case ;
|
||||
|
||||
: >variable-shuffle ( shuffle rep -- shuffle' )
|
||||
rep-component-type heap-size
|
||||
|
|
@ -89,325 +38,346 @@ MACRO: if-literals-match ( quots -- )
|
|||
[ iota >byte-array ] bi
|
||||
'[ _ n*v _ v+ ] map concat ;
|
||||
|
||||
: generate-shuffle-vector-imm ( src shuffle rep -- dst )
|
||||
dup %shuffle-vector-imm-reps member?
|
||||
[ ^^shuffle-vector-imm ]
|
||||
[
|
||||
[ >variable-shuffle ^^load-constant ] keep
|
||||
^^shuffle-vector
|
||||
] if ;
|
||||
: ^load-immediate-shuffle ( shuffle rep -- dst )
|
||||
>variable-shuffle ^^load-constant ;
|
||||
|
||||
: emit-shuffle-vector-imm ( node -- )
|
||||
! Pad the permutation with zeroes if it's too short, since we
|
||||
! can't throw an error at this point.
|
||||
[ [ rep-components 0 pad-tail ] keep generate-shuffle-vector-imm ] [unary/param]
|
||||
{ [ shuffle? ] [ representation? ] } if-literals-match ;
|
||||
|
||||
: emit-shuffle-vector-var ( node -- )
|
||||
[ ^^shuffle-vector ] [binary]
|
||||
{ [ %shuffle-vector-reps member? ] } if-literals-match ;
|
||||
|
||||
: emit-shuffle-vector ( node -- )
|
||||
dup node-input-infos {
|
||||
[ length 3 = ]
|
||||
[ first class>> byte-array class<= ]
|
||||
[ second class>> byte-array class<= ]
|
||||
[ third literal>> representation? ]
|
||||
} 1&& [ emit-shuffle-vector-var ] [ emit-shuffle-vector-imm ] if ;
|
||||
|
||||
: ^^broadcast-vector ( src n rep -- dst )
|
||||
[ rep-components swap <array> ] keep
|
||||
generate-shuffle-vector-imm ;
|
||||
|
||||
: emit-broadcast-vector ( node -- )
|
||||
[ ^^broadcast-vector ] [unary/param]
|
||||
{ [ integer? ] [ representation? ] } if-literals-match ;
|
||||
|
||||
: ^^with-vector ( src rep -- dst )
|
||||
[ ^^scalar>vector ] keep [ 0 ] dip ^^broadcast-vector ;
|
||||
|
||||
: ^^select-vector ( src n rep -- dst )
|
||||
[ ^^broadcast-vector ] keep ^^vector>scalar ;
|
||||
|
||||
: emit-select-vector ( node -- )
|
||||
[ ^^select-vector ] [unary/param]
|
||||
{ [ integer? ] [ representation? ] } if-literals-match ; inline
|
||||
|
||||
: emit-alien-vector-op ( node quot: ( rep -- ) -- )
|
||||
{ [ %alien-vector-reps member? ] } if-literals-match ; inline
|
||||
|
||||
: emit-alien-vector ( node -- )
|
||||
dup [
|
||||
'[
|
||||
ds-drop prepare-alien-getter
|
||||
_ ^^alien-vector ds-push
|
||||
]
|
||||
[ inline-alien-getter? ] inline-alien
|
||||
] with emit-alien-vector-op ;
|
||||
|
||||
: emit-set-alien-vector ( node -- )
|
||||
dup [
|
||||
'[
|
||||
ds-drop prepare-alien-setter ds-pop
|
||||
_ ##set-alien-vector
|
||||
]
|
||||
[ byte-array inline-alien-setter? ]
|
||||
inline-alien
|
||||
] with emit-alien-vector-op ;
|
||||
|
||||
: generate-not-vector ( src rep -- dst )
|
||||
dup %not-vector-reps member?
|
||||
[ ^^not-vector ]
|
||||
[ [ ^^fill-vector ] [ ^^xor-vector ] bi ] if ;
|
||||
|
||||
:: ((generate-compare-vector)) ( src1 src2 rep {cc,swap} -- dst )
|
||||
{cc,swap} first2 :> ( cc swap? )
|
||||
swap?
|
||||
[ src2 src1 rep cc ^^compare-vector ]
|
||||
[ src1 src2 rep cc ^^compare-vector ] if ;
|
||||
|
||||
:: (generate-compare-vector) ( src1 src2 rep orig-cc -- dst )
|
||||
rep orig-cc %compare-vector-ccs :> ( ccs not? )
|
||||
|
||||
ccs empty?
|
||||
[ rep not? [ ^^fill-vector ] [ ^^zero-vector ] if ]
|
||||
[
|
||||
ccs unclip :> ( rest-ccs first-cc )
|
||||
src1 src2 rep first-cc ((generate-compare-vector)) :> first-dst
|
||||
|
||||
rest-ccs first-dst
|
||||
[ [ src1 src2 rep ] dip ((generate-compare-vector)) rep ^^or-vector ]
|
||||
reduce
|
||||
|
||||
not? [ rep generate-not-vector ] when
|
||||
] if ;
|
||||
|
||||
: sign-bit-mask ( rep -- byte-array )
|
||||
unsign-rep {
|
||||
{ char-16-rep [ uchar-array{
|
||||
HEX: 80 HEX: 80 HEX: 80 HEX: 80
|
||||
HEX: 80 HEX: 80 HEX: 80 HEX: 80
|
||||
HEX: 80 HEX: 80 HEX: 80 HEX: 80
|
||||
HEX: 80 HEX: 80 HEX: 80 HEX: 80
|
||||
} underlying>> ] }
|
||||
{ short-8-rep [ ushort-array{
|
||||
HEX: 8000 HEX: 8000 HEX: 8000 HEX: 8000
|
||||
HEX: 8000 HEX: 8000 HEX: 8000 HEX: 8000
|
||||
} underlying>> ] }
|
||||
{ int-4-rep [ uint-array{
|
||||
HEX: 8000,0000 HEX: 8000,0000
|
||||
HEX: 8000,0000 HEX: 8000,0000
|
||||
} underlying>> ] }
|
||||
{ longlong-2-rep [ ulonglong-array{
|
||||
HEX: 8000,0000,0000,0000
|
||||
HEX: 8000,0000,0000,0000
|
||||
} underlying>> ] }
|
||||
} case ;
|
||||
|
||||
:: (generate-minmax-compare-vector) ( src1 src2 rep orig-cc -- dst )
|
||||
orig-cc order-cc {
|
||||
{ cc< [ src1 src2 rep ^^max-vector src1 rep cc/= (generate-compare-vector) ] }
|
||||
{ cc<= [ src1 src2 rep ^^min-vector src1 rep cc= (generate-compare-vector) ] }
|
||||
{ cc> [ src1 src2 rep ^^min-vector src1 rep cc/= (generate-compare-vector) ] }
|
||||
{ cc>= [ src1 src2 rep ^^max-vector src1 rep cc= (generate-compare-vector) ] }
|
||||
} case ;
|
||||
|
||||
:: generate-compare-vector ( src1 src2 rep orig-cc -- dst )
|
||||
{
|
||||
{
|
||||
[ rep orig-cc %compare-vector-reps member? ]
|
||||
[ src1 src2 rep orig-cc (generate-compare-vector) ]
|
||||
}
|
||||
{
|
||||
[ rep %min-vector-reps member? ]
|
||||
[ src1 src2 rep orig-cc (generate-minmax-compare-vector) ]
|
||||
}
|
||||
{
|
||||
[ rep unsign-rep orig-cc %compare-vector-reps member? ]
|
||||
[
|
||||
rep sign-bit-mask ^^load-constant :> sign-bits
|
||||
src1 sign-bits rep ^^xor-vector
|
||||
src2 sign-bits rep ^^xor-vector
|
||||
rep unsign-rep orig-cc (generate-compare-vector)
|
||||
]
|
||||
}
|
||||
} cond ;
|
||||
|
||||
:: generate-unpack-vector-head ( src rep -- dst )
|
||||
{
|
||||
{
|
||||
[ rep %unpack-vector-head-reps member? ]
|
||||
[ src rep ^^unpack-vector-head ]
|
||||
}
|
||||
{
|
||||
[ rep unsigned-int-vector-rep? ]
|
||||
[
|
||||
rep ^^zero-vector :> zero
|
||||
src zero rep ^^merge-vector-head
|
||||
]
|
||||
}
|
||||
{
|
||||
[ rep widen-vector-rep %shr-vector-imm-reps member? ]
|
||||
[
|
||||
src src rep ^^merge-vector-head
|
||||
rep rep-component-type
|
||||
heap-size 8 * rep widen-vector-rep ^^shr-vector-imm
|
||||
]
|
||||
}
|
||||
[
|
||||
rep ^^zero-vector :> zero
|
||||
zero src rep cc> ^^compare-vector :> sign
|
||||
src sign rep ^^merge-vector-head
|
||||
]
|
||||
} cond ;
|
||||
|
||||
:: generate-unpack-vector-tail ( src rep -- dst )
|
||||
{
|
||||
{
|
||||
[ rep %unpack-vector-tail-reps member? ]
|
||||
[ src rep ^^unpack-vector-tail ]
|
||||
}
|
||||
{
|
||||
[ rep %unpack-vector-head-reps member? ]
|
||||
[
|
||||
src rep ^^tail>head-vector :> tail
|
||||
tail rep ^^unpack-vector-head
|
||||
]
|
||||
}
|
||||
{
|
||||
[ rep unsigned-int-vector-rep? ]
|
||||
[
|
||||
rep ^^zero-vector :> zero
|
||||
src zero rep ^^merge-vector-tail
|
||||
]
|
||||
}
|
||||
{
|
||||
[ rep widen-vector-rep %shr-vector-imm-reps member? ]
|
||||
[
|
||||
src src rep ^^merge-vector-tail
|
||||
rep rep-component-type
|
||||
heap-size 8 * rep widen-vector-rep ^^shr-vector-imm
|
||||
]
|
||||
}
|
||||
[
|
||||
rep ^^zero-vector :> zero
|
||||
zero src rep cc> ^^compare-vector :> sign
|
||||
src sign rep ^^merge-vector-tail
|
||||
]
|
||||
} cond ;
|
||||
|
||||
:: generate-load-neg-zero-vector ( rep -- dst )
|
||||
rep {
|
||||
{ float-4-rep [ float-array{ -0.0 -0.0 -0.0 -0.0 } underlying>> ^^load-constant ] }
|
||||
{ double-2-rep [ double-array{ -0.0 -0.0 } underlying>> ^^load-constant ] }
|
||||
[ drop rep ^^zero-vector ]
|
||||
} case ;
|
||||
|
||||
:: generate-neg-vector ( src rep -- dst )
|
||||
rep generate-load-neg-zero-vector
|
||||
src rep ^^sub-vector ;
|
||||
|
||||
:: generate-blend-vector ( mask true false rep -- dst )
|
||||
mask true rep ^^and-vector
|
||||
:: ^blend-vector ( mask true false rep -- dst )
|
||||
true mask rep ^^and-vector
|
||||
mask false rep ^^andn-vector
|
||||
rep ^^or-vector ;
|
||||
|
||||
:: generate-abs-vector ( src rep -- dst )
|
||||
: ^compare-vector ( src1 src2 rep cc -- dst )
|
||||
... ;
|
||||
|
||||
: ^widened-shr-vector-imm ( src shift rep -- dst )
|
||||
widen-vector-rep ^^shr-vector-imm ;
|
||||
|
||||
! intrinsic emitters
|
||||
|
||||
: emit-simd-v+ ( node -- )
|
||||
{
|
||||
{
|
||||
[ rep unsigned-int-vector-rep? ]
|
||||
[ src ]
|
||||
}
|
||||
{
|
||||
[ rep %abs-vector-reps member? ]
|
||||
[ src rep ^^abs-vector ]
|
||||
}
|
||||
{
|
||||
[ rep float-vector-rep? ]
|
||||
[
|
||||
rep generate-load-neg-zero-vector
|
||||
src rep ^^andn-vector
|
||||
]
|
||||
}
|
||||
[
|
||||
[ ^^add-vector ]
|
||||
} emit-vv-vector-op ;
|
||||
|
||||
: emit-simd-v- ( node -- )
|
||||
{
|
||||
[ ^^sub-vector ]
|
||||
} emit-vv-vector-op ;
|
||||
|
||||
: emit-simd-vneg ( node -- )
|
||||
{
|
||||
{ float-vector-rep [ [ ^load-neg-zero-vector ] [ ^^sub-vector ] bi ] }
|
||||
{ int-vector-rep [ [ ^^zero-vector ] [ ^^sub-vector ] bi ] }
|
||||
} emit-v-vector-op ;
|
||||
|
||||
: emit-simd-v+- ( node -- )
|
||||
{
|
||||
[ ^^add-sub-vector ]
|
||||
{ float-vector-rep [| src1 src2 rep |
|
||||
rep ^load-add-sub-vector :> signs
|
||||
src2 signs rep ^^xor-vector :> src2'
|
||||
src1 src2' rep ^^add-vector
|
||||
] }
|
||||
{ int-vector-rep [| src1 src2 rep |
|
||||
rep ^load-add-sub-vector :> signs
|
||||
src2 signs rep ^^xor-vector :> src2'
|
||||
src2' signs rep ^^sub-vector :> src2''
|
||||
src1 src2'' rep ^^add-vector
|
||||
] }
|
||||
} emit-vv-vector-op ;
|
||||
|
||||
: emit-simd-vs+ ( node -- )
|
||||
{
|
||||
{ float-vector-rep [ ^^add-vector ] }
|
||||
{ int-vector-rep [ ^^saturated-add-vector ] }
|
||||
} emit-vv-vector-op ;
|
||||
|
||||
: emit-simd-vs- ( node -- )
|
||||
{
|
||||
{ float-vector-rep [ ^^sub-vector ] }
|
||||
{ int-vector-rep [ ^^saturated-sub-vector ] }
|
||||
} emit-vv-vector-op ;
|
||||
|
||||
: emit-simd-vs* ( node -- )
|
||||
{
|
||||
{ float-vector-rep [ ^^mul-vector ] }
|
||||
{ int-vector-rep [ ^^saturated-mul-vector ] }
|
||||
} emit-vv-vector-op ;
|
||||
|
||||
: emit-simd-v* ( node -- )
|
||||
{
|
||||
[ ^^mul-vector ]
|
||||
} emit-vv-vector-op ;
|
||||
|
||||
: emit-simd-v/ ( node -- )
|
||||
{
|
||||
[ ^^div-vector ]
|
||||
} emit-vv-vector-op ;
|
||||
|
||||
: emit-simd-vmin ( node -- )
|
||||
{
|
||||
[ ^^min-vector ]
|
||||
[
|
||||
[ cc< ^compare-vector ]
|
||||
[ ^blend-vector ] 3bi
|
||||
]
|
||||
} emit-vv-vector-op ;
|
||||
|
||||
: emit-simd-vmax ( node -- )
|
||||
{
|
||||
[ ^^max-vector ]
|
||||
[
|
||||
[ cc> ^compare-vector ]
|
||||
[ ^blend-vector ] 3bi
|
||||
]
|
||||
} emit-vv-vector-op ;
|
||||
|
||||
: emit-simd-v. ( node -- )
|
||||
{
|
||||
[ ^^dot-vector ]
|
||||
{ float-vector-rep [| src1 src2 rep |
|
||||
|
||||
] }
|
||||
{ int-vector-rep [| src1 src2 rep |
|
||||
...
|
||||
] }
|
||||
} emit-vv-vector-op ;
|
||||
|
||||
: emit-simd-vsqrt ( node -- )
|
||||
{
|
||||
[ ^^sqrt-vector ]
|
||||
} emit-v-vector-op ;
|
||||
|
||||
: emit-simd-sum ( node -- )
|
||||
... ;
|
||||
|
||||
: emit-simd-vabs ( node -- )
|
||||
{
|
||||
{ unsigned-int-vector-rep [ drop ] }
|
||||
[ ^^abs-vector ]
|
||||
{ float-vector-rep [ [ ^load-neg-zero-vector ] [ swapd ^^andn-vector ] bi ] }
|
||||
{ int-vector-rep [| src rep |
|
||||
rep ^^zero-vector :> zero
|
||||
zero src rep ^^sub-vector :> -src
|
||||
zero src rep cc> ^^compare-vector :> sign
|
||||
sign -src src rep generate-blend-vector
|
||||
]
|
||||
} cond ;
|
||||
zero src rep cc> ^compare-vector :> sign
|
||||
sign -src src rep ^blend-vector
|
||||
] }
|
||||
} emit-v-vector-op ;
|
||||
|
||||
: generate-min-vector ( src1 src2 rep -- dst )
|
||||
dup %min-vector-reps member?
|
||||
[ ^^min-vector ] [
|
||||
[ cc< generate-compare-vector ]
|
||||
[ generate-blend-vector ] 3bi
|
||||
] if ;
|
||||
: emit-simd-vand ( node -- )
|
||||
{
|
||||
[ ^^and-vector ]
|
||||
} emit-vv-vector-op ;
|
||||
|
||||
: generate-max-vector ( src1 src2 rep -- dst )
|
||||
dup %max-vector-reps member?
|
||||
[ ^^max-vector ] [
|
||||
[ cc> generate-compare-vector ]
|
||||
[ generate-blend-vector ] 3bi
|
||||
] if ;
|
||||
: emit-simd-vandn ( node -- )
|
||||
{
|
||||
[ ^^andn-vector ]
|
||||
} emit-vv-vector-op ;
|
||||
|
||||
: emit-simd-vor ( node -- )
|
||||
{
|
||||
[ ^^or-vector ]
|
||||
} emit-vv-vector-op ;
|
||||
|
||||
: emit-simd-vxor ( node -- )
|
||||
{
|
||||
[ ^^xor-vector ]
|
||||
} emit-vv-vector-op ;
|
||||
|
||||
: emit-simd-vnot ( node -- )
|
||||
{
|
||||
[ ^^not-vector ]
|
||||
[ [ ^^fill-vector ] [ ^^xor-vector ] bi ]
|
||||
} emit-v-vector-op ;
|
||||
|
||||
: emit-simd-vlshift ( node -- )
|
||||
{
|
||||
[ ^^shl-vector ]
|
||||
} {
|
||||
[ ^^shl-vector-imm ]
|
||||
} emit-vn-or-vl-vector-op ;
|
||||
|
||||
: emit-simd-vrshift ( node -- )
|
||||
{
|
||||
[ ^^shr-vector ]
|
||||
} {
|
||||
[ ^^shr-vector-imm ]
|
||||
} emit-vn-or-vl-vector-op ;
|
||||
|
||||
: emit-simd-hlshift ( node -- )
|
||||
{
|
||||
[ ^^horizontal-shl-vector-imm ]
|
||||
} emit-vl-vector-op ;
|
||||
|
||||
: emit-simd-hrshift ( node -- )
|
||||
{
|
||||
[ ^^horizontal-shr-vector-imm ]
|
||||
} emit-vl-vector-op ;
|
||||
|
||||
: emit-simd-vshuffle-elements ( node -- )
|
||||
{
|
||||
[ ^^shuffle-vector-imm ]
|
||||
[ [ ^load-immediate-shuffle ] [ ^^shuffle-vector ] ]
|
||||
} emit-vl-vector-op ;
|
||||
|
||||
: emit-simd-vshuffle-bytes ( node -- )
|
||||
{
|
||||
[ ^^shuffle-vector ]
|
||||
} emit-vv-vector-op ;
|
||||
|
||||
: emit-simd-vmerge-head ( node -- )
|
||||
{
|
||||
[ ^^merge-vector-head ]
|
||||
} emit-vv-vector-op ;
|
||||
|
||||
: emit-simd-vmerge-tail ( node -- )
|
||||
{
|
||||
[ ^^merge-vector-tail ]
|
||||
} emit-vv-vector-op ;
|
||||
|
||||
: emit-simd-v<= ( node -- )
|
||||
[ cc<= ^compare-vector ] (emit-vv-vector-op) ;
|
||||
: emit-simd-v< ( node -- )
|
||||
[ cc< ^compare-vector ] (emit-vv-vector-op) ;
|
||||
: emit-simd-v= ( node -- )
|
||||
[ cc= ^compare-vector ] (emit-vv-vector-op) ;
|
||||
: emit-simd-v> ( node -- )
|
||||
[ cc> ^compare-vector ] (emit-vv-vector-op) ;
|
||||
: emit-simd-v>= ( node -- )
|
||||
[ cc>= ^compare-vector ] (emit-vv-vector-op) ;
|
||||
: emit-simd-vunordered? ( node -- )
|
||||
[ cc/<>= ^compare-vector ] (emit-vv-vector-op) ;
|
||||
|
||||
: emit-simd-vany? ( node -- )
|
||||
[ vcc-any ^test-vector ] (emit-vv-vector-op) ;
|
||||
: emit-simd-vall? ( node -- )
|
||||
[ vcc-all ^test-vector ] (emit-vv-vector-op) ;
|
||||
: emit-simd-vnone? ( node -- )
|
||||
[ vcc-none ^test-vector ] (emit-vv-vector-op) ;
|
||||
|
||||
: emit-simd-v>float ( node -- )
|
||||
{
|
||||
{ float-vector-rep [ drop ] }
|
||||
{ int-vector-rep [ ^^integer>float-vector ] }
|
||||
} emit-vv-vector-op ;
|
||||
|
||||
: emit-simd-v>integer ( node -- )
|
||||
{
|
||||
{ float-vector-rep [ ^^float>integer-vector ] }
|
||||
{ int-vector-rep [ dup ] }
|
||||
} emit-vv-vector-op ;
|
||||
|
||||
: emit-simd-vpack-signed ( node -- )
|
||||
{
|
||||
[ ^^signed-pack-vector ]
|
||||
} emit-vv-vector-op ;
|
||||
|
||||
: emit-simd-vpack-unsigned ( node -- )
|
||||
{
|
||||
[ ^^unsigned-pack-vector ]
|
||||
} emit-vv-vector-op ;
|
||||
|
||||
! XXX shr vector rep is widened!
|
||||
: emit-simd-vunpack-head ( node -- )
|
||||
{
|
||||
[ ^^unpack-vector-head ]
|
||||
{ unsigned-int-vector-rep [ [ ^^zero-vector ] [ ^^merge-vector-head ] bi ] }
|
||||
{ signed-int-vector-rep [| src rep |
|
||||
src src rep ^^merge-vector-head :> merged
|
||||
rep rep-component-type heap-size 8 * :> bits
|
||||
merged bits rep ^widened-shr-vector-imm
|
||||
] }
|
||||
{ signed-int-vector-rep [| src rep |
|
||||
rep ^^zero-vector :> zero
|
||||
zero src rep cc> ^compare-vector :> sign
|
||||
src sign rep ^^merge-vector-head
|
||||
] }
|
||||
} emit-v-vector-op ;
|
||||
|
||||
: emit-simd-vunpack-tail ( node -- )
|
||||
{
|
||||
[ ^^unpack-vector-tail ]
|
||||
[ [ ^^tail>head-vector ] [ ^^unpack-vector-head ] bi ]
|
||||
{ unsigned-int-vector-rep [ [ ^^zero-vector ] [ ^^merge-vector-tail ] bi ] }
|
||||
{ signed-int-vector-rep [| src rep |
|
||||
src src rep ^^merge-vector-tail :> merged
|
||||
rep rep-component-type heap-size 8 * :> bits
|
||||
merged bits rep widen-vector-rep ^widened-shr-vector-imm
|
||||
] }
|
||||
{ signed-int-vector-rep [| src rep |
|
||||
rep ^^zero-vector :> zero
|
||||
zero src rep cc> ^compare-vector :> sign
|
||||
src sign rep ^^merge-vector-tail
|
||||
] }
|
||||
} emit-v-vector-op ;
|
||||
|
||||
: emit-simd-with ( node -- )
|
||||
: emit-simd-gather-2 ( node -- )
|
||||
: emit-simd-gather-4 ( node -- )
|
||||
: emit-simd-select ( node -- )
|
||||
: emit-alien-vector ( node -- )
|
||||
: emit-set-alien-vector ( node -- )
|
||||
: emit-alien-vector-aligned ( node -- )
|
||||
: emit-set-alien-vector-aligned ( node -- )
|
||||
|
||||
: enable-simd ( -- )
|
||||
{
|
||||
{ math.vectors.simd:assert-positive [ drop ] }
|
||||
{ math.vectors.simd:(simd-v+) [ [ ^^add-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vs+) [ [ ^^saturated-add-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-v+-) [ [ ^^add-sub-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-v-) [ [ ^^sub-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vs-) [ [ ^^saturated-sub-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vneg) [ [ generate-neg-vector ] emit-unary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-v*) [ [ ^^mul-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vs*) [ [ ^^saturated-mul-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-v/) [ [ ^^div-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vmin) [ [ generate-min-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vmax) [ [ generate-max-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-v.) [ [ ^^dot-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vabs) [ [ generate-abs-vector ] emit-unary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vsqrt) [ [ ^^sqrt-vector ] emit-unary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vbitand) [ [ ^^and-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vbitandn) [ [ ^^andn-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vbitor) [ [ ^^or-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vbitxor) [ [ ^^xor-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vbitnot) [ [ generate-not-vector ] emit-unary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vand) [ [ ^^and-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vandn) [ [ ^^andn-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vor) [ [ ^^or-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vxor) [ [ ^^xor-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vnot) [ [ generate-not-vector ] emit-unary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-v<=) [ [ cc<= generate-compare-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-v<) [ [ cc< generate-compare-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-v=) [ [ cc= generate-compare-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-v>) [ [ cc> generate-compare-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-v>=) [ [ cc>= generate-compare-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vunordered?) [ [ cc/<>= generate-compare-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vany?) [ [ vcc-any ^^test-vector ] emit-unary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vall?) [ [ vcc-all ^^test-vector ] emit-unary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vnone?) [ [ vcc-none ^^test-vector ] emit-unary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vlshift) [ [ ^^shl-vector-imm ] [ ^^shl-vector ] emit-shift-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vrshift) [ [ ^^shr-vector-imm ] [ ^^shr-vector ] emit-shift-vector-op ] }
|
||||
{ math.vectors.simd:(simd-hlshift) [ [ ^^horizontal-shl-vector-imm ] emit-shift-vector-imm-op ] }
|
||||
{ math.vectors.simd:(simd-hrshift) [ [ ^^horizontal-shr-vector-imm ] emit-shift-vector-imm-op ] }
|
||||
{ math.vectors.simd:(simd-with) [ [ ^^with-vector ] emit-unary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-gather-2) [ emit-gather-vector-2 ] }
|
||||
{ math.vectors.simd:(simd-gather-4) [ emit-gather-vector-4 ] }
|
||||
{ math.vectors.simd:(simd-vshuffle-elements) [ emit-shuffle-vector ] }
|
||||
{ math.vectors.simd:(simd-vshuffle-bytes) [ emit-shuffle-vector-var ] }
|
||||
{ math.vectors.simd:(simd-vmerge-head) [ [ ^^merge-vector-head ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vmerge-tail) [ [ ^^merge-vector-tail ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-v>float) [ [ ^^integer>float-vector ] emit-unary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-v>integer) [ [ ^^float>integer-vector ] emit-unary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vpack-signed) [ [ ^^signed-pack-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vpack-unsigned) [ [ ^^unsigned-pack-vector ] emit-binary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vunpack-head) [ [ generate-unpack-vector-head ] emit-unary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-vunpack-tail) [ [ generate-unpack-vector-tail ] emit-unary-vector-op ] }
|
||||
{ math.vectors.simd:(simd-select) [ emit-select-vector ] }
|
||||
{ math.vectors.simd:(simd-sum) [ [ ^^horizontal-add-vector ] emit-unary-vector-op ] }
|
||||
{ math.vectors.simd:alien-vector [ emit-alien-vector ] }
|
||||
{ math.vectors.simd:set-alien-vector [ emit-set-alien-vector ] }
|
||||
{ (simd-v+) [ emit-simd-v+ ] }
|
||||
{ (simd-v-) [ emit-simd-v- ] }
|
||||
{ (simd-vneg) [ emit-simd-vneg ] }
|
||||
{ (simd-v+-) [ emit-simd-v+- ] }
|
||||
{ (simd-vs+) [ emit-simd-vs+ ] }
|
||||
{ (simd-vs-) [ emit-simd-vs- ] }
|
||||
{ (simd-vs*) [ emit-simd-vs* ] }
|
||||
{ (simd-v*) [ emit-simd-v* ] }
|
||||
{ (simd-v/) [ emit-simd-v/ ] }
|
||||
{ (simd-vmin) [ emit-simd-vmin ] }
|
||||
{ (simd-vmax) [ emit-simd-vmax ] }
|
||||
{ (simd-v.) [ emit-simd-v. ] }
|
||||
{ (simd-vsqrt) [ emit-simd-vsqrt ] }
|
||||
{ (simd-sum) [ emit-simd-sum ] }
|
||||
{ (simd-vabs) [ emit-simd-vabs ] }
|
||||
{ (simd-vbitand) [ emit-simd-vand ] }
|
||||
{ (simd-vbitandn) [ emit-simd-vandn ] }
|
||||
{ (simd-vbitor) [ emit-simd-vor ] }
|
||||
{ (simd-vbitxor) [ emit-simd-vxor ] }
|
||||
{ (simd-vbitnot) [ emit-simd-vnot ] }
|
||||
{ (simd-vand) [ emit-simd-vand ] }
|
||||
{ (simd-vandn) [ emit-simd-vandn ] }
|
||||
{ (simd-vor) [ emit-simd-vor ] }
|
||||
{ (simd-vxor) [ emit-simd-vxor ] }
|
||||
{ (simd-vnot) [ emit-simd-vnot ] }
|
||||
{ (simd-vlshift) [ emit-simd-vlshift ] }
|
||||
{ (simd-vrshift) [ emit-simd-vrshift ] }
|
||||
{ (simd-hlshift) [ emit-simd-hlshift ] }
|
||||
{ (simd-hrshift) [ emit-simd-hrshift ] }
|
||||
{ (simd-vshuffle-elements) [ emit-simd-vshuffle-elements ] }
|
||||
{ (simd-vshuffle-bytes) [ emit-simd-vshuffle-bytes ] }
|
||||
{ (simd-vmerge-head) [ emit-simd-vmerge-head ] }
|
||||
{ (simd-vmerge-tail) [ emit-simd-vmerge-tail ] }
|
||||
{ (simd-v<=) [ emit-simd-v<= ] }
|
||||
{ (simd-v<) [ emit-simd-v< ] }
|
||||
{ (simd-v=) [ emit-simd-v= ] }
|
||||
{ (simd-v>) [ emit-simd-v> ] }
|
||||
{ (simd-v>=) [ emit-simd-v>= ] }
|
||||
{ (simd-vunordered?) [ emit-simd-vunordered? ] }
|
||||
{ (simd-vany?) [ emit-simd-vany? ] }
|
||||
{ (simd-vall?) [ emit-simd-vall? ] }
|
||||
{ (simd-vnone?) [ emit-simd-vnone? ] }
|
||||
{ (simd-v>float) [ emit-simd-v>float ] }
|
||||
{ (simd-v>integer) [ emit-simd-v>integer ] }
|
||||
{ (simd-vpack-signed) [ emit-simd-vpack-signed ] }
|
||||
{ (simd-vpack-unsigned) [ emit-simd-vpack-unsigned ] }
|
||||
{ (simd-vunpack-head) [ emit-simd-vunpack-head ] }
|
||||
{ (simd-vunpack-tail) [ emit-simd-vunpack-tail ] }
|
||||
{ (simd-with) [ emit-simd-with ] }
|
||||
{ (simd-gather-2) [ emit-simd-gather-2 ] }
|
||||
{ (simd-gather-4) [ emit-simd-gather-4 ] }
|
||||
{ (simd-select) [ emit-simd-select ] }
|
||||
{ alien-vector [ emit-alien-vector ] }
|
||||
{ set-alien-vector [ emit-set-alien-vector ] }
|
||||
{ alien-vector-aligned [ emit-alien-vector ] }
|
||||
{ set-alien-vector-aligned [ emit-set-alien-vector ] }
|
||||
} enable-intrinsics ;
|
||||
|
||||
enable-simd
|
||||
|
|
|
|||
Loading…
Reference in New Issue