mirror of
https://github.com/xen-project/xen.git
synced 2026-10-28 18:21:39 +01:00
Jan Beulich
e291c4c3e1
Naming of some of the builtins isn't fully consistent with that of pre- existing ones, so there's a need for a new BR2() wrapper macro. With the tests providing some proof of proper functioning of the emulator code also enable use of the feature by guests, as there's no other infrastructure involved in enabling this ISA extension. Signed-off-by: Jan Beulich <jbeulich@suse.com> Acked-by: Andrew Cooper <andrew.cooper3@citrix.com> Acked-by: Henry Wang <Henry.Wang@arm.com> # CHANGELOG
1928 lines
69 KiB
C
1928 lines
69 KiB
C
#include "simd.h"
|
|
|
|
ENTRY(simd_test);
|
|
|
|
#if defined(__AVX512F__)
|
|
# define ALL_TRUE (~0ULL >> (64 - ELEM_COUNT))
|
|
# if VEC_SIZE == 4
|
|
# define eq(x, y) ({ \
|
|
float x_ = (x)[0]; \
|
|
float __attribute__((vector_size(16))) y_ = { (y)[0] }; \
|
|
unsigned short r_; \
|
|
asm ( "vcmpss $0, %1, %2, %0" : "=k" (r_) : "m" (x_), "v" (y_) ); \
|
|
r_ == 1; \
|
|
})
|
|
# elif VEC_SIZE == 8
|
|
# define eq(x, y) ({ \
|
|
double x_ = (x)[0]; \
|
|
double __attribute__((vector_size(16))) y_ = { (y)[0] }; \
|
|
unsigned short r_; \
|
|
asm ( "vcmpsd $0, %1, %2, %0" : "=k" (r_) : "m" (x_), "v" (y_) ); \
|
|
r_ == 1; \
|
|
})
|
|
# elif VEC_SIZE == 2
|
|
# define eq(x, y) ({ \
|
|
_Float16 x_ = (x)[0]; \
|
|
_Float16 __attribute__((vector_size(16))) y_ = { (y)[0] }; \
|
|
unsigned int r_; \
|
|
asm ( "vcmpsh $0, %1, %2, %0" : "=k" (r_) : "m" (x_), "v" (y_) ); \
|
|
r_ == 1; \
|
|
})
|
|
# elif FLOAT_SIZE == 4
|
|
/*
|
|
* gcc's (up to at least 8.2) __builtin_ia32_cmpps256_mask() has an anomaly in
|
|
* that its return type is QI rather than UQI, and hence the value would get
|
|
* sign-extended before comapring to ALL_TRUE. The same oddity does not matter
|
|
* for __builtin_ia32_cmppd256_mask(), as there only 4 bits are significant.
|
|
* Hence the extra " & ALL_TRUE".
|
|
*/
|
|
# define eq(x, y) ((BR(cmpps, _mask, x, y, 0, -1) & ALL_TRUE) == ALL_TRUE)
|
|
# elif FLOAT_SIZE == 8
|
|
# define eq(x, y) (BR(cmppd, _mask, x, y, 0, -1) == ALL_TRUE)
|
|
# elif FLOAT_SIZE == 2
|
|
# define eq(x, y) (B(cmpph, _mask, x, y, 0, -1) == ALL_TRUE)
|
|
# elif (INT_SIZE == 1 || UINT_SIZE == 1) && defined(__AVX512BW__)
|
|
# define eq(x, y) (B(pcmpeqb, _mask, (vqi_t)(x), (vqi_t)(y), -1) == ALL_TRUE)
|
|
# elif (INT_SIZE == 2 || UINT_SIZE == 2) && defined(__AVX512BW__)
|
|
# define eq(x, y) (B(pcmpeqw, _mask, (vhi_t)(x), (vhi_t)(y), -1) == ALL_TRUE)
|
|
# elif INT_SIZE == 4 || UINT_SIZE == 4
|
|
# define eq(x, y) (B(pcmpeqd, _mask, (vsi_t)(x), (vsi_t)(y), -1) == ALL_TRUE)
|
|
# elif INT_SIZE == 8 || UINT_SIZE == 8
|
|
# define eq(x, y) (B(pcmpeqq, _mask, (vdi_t)(x), (vdi_t)(y), -1) == ALL_TRUE)
|
|
# endif
|
|
#elif VEC_SIZE == 8 && defined(__SSE__)
|
|
# define to_bool(cmp) (__builtin_ia32_pmovmskb(cmp) == 0xff)
|
|
#elif VEC_SIZE == 16
|
|
# if defined(__AVX__) && defined(FLOAT_SIZE)
|
|
# if ELEM_SIZE == 4
|
|
# define to_bool(cmp) __builtin_ia32_vtestcps(cmp, (vec_t){} == 0)
|
|
# elif ELEM_SIZE == 8
|
|
# define to_bool(cmp) __builtin_ia32_vtestcpd(cmp, (vec_t){} == 0)
|
|
# endif
|
|
# elif defined(__SSE4_1__)
|
|
# define to_bool(cmp) __builtin_ia32_ptestc128(cmp, (vdi_t){} == 0)
|
|
# elif defined(__SSE__) && ELEM_SIZE == 4
|
|
# define to_bool(cmp) (__builtin_ia32_movmskps(cmp) == 0xf)
|
|
# elif defined(__SSE2__)
|
|
# if ELEM_SIZE == 8
|
|
# define to_bool(cmp) (__builtin_ia32_movmskpd(cmp) == 3)
|
|
# else
|
|
# define to_bool(cmp) (__builtin_ia32_pmovmskb128(cmp) == 0xffff)
|
|
# endif
|
|
# endif
|
|
#elif VEC_SIZE == 32
|
|
# if defined(__AVX2__)
|
|
# define to_bool(cmp) __builtin_ia32_ptestc256(cmp, (vdi_t){} == 0)
|
|
# elif defined(__AVX__) && ELEM_SIZE == 4
|
|
# define to_bool(cmp) (__builtin_ia32_movmskps256(cmp) == 0xff)
|
|
# elif defined(__AVX__) && ELEM_SIZE == 8
|
|
# define to_bool(cmp) (__builtin_ia32_movmskpd256(cmp) == 0xf)
|
|
# endif
|
|
#endif
|
|
|
|
#ifndef to_bool
|
|
static inline bool _to_bool(byte_vec_t bv)
|
|
{
|
|
unsigned int i;
|
|
|
|
for ( i = 0; i < VEC_SIZE; ++i )
|
|
if ( bv[i] != 0xff )
|
|
return false;
|
|
|
|
return true;
|
|
}
|
|
# define to_bool(cmp) _to_bool((byte_vec_t)(cmp))
|
|
#endif
|
|
|
|
#ifndef eq
|
|
# define eq(x, y) to_bool((x) == (y))
|
|
#endif
|
|
|
|
#if VEC_SIZE == FLOAT_SIZE
|
|
# define to_int(x) ({ int i_ = (x)[0]; touch(i_); ((vec_t){ i_ }); })
|
|
# ifdef __x86_64__
|
|
# define to_wint(x) ({ long l_ = (x)[0]; touch(l_); ((vec_t){ l_ }); })
|
|
# endif
|
|
# ifdef __AVX512F__
|
|
/*
|
|
* Sadly even gcc 9.x, at the time of writing, does not carry out at least
|
|
* uint -> FP conversions using VCVTUSI2S{S,D}, so we need to use builtins
|
|
* or inline assembly here. The full-vector parameter types of the builtins
|
|
* aren't very helpful for our purposes, so use inline assembly.
|
|
*/
|
|
# if FLOAT_SIZE == 4
|
|
# define to_u_int(type, x) ({ \
|
|
unsigned type u_; \
|
|
float __attribute__((vector_size(16))) t_; \
|
|
asm ( "vcvtss2usi %1, %0" : "=r" (u_) : "m" ((x)[0]) ); \
|
|
asm ( "vcvtusi2ss%z1 %1, %0, %0" : "=v" (t_) : "m" (u_) ); \
|
|
(vec_t){ t_[0] }; \
|
|
})
|
|
# elif FLOAT_SIZE == 8
|
|
# define to_u_int(type, x) ({ \
|
|
unsigned type u_; \
|
|
double __attribute__((vector_size(16))) t_; \
|
|
asm ( "vcvtsd2usi %1, %0" : "=r" (u_) : "m" ((x)[0]) ); \
|
|
asm ( "vcvtusi2sd%z1 %1, %0, %0" : "=v" (t_) : "m" (u_) ); \
|
|
(vec_t){ t_[0] }; \
|
|
})
|
|
# elif FLOAT_SIZE == 2
|
|
# define to_u_int(type, x) ({ \
|
|
unsigned type u_; \
|
|
_Float16 __attribute__((vector_size(16))) t_; \
|
|
asm ( "vcvtsh2usi %1, %0" : "=r" (u_) : "m" ((x)[0]) ); \
|
|
asm ( "vcvtusi2sh%z1 %1, %0, %0" : "=v" (t_) : "m" (u_) ); \
|
|
(vec_t){ t_[0] }; \
|
|
})
|
|
# endif
|
|
# define to_uint(x) to_u_int(int, x)
|
|
# ifdef __x86_64__
|
|
# define to_uwint(x) to_u_int(long, x)
|
|
# endif
|
|
# endif
|
|
#elif VEC_SIZE == 8 && FLOAT_SIZE == 4 && defined(__3dNOW__)
|
|
# define to_int(x) __builtin_ia32_pi2fd(__builtin_ia32_pf2id(x))
|
|
#elif defined(FLOAT_SIZE) && VEC_SIZE > FLOAT_SIZE && defined(__AVX512F__) && \
|
|
(VEC_SIZE == 64 || defined(__AVX512VL__))
|
|
# if FLOAT_SIZE == 4
|
|
# define to_int(x) BR(cvtdq2ps, _mask, BR(cvtps2dq, _mask, x, (vsi_t)undef(), ~0), undef(), ~0)
|
|
# define to_uint(x) BR(cvtudq2ps, _mask, BR(cvtps2udq, _mask, x, (vsi_t)undef(), ~0), undef(), ~0)
|
|
# ifdef __AVX512DQ__
|
|
# define to_w_int(x, s) ({ \
|
|
vsf_half_t t_ = low_half(x); \
|
|
vdi_t lo_, hi_; \
|
|
touch(t_); \
|
|
lo_ = BR(cvtps2 ## s ## qq, _mask, t_, (vdi_t)undef(), ~0); \
|
|
t_ = high_half(x); \
|
|
touch(t_); \
|
|
hi_ = BR(cvtps2 ## s ## qq, _mask, t_, (vdi_t)undef(), ~0); \
|
|
touch(lo_); touch(hi_); \
|
|
insert_half(insert_half(undef(), \
|
|
BR(cvt ## s ## qq2ps, _mask, lo_, (vsf_half_t){}, ~0), 0), \
|
|
BR(cvt ## s ## qq2ps, _mask, hi_, (vsf_half_t){}, ~0), 1); \
|
|
})
|
|
# define to_wint(x) to_w_int(x, )
|
|
# define to_uwint(x) to_w_int(x, u)
|
|
# endif
|
|
# elif FLOAT_SIZE == 8
|
|
# define to_int(x) B(cvtdq2pd, _mask, BR(cvtpd2dq, _mask, x, (vsi_half_t){}, ~0), undef(), ~0)
|
|
# define to_uint(x) B(cvtudq2pd, _mask, BR(cvtpd2udq, _mask, x, (vsi_half_t){}, ~0), undef(), ~0)
|
|
# ifdef __AVX512DQ__
|
|
# define to_wint(x) BR(cvtqq2pd, _mask, BR(cvtpd2qq, _mask, x, (vdi_t)undef(), ~0), undef(), ~0)
|
|
# define to_uwint(x) BR(cvtuqq2pd, _mask, BR(cvtpd2uqq, _mask, x, (vdi_t)undef(), ~0), undef(), ~0)
|
|
# endif
|
|
# elif FLOAT_SIZE == 2
|
|
# define to_int(x) BR2(vcvtw2ph, _mask, BR2(vcvtph2w, _mask, x, (vhi_t)undef(), ~0), undef(), ~0)
|
|
# define to_uint(x) BR2(vcvtuw2ph, _mask, BR2(vcvtph2uw, _mask, x, (vhi_t)undef(), ~0), undef(), ~0)
|
|
# if VEC_SIZE == 16
|
|
# define low_half(x) (x)
|
|
# define high_half(x) ((vec_t)B_(movhlps, , (vsf_t)undef(), (vsf_t)(x)))
|
|
# define insert_half(x, y, p) ((vec_t)((p) ? B_(movlhps, , (vsf_t)(x), (vsf_t)(y)) \
|
|
: B_(shufps, , (vsf_t)(y), (vsf_t)(x), 0b11100100)))
|
|
# elif VEC_SIZE == 32
|
|
# define _half(x, lh) ((vhf_half_t)B(extracti32x4_, _mask, (vsi_t)(x), lh, (vsi_half_t){}, ~0))
|
|
# define low_half(x) _half(x, 0)
|
|
# define high_half(x) _half(x, 1)
|
|
# define insert_half(x, y, p) \
|
|
((vec_t)B(inserti32x4_, _mask, (vsi_t)(x), (vsi_half_t)(y), p, (vsi_t)undef(), ~0))
|
|
# elif VEC_SIZE == 64
|
|
# define _half(x, lh) \
|
|
((vhf_half_t)__builtin_ia32_extracti64x4_mask((vdi_t)(x), lh, (vdi_half_t){}, ~0))
|
|
# define low_half(x) _half(x, 0)
|
|
# define high_half(x) _half(x, 1)
|
|
# define insert_half(x, y, p) \
|
|
((vec_t)__builtin_ia32_inserti64x4_mask((vdi_t)(x), (vdi_half_t)(y), p, (vdi_t)undef(), ~0))
|
|
# endif
|
|
# define to_w_int(x, s) ({ \
|
|
vhf_half_t t_ = low_half(x); \
|
|
vsi_t lo_, hi_; \
|
|
touch(t_); \
|
|
lo_ = BR2(vcvtph2 ## s ## dq, _mask, t_, (vsi_t)undef(), ~0); \
|
|
t_ = high_half(x); \
|
|
touch(t_); \
|
|
hi_ = BR2(vcvtph2 ## s ## dq, _mask, t_, (vsi_t)undef(), ~0); \
|
|
touch(lo_); touch(hi_); \
|
|
insert_half(insert_half(undef(), \
|
|
BR2(vcvt ## s ## dq2ph, _mask, lo_, (vhf_half_t){}, ~0), 0), \
|
|
BR2(vcvt ## s ## dq2ph, _mask, hi_, (vhf_half_t){}, ~0), 1); \
|
|
})
|
|
# define to_wint(x) to_w_int(x, )
|
|
# define to_uwint(x) to_w_int(x, u)
|
|
# endif
|
|
#elif VEC_SIZE == 16 && defined(__SSE2__)
|
|
# if FLOAT_SIZE == 4
|
|
# define to_int(x) __builtin_ia32_cvtdq2ps(__builtin_ia32_cvtps2dq(x))
|
|
# elif FLOAT_SIZE == 8
|
|
# define to_int(x) __builtin_ia32_cvtdq2pd(__builtin_ia32_cvtpd2dq(x))
|
|
# endif
|
|
#elif VEC_SIZE == 32 && defined(__AVX__)
|
|
# if FLOAT_SIZE == 4
|
|
# define to_int(x) __builtin_ia32_cvtdq2ps256(__builtin_ia32_cvtps2dq256(x))
|
|
# elif FLOAT_SIZE == 8
|
|
# define to_int(x) __builtin_ia32_cvtdq2pd256(__builtin_ia32_cvtpd2dq256(x))
|
|
# endif
|
|
#endif
|
|
|
|
#if VEC_SIZE == FLOAT_SIZE
|
|
# define scalar_1op(x, op) ({ \
|
|
typeof((x)[0]) __attribute__((vector_size(16))) r_; \
|
|
asm ( op : [out] "=&x" (r_) : [in] "m" (x) ); \
|
|
(vec_t){ r_[0] }; \
|
|
})
|
|
# define scalar_2op(x, y, op) ({ \
|
|
typeof((x)[0]) __attribute__((vector_size(16))) r_ = { x[0] }; \
|
|
asm ( op : [out] "=&x" (r_) : [in1] "[out]" (r_), [in2] "m" (y) ); \
|
|
(vec_t){ r_[0] }; \
|
|
})
|
|
#endif
|
|
|
|
#if VEC_SIZE == 16 && FLOAT_SIZE == 4 && defined(__SSE__)
|
|
# define low_half(x) (x)
|
|
# define high_half(x) B_(movhlps, , undef(), x)
|
|
/*
|
|
* GCC 7 (and perhaps earlier) report a bogus type mismatch for the conditional
|
|
* expression below. All works well with this no-op wrapper.
|
|
*/
|
|
static inline vec_t movlhps(vec_t x, vec_t y) {
|
|
return __builtin_ia32_movlhps(x, y);
|
|
}
|
|
# define insert_pair(x, y, p) \
|
|
((p) ? movlhps(x, y) \
|
|
: ({ vec_t t_ = (x); t_[0] = (y)[0]; t_[1] = (y)[1]; t_; }))
|
|
#endif
|
|
|
|
#if VEC_SIZE == 8 && FLOAT_SIZE == 4 && defined(__3dNOW_A__)
|
|
# define max __builtin_ia32_pfmax
|
|
# define min __builtin_ia32_pfmin
|
|
# define recip(x) ({ \
|
|
vec_t t_ = __builtin_ia32_pfrcp(x); \
|
|
touch(x); \
|
|
t_[1] = __builtin_ia32_pfrcp(__builtin_ia32_pswapdsf(x))[0]; \
|
|
touch(x); \
|
|
__builtin_ia32_pfrcpit2(__builtin_ia32_pfrcpit1(t_, x), t_); \
|
|
})
|
|
# define rsqrt(x) ({ \
|
|
vec_t t_ = __builtin_ia32_pfrsqrt(x); \
|
|
touch(x); \
|
|
t_[1] = __builtin_ia32_pfrsqrt(__builtin_ia32_pswapdsf(x))[0]; \
|
|
touch(x); \
|
|
__builtin_ia32_pfrcpit2(__builtin_ia32_pfrsqit1(__builtin_ia32_pfmul(t_, t_), x), t_); \
|
|
})
|
|
#elif defined(FLOAT_SIZE) && VEC_SIZE == FLOAT_SIZE && defined(__AVX512F__)
|
|
# if FLOAT_SIZE == 4
|
|
# define getexp(x) scalar_1op(x, "vgetexpss %[in], %[out], %[out]")
|
|
# define getmant(x) scalar_1op(x, "vgetmantss $0, %[in], %[out], %[out]")
|
|
# ifdef __AVX512ER__
|
|
# define recip(x) scalar_1op(x, "vrcp28ss %[in], %[out], %[out]")
|
|
# define rsqrt(x) scalar_1op(x, "vrsqrt28ss %[in], %[out], %[out]")
|
|
# else
|
|
# define recip(x) scalar_1op(x, "vrcp14ss %[in], %[out], %[out]")
|
|
# define rsqrt(x) scalar_1op(x, "vrsqrt14ss %[in], %[out], %[out]")
|
|
# endif
|
|
# define scale(x, y) scalar_2op(x, y, "vscalefss %[in2], %[in1], %[out]")
|
|
# define sqrt(x) scalar_1op(x, "vsqrtss %[in], %[out], %[out]")
|
|
# define trunc(x) scalar_1op(x, "vrndscaless $0b1011, %[in], %[out], %[out]")
|
|
# elif FLOAT_SIZE == 8
|
|
# define getexp(x) scalar_1op(x, "vgetexpsd %[in], %[out], %[out]")
|
|
# define getmant(x) scalar_1op(x, "vgetmantsd $0, %[in], %[out], %[out]")
|
|
# ifdef __AVX512ER__
|
|
# define recip(x) scalar_1op(x, "vrcp28sd %[in], %[out], %[out]")
|
|
# define rsqrt(x) scalar_1op(x, "vrsqrt28sd %[in], %[out], %[out]")
|
|
# else
|
|
# define recip(x) scalar_1op(x, "vrcp14sd %[in], %[out], %[out]")
|
|
# define rsqrt(x) scalar_1op(x, "vrsqrt14sd %[in], %[out], %[out]")
|
|
# endif
|
|
# define scale(x, y) scalar_2op(x, y, "vscalefsd %[in2], %[in1], %[out]")
|
|
# define sqrt(x) scalar_1op(x, "vsqrtsd %[in], %[out], %[out]")
|
|
# define trunc(x) scalar_1op(x, "vrndscalesd $0b1011, %[in], %[out], %[out]")
|
|
# elif FLOAT_SIZE == 2
|
|
# define getexp(x) scalar_1op(x, "vgetexpsh %[in], %[out], %[out]")
|
|
# define getmant(x) scalar_1op(x, "vgetmantsh $0, %[in], %[out], %[out]")
|
|
# define recip(x) scalar_1op(x, "vrcpsh %[in], %[out], %[out]")
|
|
# define rsqrt(x) scalar_1op(x, "vrsqrtsh %[in], %[out], %[out]")
|
|
# define scale(x, y) scalar_2op(x, y, "vscalefsh %[in2], %[in1], %[out]")
|
|
# define sqrt(x) scalar_1op(x, "vsqrtsh %[in], %[out], %[out]")
|
|
# define trunc(x) scalar_1op(x, "vrndscalesh $0b1011, %[in], %[out], %[out]")
|
|
# endif
|
|
#elif defined(FLOAT_SIZE) && defined(__AVX512F__) && \
|
|
(VEC_SIZE == 64 || defined(__AVX512VL__))
|
|
# if (ELEM_COUNT == 8 && ELEM_SIZE >= 4) /* vextractf{32,64}x4 */ || \
|
|
(ELEM_COUNT == 16 && ELEM_SIZE == 4 && defined(__AVX512DQ__)) /* vextractf32x8 */ || \
|
|
(ELEM_COUNT == 4 && ELEM_SIZE == 8 && defined(__AVX512DQ__)) /* vextractf64x2 */
|
|
# define _half(x, lh) ({ \
|
|
half_t t_; \
|
|
asm ( "vextractf%c[w]x%c[n] %[sel], %[s], %[d]" \
|
|
: [d] "=m" (t_) \
|
|
: [s] "v" (x), [sel] "i" (lh), \
|
|
[w] "i" (ELEM_SIZE * 8), [n] "i" (ELEM_COUNT / 2) ); \
|
|
t_; \
|
|
})
|
|
# define low_half(x) _half(x, 0)
|
|
# define high_half(x) _half(x, 1)
|
|
# endif
|
|
# if (ELEM_COUNT == 16 && ELEM_SIZE == 4) /* vextractf32x4 */ || \
|
|
(ELEM_COUNT == 8 && ELEM_SIZE == 8 && defined(__AVX512DQ__)) /* vextractf64x2 */
|
|
# define low_quarter(x) ({ \
|
|
quarter_t t_; \
|
|
asm ( "vextractf%c[w]x%c[n] $0, %[s], %[d]" \
|
|
: [d] "=m" (t_) \
|
|
: [s] "v" (x), [w] "i" (ELEM_SIZE * 8), [n] "i" (ELEM_COUNT / 4) ); \
|
|
t_; \
|
|
})
|
|
# endif
|
|
# if FLOAT_SIZE == 4
|
|
# define broadcast(x) ({ \
|
|
vec_t t_; \
|
|
asm ( "%{evex%} vbroadcastss %1, %0" \
|
|
: "=v" (t_) : "m" (*(float[1]){ x }) ); \
|
|
t_; \
|
|
})
|
|
# if VEC_SIZE >= 32 && defined(__AVX512DQ__)
|
|
# define broadcast_pair(x) ({ \
|
|
vec_t t_; \
|
|
asm ( "vbroadcastf32x2 %1, %0" : "=v" (t_) : "m" (x) ); \
|
|
t_; \
|
|
})
|
|
# endif
|
|
# if VEC_SIZE == 64 && defined(__AVX512DQ__)
|
|
# define broadcast_octet(x) B(broadcastf32x8_, _mask, x, undef(), ~0)
|
|
# define insert_octet(x, y, p) B(insertf32x8_, _mask, x, y, p, undef(), ~0)
|
|
# endif
|
|
# ifdef __AVX512DQ__
|
|
# define frac(x) B(reduceps, _mask, x, 0b00001011, undef(), ~0)
|
|
# endif
|
|
# define getexp(x) BR(getexpps, _mask, x, undef(), ~0)
|
|
# define getmant(x) BR(getmantps, _mask, x, 0, undef(), ~0)
|
|
# ifdef __AVX512DQ__
|
|
# define max(x, y) BR(rangeps, _mask, x, y, 0b0101, undef(), ~0)
|
|
# define min(x, y) BR(rangeps, _mask, x, y, 0b0100, undef(), ~0)
|
|
# else
|
|
# define max(x, y) BR_(maxps, _mask, x, y, undef(), ~0)
|
|
# define min(x, y) BR_(minps, _mask, x, y, undef(), ~0)
|
|
# endif
|
|
# define mix(x, y) B(blendmps_, _mask, x, y, (0b1010101010101010 & ALL_TRUE))
|
|
# define scale(x, y) BR(scalefps, _mask, x, y, undef(), ~0)
|
|
# if VEC_SIZE == 64 && defined(__AVX512ER__)
|
|
# define recip(x) BR(rcp28ps, _mask, x, undef(), ~0)
|
|
# define rsqrt(x) BR(rsqrt28ps, _mask, x, undef(), ~0)
|
|
# else
|
|
# define recip(x) B(rcp14ps, _mask, x, undef(), ~0)
|
|
# define rsqrt(x) B(rsqrt14ps, _mask, x, undef(), ~0)
|
|
# endif
|
|
# define shrink1(x) BR_(cvtpd2ps, _mask, (vdf_t)(x), (vsf_half_t){}, ~0)
|
|
# define sqrt(x) BR(sqrtps, _mask, x, undef(), ~0)
|
|
# define trunc(x) BR(rndscaleps_, _mask, x, 0b1011, undef(), ~0)
|
|
# define widen1(x) ((vec_t)BR(cvtps2pd, _mask, x, (vdf_t)undef(), ~0))
|
|
# if VEC_SIZE == 16
|
|
# define interleave_hi(x, y) B(unpckhps, _mask, x, y, undef(), ~0)
|
|
# define interleave_lo(x, y) B(unpcklps, _mask, x, y, undef(), ~0)
|
|
# define swap(x) B(shufps, _mask, x, x, 0b00011011, undef(), ~0)
|
|
# define swap2(x) B_(vpermilps, _mask, x, 0b00011011, undef(), ~0)
|
|
# else
|
|
# define broadcast_quartet(x) B(broadcastf32x4_, _mask, x, undef(), ~0)
|
|
# define insert_pair(x, y, p) \
|
|
B(insertf32x4_, _mask, x, \
|
|
/* Cast needed below to work around gcc 7.x quirk. */ \
|
|
(p) & 1 ? (typeof(y))__builtin_ia32_shufps(y, y, 0b01000100) : (y), \
|
|
(p) >> 1, x, 3 << ((p) * 2))
|
|
# define insert_quartet(x, y, p) B(insertf32x4_, _mask, x, y, p, undef(), ~0)
|
|
# define interleave_hi(x, y) B(vpermi2varps, _mask, x, interleave_hi, y, ~0)
|
|
# define interleave_lo(x, y) B(vpermt2varps, _mask, interleave_lo, x, y, ~0)
|
|
# define swap(x) ({ \
|
|
vec_t t_ = B(shuf_f32x4_, _mask, x, x, VEC_SIZE == 32 ? 0b01 : 0b00011011, undef(), ~0); \
|
|
B(shufps, _mask, t_, t_, 0b00011011, undef(), ~0); \
|
|
})
|
|
# define swap2(x) B(vpermilps, _mask, \
|
|
B(shuf_f32x4_, _mask, x, x, \
|
|
VEC_SIZE == 32 ? 0b01 : 0b00011011, undef(), ~0), \
|
|
0b00011011, undef(), ~0)
|
|
# endif
|
|
# elif FLOAT_SIZE == 8
|
|
# if VEC_SIZE >= 32
|
|
# define broadcast(x) ({ \
|
|
vec_t t_; \
|
|
asm ( "%{evex%} vbroadcastsd %1, %0" : "=v" (t_) \
|
|
: "m" (*(double[1]){ x }) ); \
|
|
t_; \
|
|
})
|
|
# else
|
|
# define broadcast(x) ({ \
|
|
vec_t t_; \
|
|
asm ( "%{evex%} vpbroadcastq %1, %0" \
|
|
: "=v" (t_) : "m" (*(double[1]){ x }) ); \
|
|
t_; \
|
|
})
|
|
# endif
|
|
# if VEC_SIZE >= 32 && defined(__AVX512DQ__)
|
|
# define broadcast_pair(x) B(broadcastf64x2_, _mask, x, undef(), ~0)
|
|
# define insert_pair(x, y, p) B(insertf64x2_, _mask, x, y, p, undef(), ~0)
|
|
# endif
|
|
# if VEC_SIZE == 64
|
|
# define broadcast_quartet(x) B(broadcastf64x4_, , x, undef(), ~0)
|
|
# define insert_quartet(x, y, p) B(insertf64x4_, _mask, x, y, p, undef(), ~0)
|
|
# endif
|
|
# ifdef __AVX512DQ__
|
|
# define frac(x) B(reducepd, _mask, x, 0b00001011, undef(), ~0)
|
|
# endif
|
|
# define getexp(x) BR(getexppd, _mask, x, undef(), ~0)
|
|
# define getmant(x) BR(getmantpd, _mask, x, 0, undef(), ~0)
|
|
# ifdef __AVX512DQ__
|
|
# define max(x, y) BR(rangepd, _mask, x, y, 0b0101, undef(), ~0)
|
|
# define min(x, y) BR(rangepd, _mask, x, y, 0b0100, undef(), ~0)
|
|
# else
|
|
# define max(x, y) BR_(maxpd, _mask, x, y, undef(), ~0)
|
|
# define min(x, y) BR_(minpd, _mask, x, y, undef(), ~0)
|
|
# endif
|
|
# define mix(x, y) B(blendmpd_, _mask, x, y, 0b10101010)
|
|
# define scale(x, y) BR(scalefpd, _mask, x, y, undef(), ~0)
|
|
# if VEC_SIZE == 64 && defined(__AVX512ER__)
|
|
# define recip(x) BR(rcp28pd, _mask, x, undef(), ~0)
|
|
# define rsqrt(x) BR(rsqrt28pd, _mask, x, undef(), ~0)
|
|
# else
|
|
# define recip(x) B(rcp14pd, _mask, x, undef(), ~0)
|
|
# define rsqrt(x) B(rsqrt14pd, _mask, x, undef(), ~0)
|
|
# endif
|
|
# define sqrt(x) BR(sqrtpd, _mask, x, undef(), ~0)
|
|
# define trunc(x) BR(rndscalepd_, _mask, x, 0b1011, undef(), ~0)
|
|
# if VEC_SIZE == 16
|
|
# define interleave_hi(x, y) B(unpckhpd, _mask, x, y, undef(), ~0)
|
|
# define interleave_lo(x, y) B(unpcklpd, _mask, x, y, undef(), ~0)
|
|
# define swap(x) B(shufpd, _mask, x, x, 0b01, undef(), ~0)
|
|
# define swap2(x) B_(vpermilpd, _mask, x, 0b01, undef(), ~0)
|
|
# else
|
|
# define interleave_hi(x, y) B(vpermi2varpd, _mask, x, interleave_hi, y, ~0)
|
|
# define interleave_lo(x, y) B(vpermt2varpd, _mask, interleave_lo, x, y, ~0)
|
|
# define swap(x) ({ \
|
|
vec_t t_ = B(shuf_f64x2_, _mask, x, x, VEC_SIZE == 32 ? 0b01 : 0b00011011, undef(), ~0); \
|
|
B(shufpd, _mask, t_, t_, 0b01010101, undef(), ~0); \
|
|
})
|
|
# define swap2(x) B(vpermilpd, _mask, \
|
|
B(shuf_f64x2_, _mask, x, x, \
|
|
VEC_SIZE == 32 ? 0b01 : 0b00011011, undef(), ~0), \
|
|
0b01010101, undef(), ~0)
|
|
# endif
|
|
# elif FLOAT_SIZE == 2
|
|
# define frac(x) BR2(reduceph, _mask, x, 0b00001011, undef(), ~0)
|
|
# define getexp(x) BR(getexpph, _mask, x, undef(), ~0)
|
|
# define getmant(x) BR(getmantph, _mask, x, 0, undef(), ~0)
|
|
# define max(x, y) BR2(maxph, _mask, x, y, undef(), ~0)
|
|
# define min(x, y) BR2(minph, _mask, x, y, undef(), ~0)
|
|
# define scale(x, y) BR2(scalefph, _mask, x, y, undef(), ~0)
|
|
# define recip(x) B(rcpph, _mask, x, undef(), ~0)
|
|
# define rsqrt(x) B(rsqrtph, _mask, x, undef(), ~0)
|
|
# define shrink1(x) BR2(vcvtps2phx, _mask, (vsf_t)(x), (vhf_half_t){}, ~0)
|
|
# define shrink2(x) BR2(vcvtpd2ph, _mask, (vdf_t)(x), (vhf_quarter_t){}, ~0)
|
|
# define sqrt(x) BR2(sqrtph, _mask, x, undef(), ~0)
|
|
# define trunc(x) BR2(rndscaleph, _mask, x, 0b1011, undef(), ~0)
|
|
# define widen1(x) ((vec_t)BR2(vcvtph2psx, _mask, x, (vsf_t)undef(), ~0))
|
|
# define widen2(x) ((vec_t)BR2(vcvtph2pd, _mask, x, (vdf_t)undef(), ~0))
|
|
# endif
|
|
#elif FLOAT_SIZE == 4 && defined(__SSE__)
|
|
# if VEC_SIZE == 32 && defined(__AVX__)
|
|
# if defined(__AVX2__)
|
|
# define broadcast(x) \
|
|
__builtin_ia32_vbroadcastss_ps256((float __attribute__((vector_size(16)))){ x })
|
|
# else
|
|
# define broadcast(x) ({ float t_ = (x); __builtin_ia32_vbroadcastss256(&t_); })
|
|
# endif
|
|
# define max(x, y) __builtin_ia32_maxps256(x, y)
|
|
# define min(x, y) __builtin_ia32_minps256(x, y)
|
|
# define recip(x) __builtin_ia32_rcpps256(x)
|
|
# define rsqrt(x) __builtin_ia32_rsqrtps256(x)
|
|
# define sqrt(x) __builtin_ia32_sqrtps256(x)
|
|
# define swap(x) ({ \
|
|
vec_t t_ = __builtin_ia32_vpermilps256(x, 0b00011011); \
|
|
__builtin_ia32_vperm2f128_ps256(t_, t_, 0b00000001); \
|
|
})
|
|
# ifdef __AVX2__
|
|
# define swap2(x) __builtin_ia32_permvarsf256(x, __builtin_ia32_cvtps2dq256(inv) - 1)
|
|
# else
|
|
# define swap2(x) ({ \
|
|
vec_t t_ = __builtin_ia32_vpermilvarps256(x, __builtin_ia32_cvtps2dq256(inv) - 1); \
|
|
__builtin_ia32_vperm2f128_ps256(t_, t_, 0b00000001); \
|
|
})
|
|
# endif
|
|
# elif VEC_SIZE == 16
|
|
# if defined(__AVX2__)
|
|
# define broadcast(x) __builtin_ia32_vbroadcastss_ps((vec_t){ x })
|
|
# elif defined(__AVX__)
|
|
# define broadcast(x) ({ float t_ = (x); __builtin_ia32_vbroadcastss(&t_); })
|
|
# endif
|
|
# define interleave_hi(x, y) __builtin_ia32_unpckhps(x, y)
|
|
# define interleave_lo(x, y) __builtin_ia32_unpcklps(x, y)
|
|
# define max(x, y) __builtin_ia32_maxps(x, y)
|
|
# define min(x, y) __builtin_ia32_minps(x, y)
|
|
# define recip(x) __builtin_ia32_rcpps(x)
|
|
# define rsqrt(x) __builtin_ia32_rsqrtps(x)
|
|
# define sqrt(x) __builtin_ia32_sqrtps(x)
|
|
# define swap(x) __builtin_ia32_shufps(x, x, 0b00011011)
|
|
# ifdef __AVX__
|
|
# define swap2(x) __builtin_ia32_vpermilvarps(x, __builtin_ia32_cvtps2dq(inv) - 1)
|
|
# endif
|
|
# elif VEC_SIZE == 4
|
|
# define recip(x) scalar_1op(x, "rcpss %[in], %[out]")
|
|
# define rsqrt(x) scalar_1op(x, "rsqrtss %[in], %[out]")
|
|
# define sqrt(x) scalar_1op(x, "sqrtss %[in], %[out]")
|
|
# endif
|
|
#elif FLOAT_SIZE == 8 && defined(__SSE2__)
|
|
# if VEC_SIZE == 32 && defined(__AVX__)
|
|
# if defined(__AVX2__)
|
|
# define broadcast(x) \
|
|
__builtin_ia32_vbroadcastsd_pd256((double __attribute__((vector_size(16)))){ x })
|
|
# else
|
|
# define broadcast(x) ({ double t_ = (x); __builtin_ia32_vbroadcastsd256(&t_); })
|
|
# endif
|
|
# define max(x, y) __builtin_ia32_maxpd256(x, y)
|
|
# define min(x, y) __builtin_ia32_minpd256(x, y)
|
|
# define recip(x) ({ \
|
|
float __attribute__((vector_size(16))) t_ = __builtin_ia32_cvtpd2ps256(x); \
|
|
t_ = __builtin_ia32_vextractf128_ps256( \
|
|
__builtin_ia32_rcpps256( \
|
|
__builtin_ia32_vbroadcastf128_ps256(&t_)), 0); \
|
|
__builtin_ia32_cvtps2pd256(t_); \
|
|
})
|
|
# define rsqrt(x) ({ \
|
|
float __attribute__((vector_size(16))) t1_ = __builtin_ia32_cvtpd2ps256(x); \
|
|
float __attribute__((vector_size(32))) t2_ = __builtin_ia32_vinsertf128_ps256((typeof(t2_)){}, t1_, 0); \
|
|
t2_ = __builtin_ia32_vinsertf128_ps256(t2_, t1_, 1); \
|
|
t1_ = __builtin_ia32_vextractf128_ps256(__builtin_ia32_rsqrtps256(t2_), 0); \
|
|
__builtin_ia32_cvtps2pd256(t1_); \
|
|
})
|
|
# define sqrt(x) __builtin_ia32_sqrtpd256(x)
|
|
# define swap(x) ({ \
|
|
vec_t t_ = __builtin_ia32_vpermilpd256(x, 0b00000101); \
|
|
__builtin_ia32_vperm2f128_pd256(t_, t_, 0b00000001); \
|
|
})
|
|
# ifdef __AVX2__
|
|
# define swap2(x) __builtin_ia32_permdf256(x, 0b00011011)
|
|
# endif
|
|
# elif VEC_SIZE == 16
|
|
# define interleave_hi(x, y) __builtin_ia32_unpckhpd(x, y)
|
|
# define interleave_lo(x, y) __builtin_ia32_unpcklpd(x, y)
|
|
# define max(x, y) __builtin_ia32_maxpd(x, y)
|
|
# define min(x, y) __builtin_ia32_minpd(x, y)
|
|
# define recip(x) __builtin_ia32_cvtps2pd(__builtin_ia32_rcpps(__builtin_ia32_cvtpd2ps(x)))
|
|
# define rsqrt(x) __builtin_ia32_cvtps2pd(__builtin_ia32_rsqrtps(__builtin_ia32_cvtpd2ps(x)))
|
|
# define sqrt(x) __builtin_ia32_sqrtpd(x)
|
|
# define swap(x) __builtin_ia32_shufpd(x, x, 0b01)
|
|
# ifdef __AVX__
|
|
# define swap2(x) __builtin_ia32_vpermilvarpd(x, __builtin_ia32_pmovsxdq128( \
|
|
__builtin_ia32_cvtpd2dq(inv) - 1) << 1)
|
|
# endif
|
|
# elif VEC_SIZE == 8
|
|
# define recip(x) scalar_1op(x, "cvtsd2ss %[in], %[out]; rcpss %[out], %[out]; cvtss2sd %[out], %[out]")
|
|
# define rsqrt(x) scalar_1op(x, "cvtsd2ss %[in], %[out]; rsqrtss %[out], %[out]; cvtss2sd %[out], %[out]")
|
|
# define sqrt(x) scalar_1op(x, "sqrtsd %[in], %[out]")
|
|
# endif
|
|
#endif
|
|
#if (INT_SIZE == 4 || UINT_SIZE == 4 || INT_SIZE == 8 || UINT_SIZE == 8) && \
|
|
defined(__AVX512F__) && (VEC_SIZE == 64 || defined(__AVX512VL__))
|
|
# if ELEM_COUNT == 8 /* vextracti{32,64}x4 */ || \
|
|
(ELEM_COUNT == 16 && ELEM_SIZE == 4 && defined(__AVX512DQ__)) /* vextracti32x8 */ || \
|
|
(ELEM_COUNT == 4 && ELEM_SIZE == 8 && defined(__AVX512DQ__)) /* vextracti64x2 */
|
|
# define low_half(x) ({ \
|
|
half_t t_; \
|
|
asm ( "vextracti%c[w]x%c[n] $0, %[s], %[d]" \
|
|
: [d] "=m" (t_) \
|
|
: [s] "v" (x), [w] "i" (ELEM_SIZE * 8), [n] "i" (ELEM_COUNT / 2) ); \
|
|
t_; \
|
|
})
|
|
# endif
|
|
# if (ELEM_COUNT == 16 && ELEM_SIZE == 4) /* vextracti32x4 */ || \
|
|
(ELEM_COUNT == 8 && ELEM_SIZE == 8 && defined(__AVX512DQ__)) /* vextracti64x2 */
|
|
# define low_quarter(x) ({ \
|
|
quarter_t t_; \
|
|
asm ( "vextracti%c[w]x%c[n] $0, %[s], %[d]" \
|
|
: [d] "=m" (t_) \
|
|
: [s] "v" (x), [w] "i" (ELEM_SIZE * 8), [n] "i" (ELEM_COUNT / 4) ); \
|
|
t_; \
|
|
})
|
|
# endif
|
|
# if INT_SIZE == 4 || UINT_SIZE == 4
|
|
# define broadcast(x) ({ \
|
|
vec_t t_; \
|
|
asm ( "%{evex%} vpbroadcastd %1, %0" \
|
|
: "=v" (t_) : "m" (*(int[1]){ x }) ); \
|
|
t_; \
|
|
})
|
|
# define broadcast2(x) ({ \
|
|
vec_t t_; \
|
|
asm ( "vpbroadcastd %k1, %0" : "=v" (t_) : "r" (x) ); \
|
|
t_; \
|
|
})
|
|
# ifdef __AVX512DQ__
|
|
# define broadcast_pair(x) ({ \
|
|
vec_t t_; \
|
|
asm ( "vbroadcasti32x2 %1, %0" : "=v" (t_) : "m" (x) ); \
|
|
t_; \
|
|
})
|
|
# endif
|
|
# if VEC_SIZE == 64 && defined(__AVX512DQ__)
|
|
# define broadcast_octet(x) ((vec_t)B(broadcasti32x8_, _mask, (vsi_octet_t)(x), (vsi_t)undef(), ~0))
|
|
# define insert_octet(x, y, p) ((vec_t)B(inserti32x8_, _mask, (vsi_t)(x), (vsi_octet_t)(y), p, (vsi_t)undef(), ~0))
|
|
# endif
|
|
# if VEC_SIZE == 16
|
|
# define interleave_hi(x, y) ((vec_t)B(punpckhdq, _mask, (vsi_t)(x), (vsi_t)(y), (vsi_t)undef(), ~0))
|
|
# define interleave_lo(x, y) ((vec_t)B(punpckldq, _mask, (vsi_t)(x), (vsi_t)(y), (vsi_t)undef(), ~0))
|
|
# define swap(x) ((vec_t)B(pshufd, _mask, (vsi_t)(x), 0b00011011, (vsi_t)undef(), ~0))
|
|
# else
|
|
# define broadcast_quartet(x) ((vec_t)B(broadcasti32x4_, _mask, (vsi_quartet_t)(x), (vsi_t)undef(), ~0))
|
|
# define insert_pair(x, y, p) \
|
|
(vec_t)(B(inserti32x4_, _mask, (vsi_t)(x), \
|
|
/* First cast needed below to work around gcc 7.x quirk. */ \
|
|
(p) & 1 ? (vsi_pair_t)__builtin_ia32_pshufd((vsi_pair_t)(y), 0b01000100) \
|
|
: (vsi_pair_t)(y), \
|
|
(p) >> 1, (vsi_t)(x), 3 << ((p) * 2)))
|
|
# define insert_quartet(x, y, p) ((vec_t)B(inserti32x4_, _mask, (vsi_t)(x), (vsi_quartet_t)(y), p, (vsi_t)undef(), ~0))
|
|
# define interleave_hi(x, y) ((vec_t)B(vpermi2vard, _mask, (vsi_t)(x), interleave_hi, (vsi_t)(y), ~0))
|
|
# define interleave_lo(x, y) ((vec_t)B(vpermt2vard, _mask, interleave_lo, (vsi_t)(x), (vsi_t)(y), ~0))
|
|
# define swap(x) ((vec_t)B(pshufd, _mask, \
|
|
B(shuf_i32x4_, _mask, (vsi_t)(x), (vsi_t)(x), \
|
|
VEC_SIZE == 32 ? 0b01 : 0b00011011, (vsi_t)undef(), ~0), \
|
|
0b00011011, (vsi_t)undef(), ~0))
|
|
# define swap2(x) ((vec_t)B_(permvarsi, _mask, (vsi_t)(x), (vsi_t)(inv - 1), (vsi_t)undef(), ~0))
|
|
# endif
|
|
# define mix(x, y) ((vec_t)B(blendmd_, _mask, (vsi_t)(x), (vsi_t)(y), \
|
|
(0b1010101010101010 & ((1 << ELEM_COUNT) - 1))))
|
|
# define rotr(x, n) ((vec_t)B(alignd, _mask, (vsi_t)(x), (vsi_t)(x), n, (vsi_t)undef(), ~0))
|
|
# define shrink1(x) ((half_t)B(pmovqd, _mask, (vdi_t)(x), (vsi_half_t){}, ~0))
|
|
# elif INT_SIZE == 8 || UINT_SIZE == 8
|
|
# define broadcast(x) ({ \
|
|
vec_t t_; \
|
|
asm ( "%{evex%} vpbroadcastq %1, %0" \
|
|
: "=v" (t_) : "m" (*(long long[1]){ x }) ); \
|
|
t_; \
|
|
})
|
|
# ifdef __x86_64__
|
|
# define broadcast2(x) ({ \
|
|
vec_t t_; \
|
|
asm ( "vpbroadcastq %1, %0" : "=v" (t_) : "r" ((x) + 0ULL) ); \
|
|
t_; \
|
|
})
|
|
# endif
|
|
# if VEC_SIZE >= 32 && defined(__AVX512DQ__)
|
|
# define broadcast_pair(x) ((vec_t)B(broadcasti64x2_, _mask, (vdi_pair_t)(x), (vdi_t)undef(), ~0))
|
|
# define insert_pair(x, y, p) ((vec_t)B(inserti64x2_, _mask, (vdi_t)(x), (vdi_pair_t)(y), p, (vdi_t)undef(), ~0))
|
|
# endif
|
|
# if VEC_SIZE == 64
|
|
# define broadcast_quartet(x) ((vec_t)B(broadcasti64x4_, , (vdi_quartet_t)(x), (vdi_t)undef(), ~0))
|
|
# define insert_quartet(x, y, p) ((vec_t)B(inserti64x4_, _mask, (vdi_t)(x), (vdi_quartet_t)(y), p, (vdi_t)undef(), ~0))
|
|
# endif
|
|
# if VEC_SIZE == 16
|
|
# define interleave_hi(x, y) ((vec_t)B(punpckhqdq, _mask, (vdi_t)(x), (vdi_t)(y), (vdi_t)undef(), ~0))
|
|
# define interleave_lo(x, y) ((vec_t)B(punpcklqdq, _mask, (vdi_t)(x), (vdi_t)(y), (vdi_t)undef(), ~0))
|
|
# define swap(x) ((vec_t)B(pshufd, _mask, (vsi_t)(x), 0b01001110, (vsi_t)undef(), ~0))
|
|
# else
|
|
# define interleave_hi(x, y) ((vec_t)B(vpermi2varq, _mask, (vdi_t)(x), interleave_hi, (vdi_t)(y), ~0))
|
|
# define interleave_lo(x, y) ((vec_t)B(vpermt2varq, _mask, interleave_lo, (vdi_t)(x), (vdi_t)(y), ~0))
|
|
# define swap(x) ((vec_t)B(pshufd, _mask, \
|
|
(vsi_t)B(shuf_i64x2_, _mask, (vdi_t)(x), (vdi_t)(x), \
|
|
VEC_SIZE == 32 ? 0b01 : 0b00011011, (vdi_t)undef(), ~0), \
|
|
0b01001110, (vsi_t)undef(), ~0))
|
|
# define swap2(x) ((vec_t)B(permvardi, _mask, (vdi_t)(x), (vdi_t)(inv - 1), (vdi_t)undef(), ~0))
|
|
# endif
|
|
# define mix(x, y) ((vec_t)B(blendmq_, _mask, (vdi_t)(x), (vdi_t)(y), 0b10101010))
|
|
# define rotr(x, n) ((vec_t)B(alignq, _mask, (vdi_t)(x), (vdi_t)(x), n, (vdi_t)undef(), ~0))
|
|
# if VEC_SIZE == 32
|
|
# define swap3(x) ((vec_t)B_(permdi, _mask, (vdi_t)(x), 0b00011011, (vdi_t)undef(), ~0))
|
|
# elif VEC_SIZE == 64
|
|
# define swap3(x) ({ \
|
|
vdi_t t_ = B_(permdi, _mask, (vdi_t)(x), 0b00011011, (vdi_t)undef(), ~0); \
|
|
B(shuf_i64x2_, _mask, t_, t_, 0b01001110, (vdi_t)undef(), ~0); \
|
|
})
|
|
# endif
|
|
# endif
|
|
# if INT_SIZE == 4
|
|
# define abs(x) B(pabsd, _mask, x, undef(), ~0)
|
|
# define max(x, y) B(pmaxsd, _mask, x, y, undef(), ~0)
|
|
# define min(x, y) B(pminsd, _mask, x, y, undef(), ~0)
|
|
# define mul_full(x, y) ((vec_t)B(pmuldq, _mask, x, y, (vdi_t)undef(), ~0))
|
|
# define widen1(x) ((vec_t)B(pmovsxdq, _mask, x, (vdi_t)undef(), ~0))
|
|
# elif UINT_SIZE == 4
|
|
# define max(x, y) ((vec_t)B(pmaxud, _mask, (vsi_t)(x), (vsi_t)(y), (vsi_t)undef(), ~0))
|
|
# define min(x, y) ((vec_t)B(pminud, _mask, (vsi_t)(x), (vsi_t)(y), (vsi_t)undef(), ~0))
|
|
# define mul_full(x, y) ((vec_t)B(pmuludq, _mask, (vsi_t)(x), (vsi_t)(y), (vdi_t)undef(), ~0))
|
|
# define widen1(x) ((vec_t)B(pmovzxdq, _mask, (vsi_half_t)(x), (vdi_t)undef(), ~0))
|
|
# elif INT_SIZE == 8
|
|
# define abs(x) ((vec_t)B(pabsq, _mask, (vdi_t)(x), (vdi_t)undef(), ~0))
|
|
# define max(x, y) ((vec_t)B(pmaxsq, _mask, (vdi_t)(x), (vdi_t)(y), (vdi_t)undef(), ~0))
|
|
# define min(x, y) ((vec_t)B(pminsq, _mask, (vdi_t)(x), (vdi_t)(y), (vdi_t)undef(), ~0))
|
|
# elif UINT_SIZE == 8
|
|
# define max(x, y) ((vec_t)B(pmaxuq, _mask, (vdi_t)(x), (vdi_t)(y), (vdi_t)undef(), ~0))
|
|
# define min(x, y) ((vec_t)B(pminuq, _mask, (vdi_t)(x), (vdi_t)(y), (vdi_t)undef(), ~0))
|
|
# endif
|
|
#elif (INT_SIZE == 1 || UINT_SIZE == 1 || INT_SIZE == 2 || UINT_SIZE == 2) && \
|
|
defined(__AVX512BW__) && (VEC_SIZE == 64 || defined(__AVX512VL__))
|
|
# if INT_SIZE == 1 || UINT_SIZE == 1
|
|
# define broadcast(x) ({ \
|
|
vec_t t_; \
|
|
asm ( "%{evex%} vpbroadcastb %1, %0" \
|
|
: "=v" (t_) : "m" (*(char[1]){ x }) ); \
|
|
t_; \
|
|
})
|
|
# define broadcast2(x) ({ \
|
|
vec_t t_; \
|
|
asm ( "vpbroadcastb %k1, %0" : "=v" (t_) : "r" (x) ); \
|
|
t_; \
|
|
})
|
|
# if VEC_SIZE == 16
|
|
# define interleave_hi(x, y) ((vec_t)B(punpckhbw, _mask, (vqi_t)(x), (vqi_t)(y), (vqi_t)undef(), ~0))
|
|
# define interleave_lo(x, y) ((vec_t)B(punpcklbw, _mask, (vqi_t)(x), (vqi_t)(y), (vqi_t)undef(), ~0))
|
|
# define rotr(x, n) ((vec_t)B(palignr, _mask, (vdi_t)(x), (vdi_t)(x), (n) * 8, (vdi_t)undef(), ~0))
|
|
# define swap(x) ((vec_t)B(pshufb, _mask, (vqi_t)(x), (vqi_t)(inv - 1), (vqi_t)undef(), ~0))
|
|
# elif defined(__AVX512VBMI__)
|
|
# define interleave_hi(x, y) ((vec_t)B(vpermi2varqi, _mask, (vqi_t)(x), interleave_hi, (vqi_t)(y), ~0))
|
|
# define interleave_lo(x, y) ((vec_t)B(vpermt2varqi, _mask, interleave_lo, (vqi_t)(x), (vqi_t)(y), ~0))
|
|
# endif
|
|
# define mix(x, y) ((vec_t)B(blendmb_, _mask, (vqi_t)(x), (vqi_t)(y), \
|
|
(0b1010101010101010101010101010101010101010101010101010101010101010LL & ALL_TRUE)))
|
|
# define shrink1(x) ((half_t)B(pmovwb, _mask, (vhi_t)(x), (vqi_half_t){}, ~0))
|
|
# define shrink2(x) ((quarter_t)B(pmovdb, _mask, (vsi_t)(x), (vqi_quarter_t){}, ~0))
|
|
# define shrink3(x) ((eighth_t)B(pmovqb, _mask, (vdi_t)(x), (vqi_eighth_t){}, ~0))
|
|
# ifdef __AVX512VBMI__
|
|
# define swap2(x) ((vec_t)B(permvarqi, _mask, (vqi_t)(x), (vqi_t)(inv - 1), (vqi_t)undef(), ~0))
|
|
# endif
|
|
# elif INT_SIZE == 2 || UINT_SIZE == 2
|
|
# define broadcast(x) ({ \
|
|
vec_t t_; \
|
|
asm ( "%{evex%} vpbroadcastw %1, %0" \
|
|
: "=v" (t_) : "m" (*(short[1]){ x }) ); \
|
|
t_; \
|
|
})
|
|
# define broadcast2(x) ({ \
|
|
vec_t t_; \
|
|
asm ( "vpbroadcastw %k1, %0" : "=v" (t_) : "r" (x) ); \
|
|
t_; \
|
|
})
|
|
# if VEC_SIZE == 16
|
|
# define interleave_hi(x, y) ((vec_t)B(punpckhwd, _mask, (vhi_t)(x), (vhi_t)(y), (vhi_t)undef(), ~0))
|
|
# define interleave_lo(x, y) ((vec_t)B(punpcklwd, _mask, (vhi_t)(x), (vhi_t)(y), (vhi_t)undef(), ~0))
|
|
# define rotr(x, n) ((vec_t)B(palignr, _mask, (vdi_t)(x), (vdi_t)(x), (n) * 16, (vdi_t)undef(), ~0))
|
|
# define swap(x) ((vec_t)B(pshufd, _mask, \
|
|
(vsi_t)B(pshufhw, _mask, \
|
|
B(pshuflw, _mask, (vhi_t)(x), 0b00011011, (vhi_t)undef(), ~0), \
|
|
0b00011011, (vhi_t)undef(), ~0), \
|
|
0b01001110, (vsi_t)undef(), ~0))
|
|
# else
|
|
# define interleave_hi(x, y) ((vec_t)B(vpermi2varhi, _mask, (vhi_t)(x), interleave_hi, (vhi_t)(y), ~0))
|
|
# define interleave_lo(x, y) ((vec_t)B(vpermt2varhi, _mask, interleave_lo, (vhi_t)(x), (vhi_t)(y), ~0))
|
|
# endif
|
|
# define mix(x, y) ((vec_t)B(blendmw_, _mask, (vhi_t)(x), (vhi_t)(y), \
|
|
(0b10101010101010101010101010101010 & ALL_TRUE)))
|
|
# define shrink1(x) ((half_t)B(pmovdw, _mask, (vsi_t)(x), (vhi_half_t){}, ~0))
|
|
# define shrink2(x) ((quarter_t)B(pmovqw, _mask, (vdi_t)(x), (vhi_quarter_t){}, ~0))
|
|
# define swap2(x) ((vec_t)B(permvarhi, _mask, (vhi_t)(x), (vhi_t)(inv - 1), (vhi_t)undef(), ~0))
|
|
# endif
|
|
# if INT_SIZE == 1
|
|
# define abs(x) ((vec_t)B(pabsb, _mask, (vqi_t)(x), (vqi_t)undef(), ~0))
|
|
# define max(x, y) ((vec_t)B(pmaxsb, _mask, (vqi_t)(x), (vqi_t)(y), (vqi_t)undef(), ~0))
|
|
# define min(x, y) ((vec_t)B(pminsb, _mask, (vqi_t)(x), (vqi_t)(y), (vqi_t)undef(), ~0))
|
|
# define widen1(x) ((vec_t)B(pmovsxbw, _mask, (vqi_half_t)(x), (vhi_t)undef(), ~0))
|
|
# define widen2(x) ((vec_t)B(pmovsxbd, _mask, (vqi_quarter_t)(x), (vsi_t)undef(), ~0))
|
|
# define widen3(x) ((vec_t)B(pmovsxbq, _mask, (vqi_eighth_t)(x), (vdi_t)undef(), ~0))
|
|
# elif UINT_SIZE == 1
|
|
# define max(x, y) ((vec_t)B(pmaxub, _mask, (vqi_t)(x), (vqi_t)(y), (vqi_t)undef(), ~0))
|
|
# define min(x, y) ((vec_t)B(pminub, _mask, (vqi_t)(x), (vqi_t)(y), (vqi_t)undef(), ~0))
|
|
# define widen1(x) ((vec_t)B(pmovzxbw, _mask, (vqi_half_t)(x), (vhi_t)undef(), ~0))
|
|
# define widen2(x) ((vec_t)B(pmovzxbd, _mask, (vqi_quarter_t)(x), (vsi_t)undef(), ~0))
|
|
# define widen3(x) ((vec_t)B(pmovzxbq, _mask, (vqi_eighth_t)(x), (vdi_t)undef(), ~0))
|
|
# elif INT_SIZE == 2
|
|
# define abs(x) B(pabsw, _mask, x, undef(), ~0)
|
|
# define max(x, y) B(pmaxsw, _mask, x, y, undef(), ~0)
|
|
# define min(x, y) B(pminsw, _mask, x, y, undef(), ~0)
|
|
# define mul_hi(x, y) B(pmulhw, _mask, x, y, undef(), ~0)
|
|
# define widen1(x) ((vec_t)B(pmovsxwd, _mask, x, (vsi_t)undef(), ~0))
|
|
# define widen2(x) ((vec_t)B(pmovsxwq, _mask, x, (vdi_t)undef(), ~0))
|
|
# elif UINT_SIZE == 2
|
|
# define max(x, y) ((vec_t)B(pmaxuw, _mask, (vhi_t)(x), (vhi_t)(y), (vhi_t)undef(), ~0))
|
|
# define min(x, y) ((vec_t)B(pminuw, _mask, (vhi_t)(x), (vhi_t)(y), (vhi_t)undef(), ~0))
|
|
# define mul_hi(x, y) ((vec_t)B(pmulhuw, _mask, (vhi_t)(x), (vhi_t)(y), (vhi_t)undef(), ~0))
|
|
# define widen1(x) ((vec_t)B(pmovzxwd, _mask, (vhi_half_t)(x), (vsi_t)undef(), ~0))
|
|
# define widen2(x) ((vec_t)B(pmovzxwq, _mask, (vhi_quarter_t)(x), (vdi_t)undef(), ~0))
|
|
# endif
|
|
#elif VEC_SIZE == 16 && defined(__SSE2__)
|
|
# if INT_SIZE == 1 || UINT_SIZE == 1
|
|
# define interleave_hi(x, y) ((vec_t)__builtin_ia32_punpckhbw128((vqi_t)(x), (vqi_t)(y)))
|
|
# define interleave_lo(x, y) ((vec_t)__builtin_ia32_punpcklbw128((vqi_t)(x), (vqi_t)(y)))
|
|
# elif INT_SIZE == 2 || UINT_SIZE == 2
|
|
# define interleave_hi(x, y) ((vec_t)__builtin_ia32_punpckhwd128((vhi_t)(x), (vhi_t)(y)))
|
|
# define interleave_lo(x, y) ((vec_t)__builtin_ia32_punpcklwd128((vhi_t)(x), (vhi_t)(y)))
|
|
# define swap(x) ((vec_t)__builtin_ia32_pshufd( \
|
|
(vsi_t)__builtin_ia32_pshufhw( \
|
|
__builtin_ia32_pshuflw((vhi_t)(x), 0b00011011), 0b00011011), 0b01001110))
|
|
# elif INT_SIZE == 4 || UINT_SIZE == 4
|
|
# define interleave_hi(x, y) ((vec_t)__builtin_ia32_punpckhdq128((vsi_t)(x), (vsi_t)(y)))
|
|
# define interleave_lo(x, y) ((vec_t)__builtin_ia32_punpckldq128((vsi_t)(x), (vsi_t)(y)))
|
|
# define swap(x) ((vec_t)__builtin_ia32_pshufd((vsi_t)(x), 0b00011011))
|
|
# elif INT_SIZE == 8 || UINT_SIZE == 8
|
|
# define interleave_hi(x, y) ((vec_t)__builtin_ia32_punpckhqdq128((vdi_t)(x), (vdi_t)(y)))
|
|
# define interleave_lo(x, y) ((vec_t)__builtin_ia32_punpcklqdq128((vdi_t)(x), (vdi_t)(y)))
|
|
# define swap(x) ((vec_t)__builtin_ia32_pshufd((vsi_t)(x), 0b01001110))
|
|
# endif
|
|
# if UINT_SIZE == 1
|
|
# define max(x, y) ((vec_t)__builtin_ia32_pmaxub128((vqi_t)(x), (vqi_t)(y)))
|
|
# define min(x, y) ((vec_t)__builtin_ia32_pminub128((vqi_t)(x), (vqi_t)(y)))
|
|
# elif INT_SIZE == 2
|
|
# define max(x, y) __builtin_ia32_pmaxsw128(x, y)
|
|
# define min(x, y) __builtin_ia32_pminsw128(x, y)
|
|
# define mul_hi(x, y) __builtin_ia32_pmulhw128(x, y)
|
|
# elif UINT_SIZE == 2
|
|
# define mul_hi(x, y) ((vec_t)__builtin_ia32_pmulhuw128((vhi_t)(x), (vhi_t)(y)))
|
|
# elif UINT_SIZE == 4
|
|
# define mul_full(x, y) ((vec_t)__builtin_ia32_pmuludq128((vsi_t)(x), (vsi_t)(y)))
|
|
# endif
|
|
# define select(d, x, y, m) ({ \
|
|
void *d_ = (d); \
|
|
vqi_t m_ = (vqi_t)(m); \
|
|
__builtin_ia32_maskmovdqu((vqi_t)(x), m_, d_); \
|
|
__builtin_ia32_maskmovdqu((vqi_t)(y), ~m_, d_); \
|
|
})
|
|
#elif VEC_SIZE == 32 && defined(__AVX2__)
|
|
# define swap_lanes(x, y, func, type) ({ \
|
|
long long __attribute__((vector_size(16))) t_ = __builtin_ia32_extract128i256((vdi_t)(y), 0); \
|
|
type t1_ = (type)__builtin_ia32_insert128i256((vdi_t)(x), t_, 1), t2_; \
|
|
t_ = __builtin_ia32_extract128i256((vdi_t)(x), 1); \
|
|
t2_ = (type)__builtin_ia32_insert128i256((vdi_t)(y), t_, 0); \
|
|
func(t1_, t2_); \
|
|
})
|
|
# if INT_SIZE == 1 || UINT_SIZE == 1
|
|
# define broadcast(x) ({ char s_ = (x); vec_t d_; asm ( "vpbroadcastb %1,%0" : "=x" (d_) : "m" (s_)); d_; })
|
|
# define copysignz(x, y) ((vec_t)__builtin_ia32_psignb256((vqi_t)(x), (vqi_t)(y)))
|
|
# define rotr(x, n) ((vec_t)__builtin_ia32_palignr256(__builtin_ia32_permti256((vdi_t)(x), (vdi_t)(x), 0b00000001), \
|
|
(vdi_t)(x), (n) * 8))
|
|
# elif INT_SIZE == 2 || UINT_SIZE == 2
|
|
# define broadcast(x) ({ short s_ = (x); vec_t d_; asm ( "vpbroadcastw %1,%0" : "=x" (d_) : "m" (s_)); d_; })
|
|
# define copysignz(x, y) ((vec_t)__builtin_ia32_psignw256((vhi_t)(x), (vhi_t)(y)))
|
|
# define hadd(x, y) ((vec_t)swap_lanes(x, y, __builtin_ia32_phaddw256, vhi_t))
|
|
# define hsub(x, y) ((vec_t)swap_lanes(x, y, __builtin_ia32_phsubw256, vhi_t))
|
|
# define mix(x, y) ((vec_t)__builtin_ia32_pblendw256((vhi_t)(x), (vhi_t)(y), 0b10101010))
|
|
# define rotr(x, n) ((vec_t)__builtin_ia32_palignr256(__builtin_ia32_permti256((vdi_t)(x), (vdi_t)(x), 0b00000001), \
|
|
(vdi_t)(x), (n) * 16))
|
|
# elif INT_SIZE == 4 || UINT_SIZE == 4
|
|
# define broadcast(x) ({ int s_ = (x); vec_t d_; asm ( "vpbroadcastd %1,%0" : "=x" (d_) : "m" (s_)); d_; })
|
|
# define copysignz(x, y) ((vec_t)__builtin_ia32_psignd256((vsi_t)(x), (vsi_t)(y)))
|
|
# define hadd(x, y) ((vec_t)swap_lanes(x, y, __builtin_ia32_phaddd256, vsi_t))
|
|
# define hsub(x, y) ((vec_t)swap_lanes(x, y, __builtin_ia32_phsubd256, vsi_t))
|
|
# define mix(x, y) ((vec_t)__builtin_ia32_pblendd256((vsi_t)(x), (vsi_t)(y), 0b10101010))
|
|
# define rotr(x, n) ((vec_t)__builtin_ia32_palignr256(__builtin_ia32_permti256((vdi_t)(x), (vdi_t)(x), 0b00000001), \
|
|
(vdi_t)(x), (n) * 32))
|
|
# define select(d, x, y, m) ({ \
|
|
vsi_t m_ = (vsi_t)(m); \
|
|
*(d) = (vec_t)__builtin_ia32_maskloadd256((vsi_t *)&(x), m_); \
|
|
__builtin_ia32_maskstored256((vsi_t *)(d), ~m_, (vsi_t)(y)); \
|
|
})
|
|
# define swap(x) ((vec_t)__builtin_ia32_permvarsi256((vsi_t)(x), (vsi_t)inv - 1))
|
|
# elif INT_SIZE == 8 || UINT_SIZE == 8
|
|
# define mix(x, y) ((vec_t)__builtin_ia32_pblendd256((vsi_t)(x), (vsi_t)(y), 0b11001100))
|
|
# define rotr(x, n) ((vec_t)__builtin_ia32_palignr256(__builtin_ia32_permti256((vdi_t)(x), (vdi_t)(x), 0b00000001), \
|
|
(vdi_t)(x), (n) * 64))
|
|
# define select(d, x, y, m) ({ \
|
|
vdi_t m_ = (vdi_t)(m); \
|
|
*(d) = (vec_t)__builtin_ia32_maskloadq256((vdi_t *)&(x), m_); \
|
|
__builtin_ia32_maskstoreq256((vdi_t *)(d), ~m_, (vdi_t)(y)); \
|
|
})
|
|
# define swap(x) ((vec_t)__builtin_ia32_permdi256((vdi_t)(x), 0b00011011))
|
|
# define swap2(x) ({ \
|
|
vdi_t t_ = __builtin_ia32_permdi256((vdi_t)(x), 0b10110001); \
|
|
(vec_t)__builtin_ia32_permti256(t_, t_, 0b00000001); \
|
|
})
|
|
# endif
|
|
# if INT_SIZE == 1
|
|
# define abs(x) ((vec_t)__builtin_ia32_pabsb256((vqi_t)(x)))
|
|
# define max(x, y) ((vec_t)__builtin_ia32_pmaxsb256((vqi_t)(x), (vqi_t)(y)))
|
|
# define min(x, y) ((vec_t)__builtin_ia32_pminsb256((vqi_t)(x), (vqi_t)(y)))
|
|
# define widen1(x) ((vec_t)__builtin_ia32_pmovsxbw256((vqi_t)(x)))
|
|
# define widen2(x) ((vec_t)__builtin_ia32_pmovsxbd256((vqi_t)(x)))
|
|
# define widen3(x) ((vec_t)__builtin_ia32_pmovsxbq256((vqi_t)(x)))
|
|
# elif UINT_SIZE == 1
|
|
# define max(x, y) ((vec_t)__builtin_ia32_pmaxub256((vqi_t)(x), (vqi_t)(y)))
|
|
# define min(x, y) ((vec_t)__builtin_ia32_pminub256((vqi_t)(x), (vqi_t)(y)))
|
|
# define widen1(x) ((vec_t)__builtin_ia32_pmovzxbw256((vqi_t)(x)))
|
|
# define widen2(x) ((vec_t)__builtin_ia32_pmovzxbd256((vqi_t)(x)))
|
|
# define widen3(x) ((vec_t)__builtin_ia32_pmovzxbq256((vqi_t)(x)))
|
|
# elif INT_SIZE == 2
|
|
# define abs(x) __builtin_ia32_pabsw256(x)
|
|
# define max(x, y) __builtin_ia32_pmaxsw256(x, y)
|
|
# define min(x, y) __builtin_ia32_pminsw256(x, y)
|
|
# define mul_hi(x, y) __builtin_ia32_pmulhw256(x, y)
|
|
# define widen1(x) ((vec_t)__builtin_ia32_pmovsxwd256(x))
|
|
# define widen2(x) ((vec_t)__builtin_ia32_pmovsxwq256(x))
|
|
# elif UINT_SIZE == 2
|
|
# define max(x, y) ((vec_t)__builtin_ia32_pmaxuw256((vhi_t)(x), (vhi_t)(y)))
|
|
# define min(x, y) ((vec_t)__builtin_ia32_pminuw256((vhi_t)(x), (vhi_t)(y)))
|
|
# define mul_hi(x, y) ((vec_t)__builtin_ia32_pmulhuw256((vhi_t)(x), (vhi_t)(y)))
|
|
# define widen1(x) ((vec_t)__builtin_ia32_pmovzxwd256((vhi_t)(x)))
|
|
# define widen2(x) ((vec_t)__builtin_ia32_pmovzxwq256((vhi_t)(x)))
|
|
# elif INT_SIZE == 4
|
|
# define abs(x) __builtin_ia32_pabsd256(x)
|
|
# define max(x, y) __builtin_ia32_pmaxsd256(x, y)
|
|
# define min(x, y) __builtin_ia32_pminsd256(x, y)
|
|
# define widen1(x) ((vec_t)__builtin_ia32_pmovsxdq256(x))
|
|
# elif UINT_SIZE == 4
|
|
# define max(x, y) ((vec_t)__builtin_ia32_pmaxud256((vsi_t)(x), (vsi_t)(y)))
|
|
# define min(x, y) ((vec_t)__builtin_ia32_pminud256((vsi_t)(x), (vsi_t)(y)))
|
|
# define mul_full(x, y) ((vec_t)__builtin_ia32_pmuludq256((vsi_t)(x), (vsi_t)(y)))
|
|
# define widen1(x) ((vec_t)__builtin_ia32_pmovzxdq256((vsi_t)(x)))
|
|
# elif INT_SIZE == 8
|
|
# define broadcast(x) ({ \
|
|
long long s_ = (x); \
|
|
long long __attribute__((vector_size(16))) t_; \
|
|
vec_t d_; \
|
|
asm ( "vpbroadcastq %1,%0" : "=x" (t_) : "m" (s_)); \
|
|
asm ( "vbroadcasti128 %1,%0" : "=x" (d_) : "m" (t_)); \
|
|
d_; \
|
|
})
|
|
# elif UINT_SIZE == 8
|
|
# define broadcast(x) ({ long long s_ = (x); vec_t d_; asm ( "vpbroadcastq %1,%0" : "=x" (d_) : "m" (s_)); d_; })
|
|
# endif
|
|
#endif
|
|
#if VEC_SIZE == 16 && defined(__SSE3__)
|
|
# if FLOAT_SIZE == 4
|
|
# define addsub(x, y) __builtin_ia32_addsubps(x, y)
|
|
# define dup_hi(x) __builtin_ia32_movshdup(x)
|
|
# define dup_lo(x) __builtin_ia32_movsldup(x)
|
|
# define hadd(x, y) __builtin_ia32_haddps(x, y)
|
|
# define hsub(x, y) __builtin_ia32_hsubps(x, y)
|
|
# elif FLOAT_SIZE == 8
|
|
# define addsub(x, y) __builtin_ia32_addsubpd(x, y)
|
|
# define dup_lo(x) ({ \
|
|
double __attribute__((vector_size(16))) r_; \
|
|
asm ( "movddup %1,%0" : "=x" (r_) : "m" ((x)[0]) ); \
|
|
r_; \
|
|
})
|
|
# define hadd(x, y) __builtin_ia32_haddpd(x, y)
|
|
# define hsub(x, y) __builtin_ia32_hsubpd(x, y)
|
|
# endif
|
|
#elif VEC_SIZE == 32 && defined(__AVX__)
|
|
# if FLOAT_SIZE == 4
|
|
# define addsub(x, y) __builtin_ia32_addsubps256(x, y)
|
|
# define dup_hi(x) __builtin_ia32_movshdup256(x)
|
|
# define dup_lo(x) __builtin_ia32_movsldup256(x)
|
|
# ifdef __AVX2__
|
|
# define hadd(x, y) __builtin_ia32_permvarsf256(__builtin_ia32_haddps256(x, y), \
|
|
(vsi_t){0, 1, 4, 5, 2, 3, 6, 7})
|
|
# define hsub(x, y) __builtin_ia32_permvarsf256(__builtin_ia32_hsubps256(x, y), \
|
|
(vsi_t){0, 1, 4, 5, 2, 3, 6, 7})
|
|
# else
|
|
# define hadd(x, y) ({ \
|
|
vec_t t_ = __builtin_ia32_haddps256(x, y); \
|
|
(vec_t){t_[0], t_[1], t_[4], t_[5], t_[2], t_[3], t_[6], t_[7]}; \
|
|
})
|
|
# define hsub(x, y) ({ \
|
|
vec_t t_ = __builtin_ia32_hsubps256(x, y); \
|
|
(vec_t){t_[0], t_[1], t_[4], t_[5], t_[2], t_[3], t_[6], t_[7]}; \
|
|
})
|
|
# endif
|
|
# elif FLOAT_SIZE == 8
|
|
# define addsub(x, y) __builtin_ia32_addsubpd256(x, y)
|
|
# define dup_lo(x) __builtin_ia32_movddup256(x)
|
|
# ifdef __AVX2__
|
|
# define hadd(x, y) __builtin_ia32_permdf256(__builtin_ia32_haddpd256(x, y), 0b11011000)
|
|
# define hsub(x, y) __builtin_ia32_permdf256(__builtin_ia32_hsubpd256(x, y), 0b11011000)
|
|
# else
|
|
# define hadd(x, y) ({ \
|
|
vec_t t_ = __builtin_ia32_haddpd256(x, y); \
|
|
(vec_t){t_[0], t_[2], t_[1], t_[3]}; \
|
|
})
|
|
# define hsub(x, y) ({ \
|
|
vec_t t_ = __builtin_ia32_hsubpd256(x, y); \
|
|
(vec_t){t_[0], t_[2], t_[1], t_[3]}; \
|
|
})
|
|
# endif
|
|
# endif
|
|
#elif VEC_SIZE == 64
|
|
# if FLOAT_SIZE == 4
|
|
# define dup_hi(x) B(movshdup, _mask, x, undef(), ~0)
|
|
# define dup_lo(x) B(movsldup, _mask, x, undef(), ~0)
|
|
# elif FLOAT_SIZE == 8
|
|
# define dup_lo(x) B(movddup, _mask, x, undef(), ~0)
|
|
# endif
|
|
#endif
|
|
#if FLOAT_SIZE == 2 && ELEM_COUNT > 1
|
|
# define dup_hi(x) ((vec_t)B(pshufhw, _mask, \
|
|
B(pshuflw, _mask, (vhi_t)(x), 0b11110101, \
|
|
(vhi_t)undef(), ~0), \
|
|
0b11110101, (vhi_t)undef(), ~0))
|
|
# define dup_lo(x) ((vec_t)B(pshufhw, _mask, \
|
|
B(pshuflw, _mask, (vhi_t)(x), 0b10100000, \
|
|
(vhi_t)undef(), ~0), \
|
|
0b10100000, (vhi_t)undef(), ~0))
|
|
#endif
|
|
#if VEC_SIZE == 16 && defined(__SSSE3__) && !defined(__AVX512VL__)
|
|
# if INT_SIZE == 1
|
|
# define abs(x) ((vec_t)__builtin_ia32_pabsb128((vqi_t)(x)))
|
|
# elif INT_SIZE == 2
|
|
# define abs(x) __builtin_ia32_pabsw128(x)
|
|
# elif INT_SIZE == 4
|
|
# define abs(x) __builtin_ia32_pabsd128(x)
|
|
# endif
|
|
# if INT_SIZE == 1 || UINT_SIZE == 1
|
|
# define copysignz(x, y) ((vec_t)__builtin_ia32_psignb128((vqi_t)(x), (vqi_t)(y)))
|
|
# define swap(x) ((vec_t)__builtin_ia32_pshufb128((vqi_t)(x), (vqi_t)(inv - 1)))
|
|
# define rotr(x, n) ((vec_t)__builtin_ia32_palignr128((vdi_t)(x), (vdi_t)(x), (n) * 8))
|
|
# elif INT_SIZE == 2 || UINT_SIZE == 2
|
|
# define copysignz(x, y) ((vec_t)__builtin_ia32_psignw128((vhi_t)(x), (vhi_t)(y)))
|
|
# define hadd(x, y) ((vec_t)__builtin_ia32_phaddw128((vhi_t)(x), (vhi_t)(y)))
|
|
# define hsub(x, y) ((vec_t)__builtin_ia32_phsubw128((vhi_t)(x), (vhi_t)(y)))
|
|
# define rotr(x, n) ((vec_t)__builtin_ia32_palignr128((vdi_t)(x), (vdi_t)(x), (n) * 16))
|
|
# elif INT_SIZE == 4 || UINT_SIZE == 4
|
|
# define copysignz(x, y) ((vec_t)__builtin_ia32_psignd128((vsi_t)(x), (vsi_t)(y)))
|
|
# define hadd(x, y) ((vec_t)__builtin_ia32_phaddd128((vsi_t)(x), (vsi_t)(y)))
|
|
# define hsub(x, y) ((vec_t)__builtin_ia32_phsubd128((vsi_t)(x), (vsi_t)(y)))
|
|
# define rotr(x, n) ((vec_t)__builtin_ia32_palignr128((vdi_t)(x), (vdi_t)(x), (n) * 32))
|
|
# elif INT_SIZE == 8 || UINT_SIZE == 8
|
|
# define rotr(x, n) ((vec_t)__builtin_ia32_palignr128((vdi_t)(x), (vdi_t)(x), (n) * 64))
|
|
# endif
|
|
#endif
|
|
#if VEC_SIZE == 16 && defined(__SSE4_1__) && !defined(__AVX512VL__)
|
|
# if INT_SIZE == 1
|
|
# define max(x, y) ((vec_t)__builtin_ia32_pmaxsb128((vqi_t)(x), (vqi_t)(y)))
|
|
# define min(x, y) ((vec_t)__builtin_ia32_pminsb128((vqi_t)(x), (vqi_t)(y)))
|
|
# define widen1(x) ((vec_t)__builtin_ia32_pmovsxbw128((vqi_t)(x)))
|
|
# define widen2(x) ((vec_t)__builtin_ia32_pmovsxbd128((vqi_t)(x)))
|
|
# define widen3(x) ((vec_t)__builtin_ia32_pmovsxbq128((vqi_t)(x)))
|
|
# elif INT_SIZE == 2
|
|
# define widen1(x) ((vec_t)__builtin_ia32_pmovsxwd128(x))
|
|
# define widen2(x) ((vec_t)__builtin_ia32_pmovsxwq128(x))
|
|
# elif INT_SIZE == 4
|
|
# define max(x, y) __builtin_ia32_pmaxsd128(x, y)
|
|
# define min(x, y) __builtin_ia32_pminsd128(x, y)
|
|
# define mul_full(x, y) ((vec_t)__builtin_ia32_pmuldq128(x, y))
|
|
# define widen1(x) ((vec_t)__builtin_ia32_pmovsxdq128(x))
|
|
# elif UINT_SIZE == 1
|
|
# define widen1(x) ((vec_t)__builtin_ia32_pmovzxbw128((vqi_t)(x)))
|
|
# define widen2(x) ((vec_t)__builtin_ia32_pmovzxbd128((vqi_t)(x)))
|
|
# define widen3(x) ((vec_t)__builtin_ia32_pmovzxbq128((vqi_t)(x)))
|
|
# elif UINT_SIZE == 2
|
|
# define max(x, y) ((vec_t)__builtin_ia32_pmaxuw128((vhi_t)(x), (vhi_t)(y)))
|
|
# define min(x, y) ((vec_t)__builtin_ia32_pminuw128((vhi_t)(x), (vhi_t)(y)))
|
|
# define widen1(x) ((vec_t)__builtin_ia32_pmovzxwd128((vhi_t)(x)))
|
|
# define widen2(x) ((vec_t)__builtin_ia32_pmovzxwq128((vhi_t)(x)))
|
|
# elif UINT_SIZE == 4
|
|
# define max(x, y) ((vec_t)__builtin_ia32_pmaxud128((vsi_t)(x), (vsi_t)(y)))
|
|
# define min(x, y) ((vec_t)__builtin_ia32_pminud128((vsi_t)(x), (vsi_t)(y)))
|
|
# define widen1(x) ((vec_t)__builtin_ia32_pmovzxdq128((vsi_t)(x)))
|
|
# endif
|
|
# undef select
|
|
# if defined(INT_SIZE) || defined(UINT_SIZE)
|
|
# define select(d, x, y, m) \
|
|
(*(d) = (vec_t)__builtin_ia32_pblendvb128((vqi_t)(y), (vqi_t)(x), (vqi_t)(m)))
|
|
# elif FLOAT_SIZE == 4
|
|
# define dot_product(x, y) __builtin_ia32_dpps(x, y, 0b11110001)
|
|
# define select(d, x, y, m) (*(d) = __builtin_ia32_blendvps(y, x, m))
|
|
# define trunc(x) __builtin_ia32_roundps(x, 0b1011)
|
|
# elif FLOAT_SIZE == 8
|
|
# define dot_product(x, y) __builtin_ia32_dppd(x, y, 0b00110001)
|
|
# define select(d, x, y, m) (*(d) = __builtin_ia32_blendvpd(y, x, m))
|
|
# define trunc(x) __builtin_ia32_roundpd(x, 0b1011)
|
|
# endif
|
|
# if INT_SIZE == 2 || UINT_SIZE == 2
|
|
# define mix(x, y) ((vec_t)__builtin_ia32_pblendw128((vhi_t)(x), (vhi_t)(y), 0b10101010))
|
|
# elif INT_SIZE == 4 || UINT_SIZE == 4
|
|
# define mix(x, y) ((vec_t)__builtin_ia32_pblendw128((vhi_t)(x), (vhi_t)(y), 0b11001100))
|
|
# elif INT_SIZE == 8 || UINT_SIZE == 8
|
|
# define mix(x, y) ((vec_t)__builtin_ia32_pblendw128((vhi_t)(x), (vhi_t)(y), 0b11110000))
|
|
# elif FLOAT_SIZE == 4
|
|
# define mix(x, y) __builtin_ia32_blendps(x, y, 0b1010)
|
|
# elif FLOAT_SIZE == 8
|
|
# define mix(x, y) __builtin_ia32_blendpd(x, y, 0b10)
|
|
# endif
|
|
#endif
|
|
#if VEC_SIZE == 32 && defined(__AVX__) && !defined(__AVX512VL__)
|
|
# if FLOAT_SIZE == 4
|
|
# define dot_product(x, y) ({ \
|
|
vec_t t_ = __builtin_ia32_dpps256(x, y, 0b11110001); \
|
|
(vec_t){t_[0] + t_[4]}; \
|
|
})
|
|
# define mix(x, y) __builtin_ia32_blendps256(x, y, 0b10101010)
|
|
# define select(d, x, y, m) (*(d) = __builtin_ia32_blendvps256(y, x, m))
|
|
# define select2(d, x, y, m) ({ \
|
|
vsi_t m_ = (vsi_t)(m); \
|
|
*(d) = __builtin_ia32_maskloadps256(&(x), m_); \
|
|
__builtin_ia32_maskstoreps256(d, ~m_, y); \
|
|
})
|
|
# define trunc(x) __builtin_ia32_roundps256(x, 0b1011)
|
|
# elif FLOAT_SIZE == 8
|
|
# define mix(x, y) __builtin_ia32_blendpd256(x, y, 0b1010)
|
|
# define select(d, x, y, m) (*(d) = __builtin_ia32_blendvpd256(y, x, m))
|
|
# define select2(d, x, y, m) ({ \
|
|
vdi_t m_ = (vdi_t)(m); \
|
|
*(d) = __builtin_ia32_maskloadpd256(&(x), m_); \
|
|
__builtin_ia32_maskstorepd256(d, ~m_, y); \
|
|
})
|
|
# define trunc(x) __builtin_ia32_roundpd256(x, 0b1011)
|
|
# endif
|
|
#endif
|
|
#if VEC_SIZE == FLOAT_SIZE
|
|
# define max(x, y) ((vec_t){({ typeof(x[0]) x_ = (x)[0], y_ = (y)[0]; x_ > y_ ? x_ : y_; })})
|
|
# define min(x, y) ((vec_t){({ typeof(x[0]) x_ = (x)[0], y_ = (y)[0]; x_ < y_ ? x_ : y_; })})
|
|
# if defined(__SSE4_1__) && !defined(__AVX512F__)
|
|
# if FLOAT_SIZE == 4
|
|
# define trunc(x) scalar_1op(x, "roundss $0b1011, %[in], %[out]")
|
|
# elif FLOAT_SIZE == 8
|
|
# define trunc(x) scalar_1op(x, "roundsd $0b1011, %[in], %[out]")
|
|
# endif
|
|
# endif
|
|
#endif
|
|
#ifdef __XOP__
|
|
# undef select
|
|
# if VEC_SIZE == 16
|
|
# if INT_SIZE == 2 || INT_SIZE == 4
|
|
# include "simd-fma.c"
|
|
# endif
|
|
# define select(d, x, y, m) \
|
|
(*(d) = (vec_t)__builtin_ia32_vpcmov((vdi_t)(x), (vdi_t)(y), (vdi_t)(m)))
|
|
# if INT_SIZE == 1 || UINT_SIZE == 1
|
|
# define swap2(x) ((vec_t)__builtin_ia32_vpperm((vqi_t)(x), (vqi_t)(x), (vqi_t)inv - 1))
|
|
# elif INT_SIZE == 2 || UINT_SIZE == 2
|
|
# define swap2(x) \
|
|
((vec_t)__builtin_ia32_vpperm((vqi_t)(x), (vqi_t)(x), \
|
|
(vqi_t)(__builtin_ia32_vprotwi(2 * (vhi_t)inv - 1, 8) | \
|
|
(2 * inv - 2))))
|
|
# elif FLOAT_SIZE == 4
|
|
# define frac(x) __builtin_ia32_vfrczps(x)
|
|
# undef swap2
|
|
# define swap2(x) ({ \
|
|
/* Buggy in gcc 7.1.0 and earlier. */ \
|
|
/* __builtin_ia32_vpermil2ps((vec_t){}, x, __builtin_ia32_cvtps2dq(inv) + 3, 0) */ \
|
|
vec_t t_; \
|
|
asm ( "vpermil2ps $0, %3, %2, %1, %0" : \
|
|
"=x" (t_) : \
|
|
"x" ((vec_t){}), "m" (x), "x" (__builtin_ia32_cvtps2dq(inv) + 3) ); \
|
|
t_; \
|
|
})
|
|
# elif FLOAT_SIZE == 8
|
|
# define frac(x) __builtin_ia32_vfrczpd(x)
|
|
# undef swap2
|
|
# define swap2(x) ({ \
|
|
/* Buggy in gcc 7.1.0 and earlier. */ \
|
|
/* __builtin_ia32_vpermil2pd((vec_t){}, x, */ \
|
|
/* __builtin_ia32_pmovsxdq128( */ \
|
|
/* __builtin_ia32_cvtpd2dq(inv) + 1) << 1, 0) */ \
|
|
vdi_t s_ = __builtin_ia32_pmovsxdq128( \
|
|
__builtin_ia32_cvtpd2dq(inv) + 1) << 1; \
|
|
vec_t t_; \
|
|
asm ( "vpermil2pd $0, %3, %2, %1, %0" : \
|
|
"=x" (t_) : "x" ((vec_t){}), "x" (x), "m" (s_) ); \
|
|
t_; \
|
|
})
|
|
# endif
|
|
# if INT_SIZE == 1
|
|
# define hadd(x, y) ((vec_t)__builtin_ia32_packsswb128(__builtin_ia32_vphaddbw((vqi_t)(x)), \
|
|
__builtin_ia32_vphaddbw((vqi_t)(y))))
|
|
# define hsub(x, y) ((vec_t)__builtin_ia32_packsswb128(__builtin_ia32_vphsubbw((vqi_t)(x)), \
|
|
__builtin_ia32_vphsubbw((vqi_t)(y))))
|
|
# elif UINT_SIZE == 1
|
|
# define hadd(x, y) ((vec_t)__builtin_ia32_packuswb128(__builtin_ia32_vphaddubw((vqi_t)(x)), \
|
|
__builtin_ia32_vphaddubw((vqi_t)(y))))
|
|
# elif INT_SIZE == 2
|
|
# undef hadd
|
|
# define hadd(x, y) __builtin_ia32_packssdw128(__builtin_ia32_vphaddwd(x), \
|
|
__builtin_ia32_vphaddwd(y))
|
|
# undef hsub
|
|
# define hsub(x, y) __builtin_ia32_packssdw128(__builtin_ia32_vphsubwd(x), \
|
|
__builtin_ia32_vphsubwd(y))
|
|
# elif UINT_SIZE == 2
|
|
# undef hadd
|
|
# define hadd(x, y) ((vec_t)__builtin_ia32_packusdw128(__builtin_ia32_vphadduwd((vhi_t)(x)), \
|
|
__builtin_ia32_vphadduwd((vhi_t)(y))))
|
|
# undef hsub
|
|
# endif
|
|
# elif VEC_SIZE == 32
|
|
# define select(d, x, y, m) \
|
|
(*(d) = (vec_t)__builtin_ia32_vpcmov256((vdi_t)(x), (vdi_t)(y), (vdi_t)(m)))
|
|
# if FLOAT_SIZE == 4
|
|
# define frac(x) __builtin_ia32_vfrczps256(x)
|
|
# elif FLOAT_SIZE == 8
|
|
# define frac(x) __builtin_ia32_vfrczpd256(x)
|
|
# endif
|
|
# elif VEC_SIZE == FLOAT_SIZE
|
|
# if VEC_SIZE == 4
|
|
# define frac(x) scalar_1op(x, "vfrczss %[in], %[out]")
|
|
# elif VEC_SIZE == 8
|
|
# define frac(x) scalar_1op(x, "vfrczsd %[in], %[out]")
|
|
# endif
|
|
# endif
|
|
#endif
|
|
|
|
#if VEC_SIZE >= 16
|
|
|
|
# if !defined(low_half) && defined(HALF_SIZE)
|
|
static inline half_t low_half(vec_t x)
|
|
{
|
|
# if HALF_SIZE < VEC_SIZE
|
|
half_t y;
|
|
unsigned int i;
|
|
|
|
for ( i = 0; i < ELEM_COUNT / 2; ++i )
|
|
y[i] = x[i];
|
|
|
|
return y;
|
|
# else
|
|
return x;
|
|
# endif
|
|
}
|
|
# endif
|
|
|
|
# if !defined(low_quarter) && defined(QUARTER_SIZE)
|
|
static inline quarter_t low_quarter(vec_t x)
|
|
{
|
|
# if QUARTER_SIZE < VEC_SIZE
|
|
quarter_t y;
|
|
unsigned int i;
|
|
|
|
for ( i = 0; i < ELEM_COUNT / 4; ++i )
|
|
y[i] = x[i];
|
|
|
|
return y;
|
|
# else
|
|
return x;
|
|
# endif
|
|
}
|
|
# endif
|
|
|
|
# if !defined(low_eighth) && defined(EIGHTH_SIZE)
|
|
static inline eighth_t low_eighth(vec_t x)
|
|
{
|
|
# if EIGHTH_SIZE < VEC_SIZE
|
|
eighth_t y;
|
|
unsigned int i;
|
|
|
|
for ( i = 0; i < ELEM_COUNT / 8; ++i )
|
|
y[i] = x[i];
|
|
|
|
return y;
|
|
# else
|
|
return x;
|
|
# endif
|
|
}
|
|
# endif
|
|
|
|
#endif
|
|
|
|
#ifdef broadcast_pair
|
|
# if ELEM_COUNT == 4
|
|
# define broadcast_half broadcast_pair
|
|
# elif ELEM_COUNT == 8
|
|
# define broadcast_quarter broadcast_pair
|
|
# elif ELEM_COUNT == 16
|
|
# define broadcast_eighth broadcast_pair
|
|
# endif
|
|
#endif
|
|
|
|
#ifdef insert_pair
|
|
# if ELEM_COUNT == 4
|
|
# define insert_half insert_pair
|
|
# elif ELEM_COUNT == 8
|
|
# define insert_quarter insert_pair
|
|
# elif ELEM_COUNT == 16
|
|
# define insert_eighth insert_pair
|
|
# endif
|
|
#endif
|
|
|
|
#ifdef broadcast_quartet
|
|
# if ELEM_COUNT == 8
|
|
# define broadcast_half broadcast_quartet
|
|
# elif ELEM_COUNT == 16
|
|
# define broadcast_quarter broadcast_quartet
|
|
# endif
|
|
#endif
|
|
|
|
#ifdef insert_quartet
|
|
# if ELEM_COUNT == 8
|
|
# define insert_half insert_quartet
|
|
# elif ELEM_COUNT == 16
|
|
# define insert_quarter insert_quartet
|
|
# endif
|
|
#endif
|
|
|
|
#if defined(broadcast_octet) && ELEM_COUNT == 16
|
|
# define broadcast_half broadcast_octet
|
|
#endif
|
|
|
|
#if defined(insert_octet) && ELEM_COUNT == 16
|
|
# define insert_half insert_octet
|
|
#endif
|
|
|
|
#if defined(__AVX512F__) && defined(FLOAT_SIZE)
|
|
# include "simd-fma.c"
|
|
#endif
|
|
|
|
int simd_test(void)
|
|
{
|
|
unsigned int i, j;
|
|
vec_t x, y, z, src, inv, alt, sh;
|
|
vint_t interleave_lo, interleave_hi;
|
|
|
|
for ( i = 0, j = ELEM_SIZE << 3; i < ELEM_COUNT; ++i )
|
|
{
|
|
src[i] = i + 1;
|
|
inv[i] = ELEM_COUNT - i;
|
|
#ifdef UINT_SIZE
|
|
alt[i] = -!(i & 1);
|
|
#else
|
|
alt[i] = i & 1 ? -1 : 1;
|
|
#endif
|
|
if ( !(i & (i + 1)) )
|
|
--j;
|
|
sh[i] = j;
|
|
|
|
interleave_lo[i] = ((i & 1) * ELEM_COUNT) | (i >> 1);
|
|
interleave_hi[i] = interleave_lo[i] + (ELEM_COUNT / 2);
|
|
}
|
|
|
|
touch(src);
|
|
x = src;
|
|
touch(x);
|
|
if ( !eq(x, src) ) return __LINE__;
|
|
|
|
touch(src);
|
|
y = x + src;
|
|
touch(src);
|
|
touch(y);
|
|
if ( !eq(y, 2 * src) ) return __LINE__;
|
|
|
|
touch(src);
|
|
z = y -= src;
|
|
touch(z);
|
|
if ( !eq(x, z) ) return __LINE__;
|
|
|
|
#if defined(UINT_SIZE)
|
|
|
|
touch(inv);
|
|
x |= inv;
|
|
touch(inv);
|
|
y &= inv;
|
|
touch(inv);
|
|
z ^= inv;
|
|
touch(inv);
|
|
touch(x);
|
|
if ( !eq(x & ~y, z) ) return __LINE__;
|
|
|
|
#elif ELEM_SIZE > 1 || VEC_SIZE <= 8
|
|
|
|
touch(src);
|
|
x *= src;
|
|
y = inv * inv;
|
|
touch(src);
|
|
z = src + inv;
|
|
touch(inv);
|
|
z *= (src - inv);
|
|
if ( !eq(x - y, z) ) return __LINE__;
|
|
|
|
#endif
|
|
|
|
#if defined(FLOAT_SIZE)
|
|
|
|
x = src * alt;
|
|
touch(alt);
|
|
y = src / alt;
|
|
if ( !eq(x, y) ) return __LINE__;
|
|
touch(alt);
|
|
touch(src);
|
|
if ( !eq(x * -alt, -src) ) return __LINE__;
|
|
|
|
# ifdef to_int
|
|
|
|
touch(src);
|
|
x = to_int(src);
|
|
touch(src);
|
|
if ( !eq(x, src) ) return __LINE__;
|
|
|
|
# ifdef recip
|
|
touch(src);
|
|
x = recip(src);
|
|
touch(src);
|
|
touch(x);
|
|
if ( !eq(to_int(recip(x)), src) ) return __LINE__;
|
|
|
|
# ifdef rsqrt
|
|
x = src * src;
|
|
touch(x);
|
|
y = rsqrt(x);
|
|
touch(y);
|
|
if ( !eq(to_int(recip(y)), src) ) return __LINE__;
|
|
touch(src);
|
|
if ( !eq(to_int(y), to_int(recip(src))) ) return __LINE__;
|
|
# endif
|
|
# endif
|
|
|
|
# endif
|
|
|
|
# ifdef to_wint
|
|
touch(src);
|
|
x = to_wint(src);
|
|
touch(src);
|
|
if ( !eq(x, src) ) return __LINE__;
|
|
# endif
|
|
|
|
# ifdef to_uint
|
|
touch(src);
|
|
x = to_uint(src);
|
|
touch(src);
|
|
if ( !eq(x, src) ) return __LINE__;
|
|
# endif
|
|
|
|
# ifdef to_uwint
|
|
touch(src);
|
|
x = to_uwint(src);
|
|
touch(src);
|
|
if ( !eq(x, src) ) return __LINE__;
|
|
# endif
|
|
|
|
# ifdef sqrt
|
|
x = src * src;
|
|
touch(x);
|
|
if ( !eq(sqrt(x), src) ) return __LINE__;
|
|
# endif
|
|
|
|
# ifdef trunc
|
|
x = 1 / src;
|
|
y = (vec_t){ 1 };
|
|
touch(x);
|
|
z = trunc(x);
|
|
if ( !eq(y, z) ) return __LINE__;
|
|
# endif
|
|
|
|
# ifdef frac
|
|
touch(src);
|
|
x = frac(src);
|
|
touch(src);
|
|
if ( !eq(x, (vec_t){}) ) return __LINE__;
|
|
|
|
x = 1 / (src + 1);
|
|
touch(x);
|
|
y = frac(x);
|
|
touch(x);
|
|
if ( !eq(x, y) ) return __LINE__;
|
|
# endif
|
|
|
|
# if defined(trunc) && defined(frac)
|
|
x = src / 4;
|
|
touch(x);
|
|
y = trunc(x);
|
|
touch(x);
|
|
z = frac(x);
|
|
touch(x);
|
|
if ( !eq(x, y + z) ) return __LINE__;
|
|
# endif
|
|
|
|
#else
|
|
|
|
# if ELEM_SIZE > 1
|
|
|
|
touch(inv);
|
|
x = src * inv;
|
|
touch(inv);
|
|
y[ELEM_COUNT - 1] = y[0] = j = ELEM_COUNT;
|
|
for ( i = 1; i < ELEM_COUNT / 2; ++i )
|
|
y[ELEM_COUNT - i - 1] = y[i] = y[i - 1] + (j -= 2);
|
|
if ( !eq(x, y) ) return __LINE__;
|
|
|
|
# ifdef mul_hi
|
|
touch(alt);
|
|
x = mul_hi(src, alt);
|
|
touch(alt);
|
|
# ifdef INT_SIZE
|
|
if ( !eq(x, alt < 0) ) return __LINE__;
|
|
# else
|
|
if ( !eq(x, (src & alt) + alt) ) return __LINE__;
|
|
# endif
|
|
# endif
|
|
|
|
# ifdef mul_full
|
|
x = src ^ alt;
|
|
touch(inv);
|
|
y = mul_full(x, inv);
|
|
touch(inv);
|
|
for ( i = 0; i < ELEM_COUNT; i += 2 )
|
|
{
|
|
unsigned long long res = x[i] * 1ULL * inv[i];
|
|
|
|
z[i] = res;
|
|
z[i + 1] = res >> (ELEM_SIZE << 3);
|
|
}
|
|
if ( !eq(y, z) ) return __LINE__;
|
|
# endif
|
|
|
|
z = src;
|
|
# ifdef INT_SIZE
|
|
z *= alt;
|
|
# endif
|
|
touch(z);
|
|
x = z << 3;
|
|
touch(z);
|
|
y = z << 2;
|
|
touch(z);
|
|
if ( !eq(x, y + y) ) return __LINE__;
|
|
|
|
touch(x);
|
|
z = x >> 2;
|
|
touch(x);
|
|
if ( !eq(y, z + z) ) return __LINE__;
|
|
|
|
z = src;
|
|
# ifdef INT_SIZE
|
|
z *= alt;
|
|
# endif
|
|
/*
|
|
* Note that despite the touch()-es here there doesn't appear to be a way
|
|
* to make the compiler use a memory operand for the shift instruction (at
|
|
* least without resorting to built-ins).
|
|
*/
|
|
j = 3;
|
|
touch(j);
|
|
x = z << j;
|
|
touch(j);
|
|
j = 2;
|
|
touch(j);
|
|
y = z << j;
|
|
touch(j);
|
|
if ( !eq(x, y + y) ) return __LINE__;
|
|
|
|
z = x >> j;
|
|
touch(j);
|
|
if ( !eq(y, z + z) ) return __LINE__;
|
|
|
|
# endif
|
|
|
|
# if ELEM_SIZE == 2 || defined(__SSE4_1__)
|
|
/*
|
|
* Even when there are no instructions with varying shift counts per
|
|
* field, the code turns out to be a nice exercise for pextr/pinsr.
|
|
*/
|
|
z = src;
|
|
# ifdef INT_SIZE
|
|
z *= alt;
|
|
# endif
|
|
/*
|
|
* Zap elements for which the shift count is zero (and the hence the
|
|
* decrement below would yield a negative count.
|
|
*/
|
|
z &= (sh > 0);
|
|
touch(sh);
|
|
x = z << sh;
|
|
touch(sh);
|
|
--sh;
|
|
touch(sh);
|
|
y = z << sh;
|
|
if ( !eq(x, y + y) ) return __LINE__;
|
|
|
|
# if (defined(__AVX2__) && ELEM_SIZE >= 4) || defined(__XOP__)
|
|
touch(sh);
|
|
x = y >> sh;
|
|
if ( !eq(x, z) ) return __LINE__;
|
|
# endif
|
|
|
|
# endif
|
|
|
|
#endif
|
|
|
|
#if defined(max) && defined(min)
|
|
# ifdef UINT_SIZE
|
|
touch(inv);
|
|
x = min(src, inv);
|
|
touch(inv);
|
|
y = max(src, inv);
|
|
touch(inv);
|
|
if ( !eq(x + y, src + inv) ) return __LINE__;
|
|
# else
|
|
x = src * alt;
|
|
y = inv * alt;
|
|
touch(y);
|
|
z = max(x, y);
|
|
touch(y);
|
|
y = min(x, y);
|
|
touch(y);
|
|
if ( !eq((y + z) * alt, src + inv) ) return __LINE__;
|
|
# endif
|
|
#endif
|
|
|
|
#ifdef abs
|
|
x = src * alt;
|
|
touch(x);
|
|
if ( !eq(abs(x), src) ) return __LINE__;
|
|
#endif
|
|
|
|
#ifdef copysignz
|
|
touch(alt);
|
|
if ( !eq(copysignz((vec_t){} + 1, alt), alt) ) return __LINE__;
|
|
#endif
|
|
|
|
#ifdef swap
|
|
touch(src);
|
|
if ( !eq(swap(src), inv) ) return __LINE__;
|
|
#endif
|
|
|
|
#ifdef swap2
|
|
touch(src);
|
|
if ( !eq(swap2(src), inv) ) return __LINE__;
|
|
#endif
|
|
|
|
#ifdef swap3
|
|
touch(src);
|
|
if ( !eq(swap3(src), inv) ) return __LINE__;
|
|
touch(src);
|
|
#endif
|
|
|
|
#ifdef broadcast
|
|
if ( !eq(broadcast(ELEM_COUNT + 1), src + inv) ) return __LINE__;
|
|
#endif
|
|
|
|
#ifdef broadcast2
|
|
if ( !eq(broadcast2(ELEM_COUNT + 1), src + inv) ) return __LINE__;
|
|
#endif
|
|
|
|
#if defined(broadcast_half) && defined(insert_half)
|
|
{
|
|
half_t aux = low_half(src);
|
|
|
|
touch(aux);
|
|
x = broadcast_half(aux);
|
|
touch(aux);
|
|
y = insert_half(src, aux, 1);
|
|
if ( !eq(x, y) ) return __LINE__;
|
|
}
|
|
#endif
|
|
|
|
#if defined(broadcast_quarter) && defined(insert_quarter)
|
|
{
|
|
quarter_t aux = low_quarter(src);
|
|
|
|
touch(aux);
|
|
x = broadcast_quarter(aux);
|
|
touch(aux);
|
|
y = insert_quarter(src, aux, 1);
|
|
touch(aux);
|
|
y = insert_quarter(y, aux, 2);
|
|
touch(aux);
|
|
y = insert_quarter(y, aux, 3);
|
|
if ( !eq(x, y) ) return __LINE__;
|
|
}
|
|
#endif
|
|
|
|
#if defined(broadcast_eighth) && defined(insert_eighth) && \
|
|
/* At least gcc 7.3 "optimizes" away all insert_eighth() calls below. */ \
|
|
__GNUC__ >= 8
|
|
{
|
|
eighth_t aux = low_eighth(src);
|
|
|
|
touch(aux);
|
|
x = broadcast_eighth(aux);
|
|
touch(aux);
|
|
y = insert_eighth(src, aux, 1);
|
|
touch(aux);
|
|
y = insert_eighth(y, aux, 2);
|
|
touch(aux);
|
|
y = insert_eighth(y, aux, 3);
|
|
touch(aux);
|
|
y = insert_eighth(y, aux, 4);
|
|
touch(aux);
|
|
y = insert_eighth(y, aux, 5);
|
|
touch(aux);
|
|
y = insert_eighth(y, aux, 6);
|
|
touch(aux);
|
|
y = insert_eighth(y, aux, 7);
|
|
if ( !eq(x, y) ) return __LINE__;
|
|
}
|
|
#endif
|
|
|
|
#if defined(interleave_lo) && defined(interleave_hi)
|
|
touch(src);
|
|
x = interleave_lo(inv, src);
|
|
touch(src);
|
|
y = interleave_hi(inv, src);
|
|
touch(src);
|
|
# ifdef UINT_SIZE
|
|
z = ((x - y) ^ ~alt) - ~alt;
|
|
# else
|
|
z = (x - y) * alt;
|
|
# endif
|
|
# ifdef broadcast
|
|
if ( !eq(z, broadcast(ELEM_COUNT / 2)) ) return __LINE__;
|
|
# else
|
|
if ( !eq(z, ELEM_COUNT / 2) ) return __LINE__;
|
|
# endif
|
|
#endif
|
|
|
|
#if defined(INT_SIZE) && defined(widen1) && defined(interleave_lo)
|
|
|
|
x = src * alt;
|
|
y = interleave_lo(x, alt < 0);
|
|
touch(x);
|
|
z = widen1(low_half(x));
|
|
touch(x);
|
|
if ( !eq(z, y) ) return __LINE__;
|
|
|
|
# ifdef widen2
|
|
y = interleave_lo(alt < 0, alt < 0);
|
|
y = interleave_lo(z, y);
|
|
touch(x);
|
|
z = widen2(low_quarter(x));
|
|
touch(x);
|
|
if ( !eq(z, y) ) return __LINE__;
|
|
|
|
# ifdef widen3
|
|
y = interleave_lo(alt < 0, alt < 0);
|
|
y = interleave_lo(y, y);
|
|
y = interleave_lo(z, y);
|
|
touch(x);
|
|
z = widen3(low_eighth(x));
|
|
touch(x);
|
|
if ( !eq(z, y) ) return __LINE__;
|
|
# endif
|
|
# endif
|
|
|
|
#endif
|
|
|
|
#if defined(UINT_SIZE) && defined(interleave_lo)
|
|
|
|
y = interleave_lo(src, (vec_t){});
|
|
z = interleave_lo(y, (vec_t){});
|
|
|
|
# ifdef widen1
|
|
touch(src);
|
|
x = widen1(low_half(src));
|
|
touch(src);
|
|
if ( !eq(x, y) ) return __LINE__;
|
|
# endif
|
|
|
|
# ifdef widen2
|
|
touch(src);
|
|
x = widen2(low_quarter(src));
|
|
touch(src);
|
|
if ( !eq(x, z) ) return __LINE__;
|
|
# endif
|
|
|
|
# ifdef widen3
|
|
touch(src);
|
|
x = widen3(low_eighth(src));
|
|
touch(src);
|
|
if ( !eq(x, interleave_lo(z, (vec_t){})) ) return __LINE__;
|
|
# endif
|
|
|
|
#endif
|
|
|
|
#if defined(widen1) && defined(shrink1)
|
|
{
|
|
half_t aux1 = low_half(src), aux2;
|
|
|
|
touch(aux1);
|
|
x = widen1(aux1);
|
|
touch(x);
|
|
aux2 = shrink1(x);
|
|
touch(aux2);
|
|
for ( i = 0; i < ELEM_COUNT / 2; ++i )
|
|
if ( aux2[i] != src[i] )
|
|
return __LINE__;
|
|
}
|
|
#endif
|
|
|
|
#if defined(widen2) && defined(shrink2)
|
|
{
|
|
quarter_t aux1 = low_quarter(src), aux2;
|
|
|
|
touch(aux1);
|
|
x = widen2(aux1);
|
|
touch(x);
|
|
aux2 = shrink2(x);
|
|
touch(aux2);
|
|
for ( i = 0; i < ELEM_COUNT / 4; ++i )
|
|
if ( aux2[i] != src[i] )
|
|
return __LINE__;
|
|
}
|
|
#endif
|
|
|
|
#if defined(widen3) && defined(shrink3)
|
|
{
|
|
eighth_t aux1 = low_eighth(src), aux2;
|
|
|
|
touch(aux1);
|
|
x = widen3(aux1);
|
|
touch(x);
|
|
aux2 = shrink3(x);
|
|
touch(aux2);
|
|
for ( i = 0; i < ELEM_COUNT / 8; ++i )
|
|
if ( aux2[i] != src[i] )
|
|
return __LINE__;
|
|
}
|
|
#endif
|
|
|
|
#ifdef dup_lo
|
|
touch(src);
|
|
x = dup_lo(src);
|
|
touch(src);
|
|
if ( !eq(x - src, (alt - 1) / 2) ) return __LINE__;
|
|
#endif
|
|
|
|
#ifdef dup_hi
|
|
touch(src);
|
|
x = dup_hi(src);
|
|
touch(src);
|
|
if ( !eq(x - src, (alt + 1) / 2) ) return __LINE__;
|
|
#endif
|
|
|
|
for ( y = src, i = 1; i < ELEM_COUNT; i += 2 )
|
|
y[i] = inv[i];
|
|
|
|
#ifdef select
|
|
# ifdef UINT_SIZE
|
|
select(&z, src, inv, alt);
|
|
# else
|
|
select(&z, src, inv, alt > 0);
|
|
# endif
|
|
if ( !eq(z, y) ) return __LINE__;
|
|
#endif
|
|
|
|
#ifdef select2
|
|
# ifdef UINT_SIZE
|
|
select2(&z, src, inv, alt);
|
|
# else
|
|
select2(&z, src, inv, alt > 0);
|
|
# endif
|
|
if ( !eq(z, y) ) return __LINE__;
|
|
#endif
|
|
|
|
#ifdef mix
|
|
touch(src);
|
|
touch(inv);
|
|
x = mix(src, inv);
|
|
if ( !eq(x, y) ) return __LINE__;
|
|
|
|
# ifdef addsub
|
|
touch(src);
|
|
touch(inv);
|
|
x = addsub(src, inv);
|
|
touch(src);
|
|
touch(inv);
|
|
y = mix(src - inv, src + inv);
|
|
if ( !eq(x, y) ) return __LINE__;
|
|
# endif
|
|
#endif
|
|
|
|
#ifdef rotr
|
|
x = rotr(src, 1);
|
|
y = (src & (ELEM_COUNT - 1)) + 1;
|
|
if ( !eq(x, y) ) return __LINE__;
|
|
#endif
|
|
|
|
#ifdef dot_product
|
|
touch(src);
|
|
touch(inv);
|
|
x = dot_product(src, inv);
|
|
if ( !eq(x, (vec_t){ (ELEM_COUNT * (ELEM_COUNT + 1) *
|
|
(ELEM_COUNT + 2)) / 6 }) ) return __LINE__;
|
|
#endif
|
|
|
|
#ifdef hadd
|
|
# if (!defined(INT_SIZE) || INT_SIZE > 1 || ELEM_COUNT < 16) && \
|
|
(!defined(UINT_SIZE) || UINT_SIZE > 1 || ELEM_COUNT <= 16)
|
|
x = src;
|
|
for ( i = ELEM_COUNT; i >>= 1; )
|
|
{
|
|
touch(x);
|
|
x = hadd((vec_t){}, x);
|
|
}
|
|
if ( x[ELEM_COUNT - 1] != (ELEM_COUNT * (ELEM_COUNT + 1)) / 2 ) return __LINE__;
|
|
# endif
|
|
|
|
# ifdef hsub
|
|
touch(src);
|
|
touch(inv);
|
|
x = hsub(src, inv);
|
|
for ( i = ELEM_COUNT; i >>= 1; )
|
|
x = hadd(x, (vec_t){});
|
|
if ( !eq(x, (vec_t){}) ) return __LINE__;
|
|
# endif
|
|
#endif
|
|
|
|
#if defined(getexp) && defined(getmant)
|
|
touch(src);
|
|
x = getmant(src);
|
|
touch(src);
|
|
y = getexp(src);
|
|
touch(src);
|
|
for ( j = i = 0; i < ELEM_COUNT; ++i )
|
|
{
|
|
if ( y[i] != j ) return __LINE__;
|
|
|
|
if ( !((i + 1) & (i + 2)) )
|
|
++j;
|
|
|
|
if ( !(i & (i + 1)) && x[i] != 1 ) return __LINE__;
|
|
}
|
|
# ifdef scale
|
|
touch(y);
|
|
z = scale(x, y);
|
|
if ( !eq(src, z) ) return __LINE__;
|
|
# endif
|
|
#endif
|
|
|
|
#if (defined(__XOP__) && VEC_SIZE == 16 && (INT_SIZE == 2 || INT_SIZE == 4)) || \
|
|
(defined(__AVX512F__) && defined(FLOAT_SIZE))
|
|
return -fma_test();
|
|
#endif
|
|
|
|
return 0;
|
|
}
|