mirror of
https://github.com/xen-project/xen.git
synced 2026-10-29 18:50:05 +01:00
Jan Beulich
b8bc4588b3
For one it was an oversight to leave dup_{hi,lo}() undefined for 512-bit
vector size. And then in FMA testing we can also arrange for the
compiler to (hopefully) recognize broadcasting potential. Plus we can
replace the broadcast(1) use in the addsub() surrogate with inline
assembly explicitly using embedded broadcast (even gcc12 still doesn't
support broadcast for any of the addsub/subadd builtins).
Signed-off-by: Jan Beulich <jbeulich@suse.com>
Acked-by: Andrew Cooper <andrew.cooper3@citrix.com>
161 lines
3.8 KiB
C
161 lines
3.8 KiB
C
#if !defined(__XOP__) && !defined(__AVX512F__)
|
|
#include "simd.h"
|
|
ENTRY(fma_test);
|
|
#endif
|
|
|
|
#if VEC_SIZE < 16 && !defined(to_bool)
|
|
# define to_bool(cmp) (!~(cmp)[0])
|
|
#elif VEC_SIZE == 16 && !defined(__AVX512VL__)
|
|
# if FLOAT_SIZE == 4
|
|
# define to_bool(cmp) __builtin_ia32_vtestcps(cmp, (vec_t){} == 0)
|
|
# elif FLOAT_SIZE == 8
|
|
# define to_bool(cmp) __builtin_ia32_vtestcpd(cmp, (vec_t){} == 0)
|
|
# endif
|
|
#elif VEC_SIZE == 32 && !defined(__AVX512VL__)
|
|
# if FLOAT_SIZE == 4
|
|
# define to_bool(cmp) __builtin_ia32_vtestcps256(cmp, (vec_t){} == 0)
|
|
# elif FLOAT_SIZE == 8
|
|
# define to_bool(cmp) __builtin_ia32_vtestcpd256(cmp, (vec_t){} == 0)
|
|
# endif
|
|
#endif
|
|
|
|
#ifndef eq
|
|
# define eq(x, y) to_bool((x) == (y))
|
|
#endif
|
|
|
|
#if defined(__AVX512F__) && VEC_SIZE > FLOAT_SIZE
|
|
# if FLOAT_SIZE == 4
|
|
# define fmaddsub(x, y, z) BR(vfmaddsubps, _mask, x, y, z, ~0)
|
|
# elif FLOAT_SIZE == 8
|
|
# define fmaddsub(x, y, z) BR(vfmaddsubpd, _mask, x, y, z, ~0)
|
|
# endif
|
|
#elif VEC_SIZE == 16
|
|
# if FLOAT_SIZE == 4
|
|
# define addsub(x, y) __builtin_ia32_addsubps(x, y)
|
|
# if defined(__FMA4__) || defined(__FMA__)
|
|
# define fmaddsub(x, y, z) __builtin_ia32_vfmaddsubps(x, y, z)
|
|
# endif
|
|
# elif FLOAT_SIZE == 8
|
|
# define addsub(x, y) __builtin_ia32_addsubpd(x, y)
|
|
# if defined(__FMA4__) || defined(__FMA__)
|
|
# define fmaddsub(x, y, z) __builtin_ia32_vfmaddsubpd(x, y, z)
|
|
# endif
|
|
# endif
|
|
#elif VEC_SIZE == 32
|
|
# if FLOAT_SIZE == 4
|
|
# define addsub(x, y) __builtin_ia32_addsubps256(x, y)
|
|
# if defined(__FMA4__) || defined(__FMA__)
|
|
# define fmaddsub(x, y, z) __builtin_ia32_vfmaddsubps256(x, y, z)
|
|
# endif
|
|
# elif FLOAT_SIZE == 8
|
|
# define addsub(x, y) __builtin_ia32_addsubpd256(x, y)
|
|
# if defined(__FMA4__) || defined(__FMA__)
|
|
# define fmaddsub(x, y, z) __builtin_ia32_vfmaddsubpd256(x, y, z)
|
|
# endif
|
|
# endif
|
|
#endif
|
|
|
|
#if defined(fmaddsub) && !defined(addsub)
|
|
# ifdef __AVX512F__
|
|
# define addsub(x, y) ({ \
|
|
vec_t t_; \
|
|
typeof(t_[0]) one_ = 1; \
|
|
asm ( "vfmaddsub231p" ELEM_SFX " %2%{1to%c4%}, %1, %0" \
|
|
: "=v" (t_) \
|
|
: "v" (x), "m" (one_), "0" (y), "i" (ELEM_COUNT) ); \
|
|
t_; \
|
|
})
|
|
# else
|
|
# define addsub(x, y) fmaddsub(x, broadcast(1), y)
|
|
# endif
|
|
#endif
|
|
|
|
int fma_test(void)
|
|
{
|
|
unsigned int i;
|
|
vec_t x, y, z, src, inv, one;
|
|
#ifdef __AVX512F__
|
|
typeof(one[0]) one_ = 1;
|
|
#endif
|
|
|
|
for ( i = 0; i < ELEM_COUNT; ++i )
|
|
{
|
|
src[i] = i + 1;
|
|
inv[i] = ELEM_COUNT - i;
|
|
one[i] = 1;
|
|
}
|
|
|
|
#ifdef __AVX512F__
|
|
# define one one_
|
|
#endif
|
|
|
|
x = (src + one) * inv;
|
|
y = (src - one) * inv;
|
|
touch(src);
|
|
z = inv * src + inv;
|
|
if ( !eq(x, z) ) return __LINE__;
|
|
|
|
touch(src);
|
|
z = -inv * src - inv;
|
|
if ( !eq(-x, z) ) return __LINE__;
|
|
|
|
touch(src);
|
|
z = inv * src - inv;
|
|
if ( !eq(y, z) ) return __LINE__;
|
|
|
|
touch(src);
|
|
z = -inv * src + inv;
|
|
if ( !eq(-y, z) ) return __LINE__;
|
|
touch(src);
|
|
|
|
x = src + inv;
|
|
y = src - inv;
|
|
touch(inv);
|
|
touch(one);
|
|
z = src * one + inv;
|
|
if ( !eq(x, z) ) return __LINE__;
|
|
|
|
touch(inv);
|
|
touch(one);
|
|
z = -src * one - inv;
|
|
if ( !eq(-x, z) ) return __LINE__;
|
|
|
|
touch(inv);
|
|
touch(one);
|
|
z = src * one - inv;
|
|
if ( !eq(y, z) ) return __LINE__;
|
|
|
|
touch(inv);
|
|
touch(one);
|
|
z = -src * one + inv;
|
|
if ( !eq(-y, z) ) return __LINE__;
|
|
touch(inv);
|
|
|
|
#undef one
|
|
|
|
#if defined(addsub) && defined(fmaddsub)
|
|
x = addsub(src * inv, one);
|
|
y = addsub(src * inv, -one);
|
|
touch(one);
|
|
z = fmaddsub(src, inv, one);
|
|
if ( !eq(x, z) ) return __LINE__;
|
|
|
|
touch(one);
|
|
z = fmaddsub(src, inv, -one);
|
|
if ( !eq(y, z) ) return __LINE__;
|
|
touch(one);
|
|
|
|
x = addsub(src * inv, one);
|
|
touch(inv);
|
|
z = fmaddsub(src, inv, one);
|
|
if ( !eq(x, z) ) return __LINE__;
|
|
|
|
touch(inv);
|
|
z = fmaddsub(src, inv, -one);
|
|
if ( !eq(y, z) ) return __LINE__;
|
|
touch(inv);
|
|
#endif
|
|
|
|
return 0;
|
|
}
|