27#if defined(HIGHWAY_HWY_TESTS_TEST_UTIL_INL_H_) == \
28 defined(HWY_TARGET_TOGGLE)
29#ifdef HIGHWAY_HWY_TESTS_TEST_UTIL_INL_H_
30#undef HIGHWAY_HWY_TESTS_TEST_UTIL_INL_H_
32#define HIGHWAY_HWY_TESTS_TEST_UTIL_INL_H_
42template <
class D,
typename T = TFromD<D>,
class V = Vec<D>>
44 const char* filename,
const int line) {
46 auto actual_lanes = AllocateAligned<T>(
N);
47 Store(actual,
d, actual_lanes.get());
49 const auto info = hwy::detail::MakeTypeInfo<T>();
52 target_name, filename,
line);
58template <
class D,
typename T = TFromD<D>,
class V = Vec<D>>
60 const char* filename,
int line) {
61 auto expected_lanes = AllocateAligned<T>(
Lanes(
d));
62 Store(expected,
d, expected_lanes.get());
69 const char* filename,
int line) {
81#if HWY_TARGET == HWY_SCALAR
86 const size_t N8 =
Lanes(d8);
87 auto bits_a = AllocateAligned<uint8_t>(
HWY_MAX(8, N8));
88 auto bits_b = AllocateAligned<uint8_t>(
HWY_MAX(8, N8));
89 memset(bits_a.get(), 0, N8);
90 memset(bits_b.get(), 0, N8);
96 for (; i <
N / 8; ++i) {
97 if (bits_a[i] != bits_b[i]) {
98 fprintf(stderr,
"Mismatch in byte %" PRIu64
": %d != %d\n",
99 static_cast<uint64_t
>(i), bits_a[i], bits_b[i]);
100 Print(d8,
"expect",
Load(d8, bits_a.get()), 0, N8);
101 Print(d8,
"actual",
Load(d8, bits_b.get()), 0, N8);
102 hwy::Abort(filename,
line,
"Masks not equal");
106 const size_t remainder =
N % 8;
107 if (remainder != 0) {
108 const int mask = (1 << remainder) - 1;
109 const int valid_a = bits_a[i] & mask;
110 const int valid_b = bits_b[i] & mask;
111 if (valid_a != valid_b) {
112 fprintf(stderr,
"Mismatch in last byte %" PRIu64
": %d != %d\n",
113 static_cast<uint64_t
>(i), valid_a, valid_b);
114 Print(d8,
"expect",
Load(d8, bits_a.get()), 0, N8);
115 Print(d8,
"actual",
Load(d8, bits_b.get()), 0, N8);
116 hwy::Abort(filename,
line,
"Masks not equal");
138#define HWY_ASSERT_EQ(expected, actual) \
139 hwy::AssertEqual(expected, actual, hwy::TargetName(HWY_TARGET), __FILE__, \
142#define HWY_ASSERT_ARRAY_EQ(expected, actual, count) \
143 hwy::AssertArrayEqual(expected, actual, count, hwy::TargetName(HWY_TARGET), \
146#define HWY_ASSERT_STRING_EQ(expected, actual) \
147 hwy::AssertStringEqual(expected, actual, hwy::TargetName(HWY_TARGET), \
150#define HWY_ASSERT_VEC_EQ(d, expected, actual) \
151 AssertVecEqual(d, expected, actual, __FILE__, __LINE__)
153#define HWY_ASSERT_MASK_EQ(d, expected, actual) \
154 AssertMaskEqual(d, expected, actual, __FILE__, __LINE__)
166template <
typename T,
size_t kMul,
size_t kMinArg,
class Test>
168 static void Do(
size_t min_lanes,
size_t max_lanes) {
172 const size_t lanes =
Lanes(
d);
173 if (lanes < min_lanes)
return;
175 if (lanes <= max_lanes) {
183template <
typename T,
size_t kMinArg,
class Test>
185 static void Do(
size_t,
size_t) {}
191constexpr int MinPow2() {
200template <
typename T,
int kPow2,
int kAddPow2,
class Test>
201struct ForeachShiftR {
202 static void Do(
size_t min_lanes) {
203 const ScalableTag<T, kPow2 + kAddPow2>
d;
207 if (
Lanes(
d) >= min_lanes) {
210 fprintf(stderr,
"%d lanes < %d: T=%d pow=%d\n",
211 static_cast<int>(
Lanes(
d)),
static_cast<int>(min_lanes),
212 static_cast<int>(
sizeof(T)), kPow2 + kAddPow2);
216 ForeachShiftR<T, kPow2 + 1, kAddPow2, Test>::Do(min_lanes);
221template <
typename T,
int kAddPow2,
class Test>
222struct ForeachShiftR<T, 4, kAddPow2, Test> {
223 static void Do(
size_t) {}
241template <
class Test,
int kPow2 = 1>
248 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
252 template <
typename T>
255 constexpr size_t kMaxCapped =
HWY_LANES(T);
260#if HWY_TARGET == HWY_SCALAR
264#if HWY_TARGET == HWY_RVV
266 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2, -kPow2, Test>::Do(1);
267#elif HWY_HAVE_SCALABLE
269 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2 + 3, -kPow2 - 3,
278template <
class Test,
int kPow2 = 1>
285 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
289 template <
typename T>
292 constexpr size_t kMinLanes =
size_t{1} << kPow2;
293 constexpr size_t kMaxCapped =
HWY_LANES(T);
295 constexpr size_t max_lanes = kMaxCapped;
300#if HWY_TARGET == HWY_SCALAR
304 kMinLanes, max_lanes);
305#if HWY_TARGET == HWY_RVV
307 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2, 0, Test>::Do(
309#elif HWY_HAVE_SCALABLE
311 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2 + 3, -3, Test>::Do(
320template <
size_t kMinBits,
class Test>
327 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
331 template <
typename T>
334 constexpr size_t kMaxCapped =
HWY_LANES(T);
335 constexpr size_t kMinLanes = kMinBits / 8 /
sizeof(T);
337 constexpr size_t max_lanes = kMaxCapped;
339#if HWY_TARGET == HWY_SCALAR
343 kMinLanes, max_lanes);
344#if HWY_TARGET == HWY_RVV
346 constexpr size_t kRatio = 128 / kMinBits;
347 constexpr int kMinPow2 =
348 kRatio == 0 ? 0 : -
static_cast<int>(
CeilLog2(kRatio));
350 detail::ForeachShiftR<T, kMinPow2, 0, Test>::Do(kMinLanes);
351#elif HWY_HAVE_SCALABLE
353 constexpr size_t kRatio = 128 / kMinBits;
354 constexpr int kMinPow2 =
355 kRatio == 0 ? 0 : -
static_cast<int>(
CeilLog2(kRatio));
357 detail::ForeachShiftR<T, kMinPow2 + 3, -3, Test>::Do(kMinLanes);
368template <
class Test,
int kPow2 = 1>
375 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
379 template <
typename T>
382 constexpr size_t kFactor =
size_t{1} << kPow2;
383 static_assert(kFactor >= 2 && kFactor *
sizeof(T) <=
sizeof(uint64_t),
"");
384 constexpr size_t kMaxCapped =
HWY_LANES(T);
385 constexpr size_t kMinLanes = kFactor;
391#if HWY_TARGET == HWY_SCALAR
397#if HWY_TARGET == HWY_RVV
399 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2, -kPow2, Test>::Do(
401#elif HWY_HAVE_SCALABLE
403 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2 + 3, -kPow2 - 3,
404 Test>::Do(kMinLanes);
412template <
class Test,
int kPow2 = 1>
419 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
423 template <
typename T>
426 constexpr size_t kMinLanes =
size_t{1} << kPow2;
427 constexpr size_t kMaxCapped =
HWY_LANES(T);
429 constexpr size_t max_lanes = kMaxCapped;
434#if HWY_TARGET == HWY_SCALAR
438 kMinLanes, max_lanes);
441#if HWY_TARGET == HWY_RVV
443 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2, 0, Test>::Do(
445#elif HWY_HAVE_SCALABLE
447 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2 + 3, -3, Test>::Do(
455template <
class Test,
int kPow2 = 1>
462 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
466 template <
typename T>
469#if HWY_TARGET == HWY_SCALAR
472 constexpr size_t kMinLanes =
size_t{1} << kPow2;
474 constexpr size_t kMaxCapped =
HWY_LANES(T);
476 kMinLanes, kMaxCapped);
479#if HWY_TARGET == HWY_RVV
481 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2, 0, Test>::Do(
483#elif HWY_HAVE_SCALABLE
485 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2 + 3, -3, Test>::Do(
501 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
505 template <
typename T>
508#if HWY_TARGET == HWY_SCALAR
524#if HWY_HAVE_INTEGER64
534#if HWY_HAVE_INTEGER64
593#if HWY_HAVE_INTEGER64
635 return HWY_MAX(max_reps / 32, 2);
636#elif HWY_IS_DEBUG_BUILD
637 return HWY_MAX(max_reps / 8, 2);
639 return HWY_MAX(max_reps / 4, 2);
649 return HWY_MIN(max_pow2 - 4, max_pow2);
650#elif HWY_IS_DEBUG_BUILD
651 return HWY_MIN(max_pow2 - 1, max_pow2);
653 return HWY_MIN(max_pow2 - 1, max_pow2);
#define HWY_MAX(a, b)
Definition: base.h:126
#define HWY_NOINLINE
Definition: base.h:63
#define HWY_MIN(a, b)
Definition: base.h:125
#define HWY_ABORT(format,...)
Definition: base.h:141
#define HWY_INLINE
Definition: base.h:62
#define HWY_ASSERT(condition)
Definition: base.h:145
Definition: test_util-inl.h:413
~ForDemoteVectors()
Definition: test_util-inl.h:417
void operator()(T) const
Definition: test_util-inl.h:424
bool called_
Definition: test_util-inl.h:414
Definition: test_util-inl.h:242
void operator()(T) const
Definition: test_util-inl.h:253
bool called_
Definition: test_util-inl.h:243
~ForExtendableVectors()
Definition: test_util-inl.h:246
Definition: test_util-inl.h:321
bool called_
Definition: test_util-inl.h:322
~ForGEVectors()
Definition: test_util-inl.h:325
void operator()(T) const
Definition: test_util-inl.h:332
Definition: test_util-inl.h:456
~ForHalfVectors()
Definition: test_util-inl.h:460
bool called_
Definition: test_util-inl.h:457
void operator()(T) const
Definition: test_util-inl.h:467
Definition: test_util-inl.h:495
bool called_
Definition: test_util-inl.h:496
void operator()(T t) const
Definition: test_util-inl.h:506
~ForPartialVectors()
Definition: test_util-inl.h:499
Definition: test_util-inl.h:279
void operator()(T) const
Definition: test_util-inl.h:290
bool called_
Definition: test_util-inl.h:280
~ForShrinkableVectors()
Definition: test_util-inl.h:283
#define HWY_TARGET
Definition: detect_targets.h:341
d
Definition: rvv-inl.h:1742
V VecArg
Definition: ops/shared-inl.h:306
HWY_API Mask128< TTo, N > RebindMask(Simd< TTo, N, 0 > dto, Mask128< TFrom, N > m)
Definition: arm_neon-inl.h:2189
constexpr size_t AdjustedReps(size_t max_reps)
Definition: test_util-inl.h:633
HWY_API bool AllTrue(const Full128< T > d, const Mask128< T > m)
Definition: arm_neon-inl.h:5305
HWY_API auto Lt(V a, V b) -> decltype(a==b)
Definition: arm_neon-inl.h:6309
void ForUIF32(const Func &func)
Definition: test_util-inl.h:586
void ForUI163264(const Func &func)
Definition: test_util-inl.h:620
HWY_API Mask128< T, N > FirstN(const Simd< T, N, 0 > d, size_t num)
Definition: arm_neon-inl.h:2409
HWY_API size_t StoreMaskBits(Simd< T, N, 0 >, const Mask128< T, N > mask, uint8_t *bits)
Definition: arm_neon-inl.h:5290
void ForUIF3264(const Func &func)
Definition: test_util-inl.h:614
void ForUIF163264(const Func &func)
Definition: test_util-inl.h:626
constexpr size_t AdjustedLog2Reps(size_t max_pow2)
Definition: test_util-inl.h:646
typename detail::CappedTagChecker< T, kLimit >::type CappedTag
Definition: ops/shared-inl.h:172
void ForUI32(const Func &func)
Definition: test_util-inl.h:580
void ForAllTypes(const Func &func)
Definition: test_util-inl.h:554
Rebind< MakeSigned< TFromD< D > >, D > RebindToSigned
Definition: ops/shared-inl.h:198
void ForFloatTypes(const Func &func)
Definition: test_util-inl.h:546
void Print(const D d, const char *caption, VecArg< V > v, size_t lane_u=0, size_t max_lanes=7)
Definition: print-inl.h:39
HWY_NOINLINE void AssertMaskEqual(D d, VecArg< Mask< D > > a, VecArg< Mask< D > > b, const char *filename, int line)
Definition: test_util-inl.h:68
HWY_API size_t CountTrue(Full128< T >, const Mask128< T > mask)
Definition: arm_neon-inl.h:5269
HWY_API Vec128< T, N > VecFromMask(Simd< T, N, 0 > d, const Mask128< T, N > v)
Definition: arm_neon-inl.h:2182
HWY_INLINE void AssertVecEqual(D d, const T *expected, VecArg< V > actual, const char *filename, const int line)
Definition: test_util-inl.h:43
void ForIntegerTypes(const Func &func)
Definition: test_util-inl.h:540
HWY_API constexpr size_t Lanes(Simd< T, N, kPow2 >)
Definition: arm_sve-inl.h:236
HWY_API Vec128< T, N > Load(Simd< T, N, 0 > d, const T *HWY_RESTRICT p)
Definition: arm_neon-inl.h:2706
HWY_INLINE Mask< D > MaskFalse(const D d)
Definition: test_util-inl.h:131
void ForUI8(const Func &func)
Definition: test_util-inl.h:560
void ForUI3264(const Func &func)
Definition: test_util-inl.h:608
typename detail::ScalableTagChecker< T, kPow2 >::type ScalableTag
Definition: ops/shared-inl.h:161
HWY_API bool AllFalse(const Simd< T, N, 0 > d, const Mask128< T, N > m)
Definition: arm_neon-inl.h:5299
void ForUIF64(const Func &func)
Definition: test_util-inl.h:600
void ForUI16(const Func &func)
Definition: test_util-inl.h:566
typename D::template Rebind< T > Rebind
Definition: ops/shared-inl.h:195
HWY_API Vec128< T, N > Zero(Simd< T, N, 0 > d)
Definition: arm_neon-inl.h:1011
void ForUI64(const Func &func)
Definition: test_util-inl.h:592
void ForSignedTypes(const Func &func)
Definition: test_util-inl.h:520
void ForUIF16(const Func &func)
Definition: test_util-inl.h:572
typename D::template Repartition< T > Repartition
Definition: ops/shared-inl.h:206
decltype(MaskFromVec(Zero(D()))) Mask
Definition: generic_ops-inl.h:38
N
Definition: rvv-inl.h:1742
HWY_INLINE Mask< D > MaskTrue(const D d)
Definition: test_util-inl.h:126
HWY_API void Store(Vec128< T, N > v, Simd< T, N, 0 > d, T *HWY_RESTRICT aligned)
Definition: arm_neon-inl.h:2882
void ForUnsignedTypes(const Func &func)
Definition: test_util-inl.h:530
decltype(Zero(D())) Vec
Definition: generic_ops-inl.h:32
HWY_TEST_DLLEXPORT void AssertArrayEqual(const TypeInfo &info, const void *expected_void, const void *actual_void, size_t N, const char *target_name, const char *filename, int line)
Definition: aligned_allocator.h:27
static HWY_MAYBE_UNUSED const char * TargetName(uint32_t target)
Definition: targets.h:77
FuncOutput(*)(const void *, FuncInput) Func
Definition: nanobenchmark.h:105
constexpr size_t CeilLog2(TI x)
Definition: base.h:777
HWY_INLINE void AssertEqual(const T expected, const T actual, const char *target_name, const char *filename, int line, size_t lane=0)
Definition: test_util.h:151
HWY_DLLEXPORT HWY_NORETURN void int line
Definition: base.h:848
#define HWY_LANES(T)
Definition: set_macros-inl.h:85
#define HWY_NAMESPACE
Definition: set_macros-inl.h:82
static void Do(size_t, size_t)
Definition: test_util-inl.h:185
Definition: test_util-inl.h:167
static void Do(size_t min_lanes, size_t max_lanes)
Definition: test_util-inl.h:168