27 #if defined(HIGHWAY_HWY_TESTS_TEST_UTIL_INL_H_) == \
28 defined(HWY_TARGET_TOGGLE)
29 #ifdef HIGHWAY_HWY_TESTS_TEST_UTIL_INL_H_
30 #undef HIGHWAY_HWY_TESTS_TEST_UTIL_INL_H_
32 #define HIGHWAY_HWY_TESTS_TEST_UTIL_INL_H_
42 template <
class D,
typename T = TFromD<D>,
class V = Vec<D>>
44 const char* filename,
const int line) {
46 auto actual_lanes = AllocateAligned<T>(
N);
47 Store(actual,
d, actual_lanes.get());
49 const auto info = hwy::detail::MakeTypeInfo<T>();
52 target_name, filename,
line);
58 template <
class D,
typename T = TFromD<D>,
class V = Vec<D>>
60 const char* filename,
int line) {
61 auto expected_lanes = AllocateAligned<T>(
Lanes(
d));
62 Store(expected,
d, expected_lanes.get());
69 const char* filename,
int line) {
82 const size_t N8 =
Lanes(d8);
83 auto bits_a = AllocateAligned<uint8_t>(
HWY_MAX(8, N8));
84 auto bits_b = AllocateAligned<uint8_t>(
HWY_MAX(8, N8));
85 memset(bits_a.get(), 0, N8);
86 memset(bits_b.get(), 0, N8);
92 for (; i <
N / 8; ++i) {
93 if (bits_a[i] != bits_b[i]) {
94 fprintf(stderr,
"Mismatch in byte %" PRIu64
": %d != %d\n",
95 static_cast<uint64_t
>(i), bits_a[i], bits_b[i]);
96 Print(d8,
"expect",
Load(d8, bits_a.get()), 0, N8);
97 Print(d8,
"actual",
Load(d8, bits_b.get()), 0, N8);
98 hwy::Abort(filename,
line,
"Masks not equal");
102 const size_t remainder =
N % 8;
103 if (remainder != 0) {
104 const int mask = (1 << remainder) - 1;
105 const int valid_a = bits_a[i] & mask;
106 const int valid_b = bits_b[i] & mask;
107 if (valid_a != valid_b) {
108 fprintf(stderr,
"Mismatch in last byte %" PRIu64
": %d != %d\n",
109 static_cast<uint64_t
>(i), valid_a, valid_b);
110 Print(d8,
"expect",
Load(d8, bits_a.get()), 0, N8);
111 Print(d8,
"actual",
Load(d8, bits_b.get()), 0, N8);
112 hwy::Abort(filename,
line,
"Masks not equal");
132 #ifndef HWY_ASSERT_EQ
134 #define HWY_ASSERT_EQ(expected, actual) \
135 hwy::AssertEqual(expected, actual, hwy::TargetName(HWY_TARGET), __FILE__, \
138 #define HWY_ASSERT_STRING_EQ(expected, actual) \
139 hwy::AssertStringEqual(expected, actual, hwy::TargetName(HWY_TARGET), \
142 #define HWY_ASSERT_VEC_EQ(d, expected, actual) \
143 AssertVecEqual(d, expected, actual, __FILE__, __LINE__)
145 #define HWY_ASSERT_MASK_EQ(d, expected, actual) \
146 AssertMaskEqual(d, expected, actual, __FILE__, __LINE__)
158 template <
typename T,
size_t kMul,
size_t kMinArg,
class Test>
160 static void Do(
size_t min_lanes,
size_t max_lanes) {
164 const size_t lanes =
Lanes(
d);
165 if (lanes < min_lanes)
return;
167 if (lanes <= max_lanes) {
175 template <
typename T,
size_t kMinArg,
class Test>
177 static void Do(
size_t,
size_t) {}
180 #if HWY_HAVE_SCALABLE
182 template <
typename T>
183 constexpr
int MinPow2() {
192 template <
typename T,
int kPow2,
int kAddPow2,
class Test>
193 struct ForeachShiftR {
194 static void Do(
size_t min_lanes) {
195 const ScalableTag<T, kPow2 + kAddPow2>
d;
199 if (
Lanes(
d) >= min_lanes) {
202 fprintf(stderr,
"%d lanes < %d: T=%d pow=%d\n",
203 static_cast<int>(
Lanes(
d)),
static_cast<int>(min_lanes),
204 static_cast<int>(
sizeof(T)), kPow2 + kAddPow2);
208 ForeachShiftR<T, kPow2 + 1, kAddPow2, Test>::Do(min_lanes);
213 template <
typename T,
int kAddPow2,
class Test>
214 struct ForeachShiftR<T, 4, kAddPow2, Test> {
215 static void Do(
size_t) {}
233 template <
class Test,
int kPow2 = 1>
240 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
244 template <
typename T>
247 constexpr
size_t kMaxCapped =
HWY_LANES(T);
252 #if HWY_TARGET == HWY_SCALAR
256 #if HWY_TARGET == HWY_RVV
258 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2, -kPow2, Test>::Do(1);
259 #elif HWY_HAVE_SCALABLE
261 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2 + 3, -kPow2 - 3,
270 template <
class Test,
int kPow2 = 1>
277 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
281 template <
typename T>
284 constexpr
size_t kMinLanes =
size_t{1} << kPow2;
285 constexpr
size_t kMaxCapped =
HWY_LANES(T);
287 constexpr
size_t max_lanes = kMaxCapped;
292 #if HWY_TARGET == HWY_SCALAR
296 kMinLanes, max_lanes);
297 #if HWY_TARGET == HWY_RVV
299 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2, 0, Test>::Do(
301 #elif HWY_HAVE_SCALABLE
303 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2 + 3, -3, Test>::Do(
312 template <
size_t kMinBits,
class Test>
319 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
323 template <
typename T>
326 constexpr
size_t kMaxCapped =
HWY_LANES(T);
327 constexpr
size_t kMinLanes = kMinBits / 8 /
sizeof(T);
329 constexpr
size_t max_lanes = kMaxCapped;
331 #if HWY_TARGET == HWY_SCALAR
335 kMinLanes, max_lanes);
336 #if HWY_TARGET == HWY_RVV
338 constexpr
size_t kRatio = 128 / kMinBits;
339 constexpr
int kMinPow2 =
340 kRatio == 0 ? 0 : -
static_cast<int>(
CeilLog2(kRatio));
342 detail::ForeachShiftR<T, kMinPow2, 0, Test>::Do(kMinLanes);
343 #elif HWY_HAVE_SCALABLE
345 constexpr
size_t kRatio = 128 / kMinBits;
346 constexpr
int kMinPow2 =
347 kRatio == 0 ? 0 : -
static_cast<int>(
CeilLog2(kRatio));
349 detail::ForeachShiftR<T, kMinPow2 + 3, -3, Test>::Do(kMinLanes);
355 template <
class Test>
360 template <
class Test,
int kPow2 = 1>
367 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
371 template <
typename T>
374 constexpr
size_t kFactor =
size_t{1} << kPow2;
375 static_assert(kFactor >= 2 && kFactor *
sizeof(T) <=
sizeof(uint64_t),
"");
376 constexpr
size_t kMaxCapped =
HWY_LANES(T);
377 constexpr
size_t kMinLanes = kFactor;
383 #if HWY_TARGET == HWY_SCALAR
389 #if HWY_TARGET == HWY_RVV
391 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2, -kPow2, Test>::Do(
393 #elif HWY_HAVE_SCALABLE
395 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2 + 3, -kPow2 - 3,
396 Test>::Do(kMinLanes);
404 template <
class Test,
int kPow2 = 1>
411 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
415 template <
typename T>
418 constexpr
size_t kMinLanes =
size_t{1} << kPow2;
419 constexpr
size_t kMaxCapped =
HWY_LANES(T);
421 constexpr
size_t max_lanes = kMaxCapped;
426 #if HWY_TARGET == HWY_SCALAR
430 kMinLanes, max_lanes);
433 #if HWY_TARGET == HWY_RVV
435 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2, 0, Test>::Do(
437 #elif HWY_HAVE_SCALABLE
439 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2 + 3, -3, Test>::Do(
447 template <
class Test,
int kPow2 = 1>
454 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
458 template <
typename T>
461 #if HWY_TARGET == HWY_SCALAR
464 constexpr
size_t kMinLanes =
size_t{1} << kPow2;
466 constexpr
size_t kMaxCapped =
HWY_LANES(T);
468 kMinLanes, kMaxCapped);
471 #if HWY_TARGET == HWY_RVV
473 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2, 0, Test>::Do(
475 #elif HWY_HAVE_SCALABLE
477 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2 + 3, -3, Test>::Do(
486 template <
class Test>
493 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
497 template <
typename T>
506 template <
class Func>
511 #if HWY_HAVE_INTEGER64
516 template <
class Func>
521 #if HWY_HAVE_INTEGER64
526 template <
class Func>
532 template <
class Func>
540 template <
class Func>
546 template <
class Func>
552 template <
class Func>
558 template <
class Func>
566 template <
class Func>
572 template <
class Func>
578 template <
class Func>
580 #if HWY_HAVE_INTEGER64
586 template <
class Func>
594 template <
class Func>
600 template <
class Func>
606 template <
class Func>
612 template <
class Func>
622 return HWY_MAX(max_reps / 32, 2);
623 #elif HWY_IS_DEBUG_BUILD
624 return HWY_MAX(max_reps / 8, 2);
626 return HWY_MAX(max_reps / 4, 2);
636 return HWY_MIN(max_pow2 - 4, max_pow2);
637 #elif HWY_IS_DEBUG_BUILD
638 return HWY_MIN(max_pow2 - 1, max_pow2);
640 return HWY_MIN(max_pow2 - 1, max_pow2);
#define HWY_MAX(a, b)
Definition: base.h:128
#define HWY_NOINLINE
Definition: base.h:65
#define HWY_MIN(a, b)
Definition: base.h:127
#define HWY_ABORT(format,...)
Definition: base.h:143
#define HWY_INLINE
Definition: base.h:64
#define HWY_ASSERT(condition)
Definition: base.h:147
Definition: test_util-inl.h:405
~ForDemoteVectors()
Definition: test_util-inl.h:409
void operator()(T) const
Definition: test_util-inl.h:416
bool called_
Definition: test_util-inl.h:406
Definition: test_util-inl.h:234
void operator()(T) const
Definition: test_util-inl.h:245
bool called_
Definition: test_util-inl.h:235
~ForExtendableVectors()
Definition: test_util-inl.h:238
Definition: test_util-inl.h:313
bool called_
Definition: test_util-inl.h:314
~ForGEVectors()
Definition: test_util-inl.h:317
void operator()(T) const
Definition: test_util-inl.h:324
Definition: test_util-inl.h:448
~ForHalfVectors()
Definition: test_util-inl.h:452
bool called_
Definition: test_util-inl.h:449
void operator()(T) const
Definition: test_util-inl.h:459
Definition: test_util-inl.h:487
bool called_
Definition: test_util-inl.h:488
void operator()(T t) const
Definition: test_util-inl.h:498
~ForPartialVectors()
Definition: test_util-inl.h:491
Definition: test_util-inl.h:271
void operator()(T) const
Definition: test_util-inl.h:282
bool called_
Definition: test_util-inl.h:272
~ForShrinkableVectors()
Definition: test_util-inl.h:275
#define HWY_TARGET
Definition: detect_targets.h:328
d
Definition: rvv-inl.h:1656
V VecArg
Definition: ops/shared-inl.h:301
HWY_API Mask128< TTo, N > RebindMask(Simd< TTo, N, 0 > dto, Mask128< TFrom, N > m)
Definition: arm_neon-inl.h:1688
constexpr size_t AdjustedReps(size_t max_reps)
Definition: test_util-inl.h:620
HWY_API auto Lt(V a, V b) -> decltype(a==b)
Definition: arm_neon-inl.h:5252
void ForUIF32(const Func &func)
Definition: test_util-inl.h:573
void ForUI163264(const Func &func)
Definition: test_util-inl.h:607
HWY_API Mask128< T, N > FirstN(const Simd< T, N, 0 > d, size_t num)
Definition: arm_neon-inl.h:1896
HWY_API size_t StoreMaskBits(Simd< T, N, 0 >, const Mask128< T, N > mask, uint8_t *bits)
Definition: arm_neon-inl.h:4761
HWY_API bool AllTrue(const Simd< T, N, 0 > d, const Mask128< T, N > m)
Definition: arm_neon-inl.h:4790
void ForUIF3264(const Func &func)
Definition: test_util-inl.h:601
void ForUIF163264(const Func &func)
Definition: test_util-inl.h:613
constexpr size_t AdjustedLog2Reps(size_t max_pow2)
Definition: test_util-inl.h:633
typename detail::CappedTagChecker< T, kLimit >::type CappedTag
Definition: ops/shared-inl.h:173
HWY_API Vec128< T, N > Load(Simd< T, N, 0 > d, const T *HWY_RESTRICT p)
Definition: arm_neon-inl.h:2205
HWY_API Vec128< T, N > Zero(Simd< T, N, 0 > d)
Definition: arm_neon-inl.h:733
HWY_API size_t Lanes(Simd< T, N, kPow2 > d)
Definition: arm_sve-inl.h:218
void ForUI32(const Func &func)
Definition: test_util-inl.h:567
void ForAllTypes(const Func &func)
Definition: test_util-inl.h:541
Rebind< MakeSigned< TFromD< D > >, D > RebindToSigned
Definition: ops/shared-inl.h:199
void ForFloatTypes(const Func &func)
Definition: test_util-inl.h:533
void Print(const D d, const char *caption, VecArg< V > v, size_t lane_u=0, size_t max_lanes=7)
Definition: print-inl.h:59
HWY_API size_t CountTrue(Full128< T >, const Mask128< T > mask)
Definition: arm_neon-inl.h:4742
HWY_API Vec128< T, N > VecFromMask(Simd< T, N, 0 > d, const Mask128< T, N > v)
Definition: arm_neon-inl.h:1681
HWY_INLINE void AssertVecEqual(D d, const T *expected, VecArg< V > actual, const char *filename, const int line)
Definition: test_util-inl.h:43
void ForIntegerTypes(const Func &func)
Definition: test_util-inl.h:527
HWY_API bool AllFalse(const Full128< T > d, const Mask128< T > m)
Definition: arm_neon-inl.h:4771
void ForUI8(const Func &func)
Definition: test_util-inl.h:547
void ForUI3264(const Func &func)
Definition: test_util-inl.h:595
typename detail::ScalableTagChecker< T, kPow2 >::type ScalableTag
Definition: ops/shared-inl.h:162
void ForUIF64(const Func &func)
Definition: test_util-inl.h:587
void ForUI16(const Func &func)
Definition: test_util-inl.h:553
void ForUI64(const Func &func)
Definition: test_util-inl.h:579
void ForSignedTypes(const Func &func)
Definition: test_util-inl.h:507
HWY_NOINLINE void AssertMaskEqual(D d, VecArg< Mask< D >> a, VecArg< Mask< D >> b, const char *filename, int line)
Definition: test_util-inl.h:68
void ForUIF16(const Func &func)
Definition: test_util-inl.h:559
typename D::template Repartition< T > Repartition
Definition: ops/shared-inl.h:207
decltype(MaskFromVec(Zero(D()))) Mask
Definition: generic_ops-inl.h:38
N
Definition: rvv-inl.h:1656
HWY_API void Store(Vec128< T, N > v, Simd< T, N, 0 > d, T *HWY_RESTRICT aligned)
Definition: arm_neon-inl.h:2397
HWY_INLINE Mask< D > MaskFalse(const D d)
Definition: test_util-inl.h:127
HWY_INLINE Mask< D > MaskTrue(const D d)
Definition: test_util-inl.h:122
void ForUnsignedTypes(const Func &func)
Definition: test_util-inl.h:517
decltype(Zero(D())) Vec
Definition: generic_ops-inl.h:32
HWY_TEST_DLLEXPORT void AssertArrayEqual(const TypeInfo &info, const void *expected_void, const void *actual_void, size_t N, const char *target_name, const char *filename, int line)
Definition: aligned_allocator.h:27
FuncOutput(*)(const void *, FuncInput) Func
Definition: nanobenchmark.h:105
static HWY_MAYBE_UNUSED const char * TargetName(uint32_t target)
Definition: targets.h:80
constexpr size_t CeilLog2(TI x)
Definition: base.h:700
HWY_INLINE void AssertEqual(const T expected, const T actual, const char *target_name, const char *filename, int line, size_t lane=0)
Definition: test_util.h:151
HWY_DLLEXPORT HWY_NORETURN void int line
Definition: base.h:763
#define HWY_LANES(T)
Definition: set_macros-inl.h:83
#define HWY_NAMESPACE
Definition: set_macros-inl.h:80
static void Do(size_t, size_t)
Definition: test_util-inl.h:177
Definition: test_util-inl.h:159
static void Do(size_t min_lanes, size_t max_lanes)
Definition: test_util-inl.h:160