18 #ifndef HWY_HIGHWAY_INCLUDED
19 #define HWY_HIGHWAY_INCLUDED
38 #define HWY_FULL1(T) hwy::HWY_NAMESPACE::Simd<T, HWY_LANES(T)>
39 #define HWY_3TH_ARG(arg1, arg2, arg3, ...) arg3
41 #define HWY_FULL_RECOMPOSER(args_with_paren) HWY_3TH_ARG args_with_paren
43 #define HWY_CHOOSE_FULL(...) \
44 HWY_FULL_RECOMPOSER((__VA_ARGS__, HWY_FULL2, HWY_FULL1, ))
45 #define HWY_FULL(...) HWY_CHOOSE_FULL(__VA_ARGS__())(__VA_ARGS__)
48 #define HWY_CAPPED(T, MAX_N) \
49 hwy::HWY_NAMESPACE::Simd<T, HWY_MIN(MAX_N, HWY_LANES(T))>
65 #if HWY_STATIC_TARGET == HWY_SCALAR
66 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_SCALAR::FUNC_NAME
67 #elif HWY_STATIC_TARGET == HWY_RVV
68 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_RVV::FUNC_NAME
69 #elif HWY_STATIC_TARGET == HWY_WASM
70 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_WASM::FUNC_NAME
71 #elif HWY_STATIC_TARGET == HWY_NEON
72 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_NEON::FUNC_NAME
73 #elif HWY_STATIC_TARGET == HWY_SVE
74 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_SVE::FUNC_NAME
75 #elif HWY_STATIC_TARGET == HWY_SVE2
76 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_SVE2::FUNC_NAME
77 #elif HWY_STATIC_TARGET == HWY_PPC8
78 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_PPC8::FUNC_NAME
79 #elif HWY_STATIC_TARGET == HWY_SSSE3
80 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_SSSE3::FUNC_NAME
81 #elif HWY_STATIC_TARGET == HWY_SSE4
82 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_SSE4::FUNC_NAME
83 #elif HWY_STATIC_TARGET == HWY_AVX2
84 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX2::FUNC_NAME
85 #elif HWY_STATIC_TARGET == HWY_AVX3
86 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX3::FUNC_NAME
87 #elif HWY_STATIC_TARGET == HWY_AVX3_DL
88 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX3_DL::FUNC_NAME
93 template <
typename RetType,
typename... Args>
105 template <FunctionType* const table[]>
115 template <
typename RetType,
typename... Args>
122 #if HWY_TARGETS & HWY_SCALAR
123 #define HWY_CHOOSE_SCALAR(FUNC_NAME) &N_SCALAR::FUNC_NAME
128 #define HWY_CHOOSE_SCALAR(FUNC_NAME) &HWY_STATIC_DISPATCH(FUNC_NAME)
131 #if HWY_TARGETS & HWY_WASM
132 #define HWY_CHOOSE_WASM(FUNC_NAME) &N_WASM::FUNC_NAME
134 #define HWY_CHOOSE_WASM(FUNC_NAME) nullptr
137 #if HWY_TARGETS & HWY_RVV
138 #define HWY_CHOOSE_RVV(FUNC_NAME) &N_RVV::FUNC_NAME
140 #define HWY_CHOOSE_RVV(FUNC_NAME) nullptr
143 #if HWY_TARGETS & HWY_NEON
144 #define HWY_CHOOSE_NEON(FUNC_NAME) &N_NEON::FUNC_NAME
146 #define HWY_CHOOSE_NEON(FUNC_NAME) nullptr
149 #if HWY_TARGETS & HWY_SVE
150 #define HWY_CHOOSE_SVE(FUNC_NAME) &N_SVE::FUNC_NAME
152 #define HWY_CHOOSE_SVE(FUNC_NAME) nullptr
155 #if HWY_TARGETS & HWY_SVE2
156 #define HWY_CHOOSE_SVE2(FUNC_NAME) &N_SVE2::FUNC_NAME
158 #define HWY_CHOOSE_SVE2(FUNC_NAME) nullptr
161 #if HWY_TARGETS & HWY_PPC8
162 #define HWY_CHOOSE_PCC8(FUNC_NAME) &N_PPC8::FUNC_NAME
164 #define HWY_CHOOSE_PPC8(FUNC_NAME) nullptr
167 #if HWY_TARGETS & HWY_SSSE3
168 #define HWY_CHOOSE_SSSE3(FUNC_NAME) &N_SSSE3::FUNC_NAME
170 #define HWY_CHOOSE_SSSE3(FUNC_NAME) nullptr
173 #if HWY_TARGETS & HWY_SSE4
174 #define HWY_CHOOSE_SSE4(FUNC_NAME) &N_SSE4::FUNC_NAME
176 #define HWY_CHOOSE_SSE4(FUNC_NAME) nullptr
179 #if HWY_TARGETS & HWY_AVX2
180 #define HWY_CHOOSE_AVX2(FUNC_NAME) &N_AVX2::FUNC_NAME
182 #define HWY_CHOOSE_AVX2(FUNC_NAME) nullptr
185 #if HWY_TARGETS & HWY_AVX3
186 #define HWY_CHOOSE_AVX3(FUNC_NAME) &N_AVX3::FUNC_NAME
188 #define HWY_CHOOSE_AVX3(FUNC_NAME) nullptr
191 #if HWY_TARGETS & HWY_AVX3_DL
192 #define HWY_CHOOSE_AVX3_DL(FUNC_NAME) &N_AVX3_DL::FUNC_NAME
194 #define HWY_CHOOSE_AVX3_DL(FUNC_NAME) nullptr
197 #define HWY_DISPATCH_TABLE(FUNC_NAME) \
198 HWY_CONCAT(FUNC_NAME, HighwayDispatchTable)
229 #if HWY_IDE || ((HWY_TARGETS & (HWY_TARGETS - 1)) == 0)
235 #define HWY_EXPORT(FUNC_NAME) \
236 HWY_MAYBE_UNUSED static decltype(&HWY_STATIC_DISPATCH(FUNC_NAME)) \
237 const HWY_DISPATCH_TABLE(FUNC_NAME)[1] = { \
238 &HWY_STATIC_DISPATCH(FUNC_NAME)}
239 #define HWY_DYNAMIC_DISPATCH(FUNC_NAME) HWY_STATIC_DISPATCH(FUNC_NAME)
245 #define HWY_EXPORT(FUNC_NAME) \
246 static decltype(&HWY_STATIC_DISPATCH(FUNC_NAME)) \
247 const HWY_DISPATCH_TABLE(FUNC_NAME)[HWY_MAX_DYNAMIC_TARGETS + 2] = { \
250 &decltype(hwy::FunctionCacheFactory(&HWY_STATIC_DISPATCH( \
251 FUNC_NAME)))::ChooseAndCall<HWY_DISPATCH_TABLE(FUNC_NAME)>, \
252 HWY_CHOOSE_TARGET_LIST(FUNC_NAME), \
253 HWY_CHOOSE_SCALAR(FUNC_NAME), \
255 #define HWY_DYNAMIC_DISPATCH(FUNC_NAME) \
256 (*(HWY_DISPATCH_TABLE(FUNC_NAME)[hwy::chosen_target.GetIndex()]))
269 #if defined(HWY_HIGHWAY_PER_TARGET) == defined(HWY_TARGET_TOGGLE)
270 #ifdef HWY_HIGHWAY_PER_TARGET
271 #undef HWY_HIGHWAY_PER_TARGET
273 #define HWY_HIGHWAY_PER_TARGET
277 #if HWY_TARGET == HWY_RVV
278 #define HWY_FULL2(T, LMUL) hwy::HWY_NAMESPACE::Simd<T, HWY_LANES(T) * (LMUL)>
280 #define HWY_FULL2(T, LMUL) hwy::HWY_NAMESPACE::Simd<T, HWY_LANES(T)>
284 #if HWY_TARGET == HWY_SSSE3 || HWY_TARGET == HWY_SSE4
286 #elif HWY_TARGET == HWY_AVX2
288 #elif HWY_TARGET == HWY_AVX3 || HWY_TARGET == HWY_AVX3_DL
290 #elif HWY_TARGET == HWY_PPC8
291 #error "PPC is not yet supported"
292 #elif HWY_TARGET == HWY_NEON
294 #elif HWY_TARGET == HWY_SVE || HWY_TARGET == HWY_SVE2
296 #elif HWY_TARGET == HWY_WASM
298 #elif HWY_TARGET == HWY_RVV
300 #elif HWY_TARGET == HWY_SCALAR
303 #pragma message("HWY_TARGET does not match any known target")
Definition: aligned_allocator.h:23
FunctionCache< RetType, Args... > FunctionCacheFactory(RetType(*)(Args...))
Definition: highway.h:116
ChosenTarget chosen_target
size_t HWY_INLINE GetIndex() const
Definition: targets.h:242
RetType() FunctionType(Args...)
Definition: highway.h:96
static RetType ChooseAndCall(Args... args)
Definition: highway.h:106