GCC Code Coverage Report

Directory: avs_core/
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 0.0% 0 / 0 / 106
Functions: -% 0 / 0 / 0
Branches: 0.0% 0 / 0 / 668
core/avs_simd_c.h
Line Branch Exec Source
1 // AviSynth+. Copyright 2025 AviSynth+ Project
2 // https://avs-plus.net
3 // http://avisynth.nl
4 // This program is free software; you can redistribute it and/or modify
5 // it under the terms of the GNU General Public License as published by
6 // the Free Software Foundation; either version 2 of the License, or
7 // (at your option) any later version.
8 //
9 // This program is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 // GNU General Public License for more details.
13 //
14 // You should have received a copy of the GNU General Public License
15 // along with this program; if not, write to the Free Software
16 // Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA, or visit
17 // http://www.gnu.org/copyleft/gpl.html .
18 //
19 // Linking Avisynth statically or dynamically with other modules is making a
20 // combined work based on Avisynth. Thus, the terms and conditions of the GNU
21 // General Public License cover the whole combination.
22 //
23 // As a special exception, the copyright holders of Avisynth give you
24 // permission to link Avisynth with independent modules that communicate with
25 // Avisynth solely through the interfaces defined in avisynth.h, regardless of the license
26 // terms of these independent modules, and to copy and distribute the
27 // resulting combined work under terms of your choice, provided that
28 // every copy of the combined work is accompanied by a complete copy of
29 // the source code of Avisynth (the version of Avisynth used to produce the
30 // combined work), being distributed under the terms of the GNU General
31 // Public License plus this exception. An independent module is a module
32 // which is not derived from or based on Avisynth, such as 3rd-party filters,
33 // import and export plugins, or graphical user interfaces.
34
35 // SIMD-like C++ classes (C)2025 Ferenc Pintér
36
37 #ifndef __AVS_SIMD_C_H__
38 #define __AVS_SIMD_C_H__
39 // Auto-vectorization friendly types for smart compilers
40 // Some helper static functions, marked with c++ 17 [[maybe_unused]] attribute
41
42 #include <avs/config.h> // force inline variants
43 #include <algorithm>
44 #include <cstdint>
45 #include <cstring>
46 #include <iostream>
47 #include <type_traits>
48 #include <stdexcept>
49
50 #if defined(_MSC_VER) && !defined(__clang__) && defined(INTEL_INTRINSICS)
51 #include <immintrin.h> // covers SSE2 through AVX2 for MSVC
52 #endif
53
54 // Determine if we can use vector attribute
55
56 // As of 2025 clang (llvm) supports vector attributes and also produces fast code
57 // gcc can use it but may produce slower code with than without vector attributes.
58 // (gcc 12.2 on aarch64 Raspberry Pi 5 is slow)
59 // We still keep it here, maybe non-arm platforms can benefit with gcc as well.
60 // Note: llvm defines __clang__ as well
61 #if defined(__GNUC__) || defined(__clang__)
62 #define HAS_VECTOR_ATTRIBUTE 1
63 #else
64 #define HAS_VECTOR_ATTRIBUTE 0
65 #endif
66
67 // Vector type definitions
68 #if HAS_VECTOR_ATTRIBUTE
69 // GCC/Clang supports vector attribute
70 using uint8_vec32_t = uint8_t __attribute__((vector_size(32))); // 256-bit (32 bytes)
71 using uint8_vec16_t = uint8_t __attribute__((vector_size(16))); // 128-bit (16 bytes)
72 using uint8_vec8_t = uint8_t __attribute__((vector_size(8))); // 64-bit (8 bytes)
73 using uint8_vec4_t = uint8_t __attribute__((vector_size(4))); // 32-bit (4 bytes)
74
75 using int16_vec16_t = int16_t __attribute__((vector_size(32))); // 256-bit (32 bytes)
76 using int16_vec8_t = int16_t __attribute__((vector_size(16))); // 128-bit (16 bytes)
77 using int16_vec4_t = int16_t __attribute__((vector_size(8))); // 64-bit (8 bytes)
78
79 using uint16_vec16_t = uint16_t __attribute__((vector_size(32))); // 256-bit (32 bytes)
80 using uint16_vec8_t = uint16_t __attribute__((vector_size(16))); // 128-bit (16 bytes)
81 using uint16_vec4_t = uint16_t __attribute__((vector_size(8))); // 64-bit (8 bytes)
82
83 using int32_vec8_t = int32_t __attribute__((vector_size(32))); // 256-bit (32 bytes)
84 using int32_vec4_t = int32_t __attribute__((vector_size(16))); // 128-bit (16 bytes)
85 using int32_vec2_t = int32_t __attribute__((vector_size(8))); // 64-bit (8 bytes)
86
87 using float_vec8_t = float __attribute__((vector_size(32))); // 256-bit (32 bytes)
88 using float_vec4_t = float __attribute__((vector_size(16))); // 128-bit (16 bytes)
89 using float_vec2_t = float __attribute__((vector_size(8))); // 64-bit (8 bytes)
90
91 #else
92
93 // For MSVC: use __declspec(align) + plain array in a struct, may help optimizer
94 template<typename T, size_t N>
95 struct alignas(N * sizeof(T)) Avs_SimdArray {
96 T v[N];
97
98 AVS_FORCEINLINE T& operator[](size_t i) { return v[i]; }
99 AVS_FORCEINLINE const T& operator[](size_t i) const { return v[i]; }
100
101 // Trivial fill
102 AVS_FORCEINLINE void fill(T val) {
103 if constexpr (sizeof(T) == 1) {
104 memset(v, val, N); // intrinsic, single rep stosd or movdqu
105 }
106 else {
107 // Seemingly MSVC (as of in 2026) will not vectorize a loop here, but will inline these
108 for (size_t i = 0; i < N; ++i) v[i] = val;
109 }
110 }
111
112 // Iterators for compatibility
113 AVS_FORCEINLINE T* begin() { return v; }
114 AVS_FORCEINLINE T* end() { return v + N; }
115 AVS_FORCEINLINE const T* begin() const { return v; }
116 AVS_FORCEINLINE const T* end() const { return v + N; }
117
118 static constexpr size_t size() { return N; }
119 };
120
121 using uint8_vec32_t = Avs_SimdArray<uint8_t, 32>;
122 using uint8_vec16_t = Avs_SimdArray<uint8_t, 16>;
123 using uint8_vec8_t = Avs_SimdArray<uint8_t, 8>;
124 using uint8_vec4_t = Avs_SimdArray<uint8_t, 4>;
125
126 using int16_vec16_t = Avs_SimdArray<int16_t, 16>;
127 using int16_vec8_t = Avs_SimdArray<int16_t, 8>;
128 using int16_vec4_t = Avs_SimdArray<int16_t, 4>;
129
130 using uint16_vec16_t = Avs_SimdArray<uint16_t, 16>;
131 using uint16_vec8_t = Avs_SimdArray<uint16_t, 8>;
132 using uint16_vec4_t = Avs_SimdArray<uint16_t, 4>;
133
134 using int32_vec8_t = Avs_SimdArray<int32_t, 8>;
135 using int32_vec4_t = Avs_SimdArray<int32_t, 4>;
136 using int32_vec2_t = Avs_SimdArray<int32_t, 2>;
137 using float_vec8_t = Avs_SimdArray<float, 8>;
138 using float_vec4_t = Avs_SimdArray<float, 4>;
139 using float_vec2_t = Avs_SimdArray<float, 2>;
140
141 #endif // HAS_VECTOR_ATTRIBUTE
142
143 // Type traits for vector elements
144 template <typename T>
145 struct vector_traits {};
146
147
148 #define DEFINE_VECTOR_TRAITS(VecType, ElemType, Size) \
149 template <> struct vector_traits<VecType> { \
150 using element_type = ElemType; \
151 static constexpr size_t size = Size; \
152 };
153
154 // Covering 8 to 32 bytes long vectors.
155 // No int64_t or double, they are not relevant in AviSynth
156 DEFINE_VECTOR_TRAITS(uint8_vec32_t, uint8_t, 32)
157 DEFINE_VECTOR_TRAITS(uint8_vec16_t, uint8_t, 16)
158 DEFINE_VECTOR_TRAITS(uint8_vec8_t, uint8_t, 8)
159 DEFINE_VECTOR_TRAITS(uint8_vec4_t, uint8_t, 4)
160 DEFINE_VECTOR_TRAITS(int16_vec16_t, int16_t, 16)
161 DEFINE_VECTOR_TRAITS(int16_vec8_t, int16_t, 8)
162 DEFINE_VECTOR_TRAITS(int16_vec4_t, int16_t, 4)
163 DEFINE_VECTOR_TRAITS(uint16_vec16_t, uint16_t, 16)
164 DEFINE_VECTOR_TRAITS(uint16_vec8_t, uint16_t, 8)
165 DEFINE_VECTOR_TRAITS(uint16_vec4_t, uint16_t, 4)
166 DEFINE_VECTOR_TRAITS(int32_vec8_t, int32_t, 8)
167 DEFINE_VECTOR_TRAITS(int32_vec4_t, int32_t, 4)
168 DEFINE_VECTOR_TRAITS(int32_vec2_t, int32_t, 2)
169 DEFINE_VECTOR_TRAITS(float_vec8_t, float, 8)
170 DEFINE_VECTOR_TRAITS(float_vec4_t, float, 4)
171 DEFINE_VECTOR_TRAITS(float_vec2_t, float, 2)
172
173 // Specialization for array-based vector operations
174 #if !HAS_VECTOR_ATTRIBUTE
175 template <typename VecType, typename ScalarType = typename vector_traits<VecType>::element_type>
176 class VectorOps {
177 public:
178 static AVS_FORCEINLINE VecType add(const VecType& a, const VecType& b) {
179 VecType result;
180 #if defined(_MSC_VER) && !defined(__clang__) && defined(INTEL_INTRINSICS) && defined(__AVX__)
181 if constexpr (sizeof(VecType) == 32) {
182 // Force the use of YMM
183 __m256 va = _mm256_loadu_ps(reinterpret_cast<const float*>(&a));
184 __m256 vb = _mm256_loadu_ps(reinterpret_cast<const float*>(&b));
185 __m256 vr = _mm256_add_ps(va, vb);
186 _mm256_storeu_ps(reinterpret_cast<float*>(&result), vr);
187 return result;
188 }
189 #endif
190 for (size_t i = 0; i < vector_traits<VecType>::size; ++i) {
191 result[i] = a[i] + b[i];
192 }
193 return result;
194 }
195
196 static AVS_FORCEINLINE VecType subtract(const VecType& a, const VecType& b) {
197 VecType result;
198 for (size_t i = 0; i < vector_traits<VecType>::size; ++i) {
199 result[i] = a[i] - b[i];
200 }
201 return result;
202 }
203
204 static AVS_FORCEINLINE VecType multiply(const VecType& a, const VecType& b) {
205 VecType result;
206 for (size_t i = 0; i < vector_traits<VecType>::size; ++i) {
207 result[i] = a[i] * b[i];
208 }
209 return result;
210 }
211
212 static AVS_FORCEINLINE VecType scalar_multiply(const VecType& a, ScalarType scalar) {
213 VecType result;
214 #if defined(_MSC_VER) && !defined(__clang__) && defined(INTEL_INTRINSICS) && defined(__AVX__)
215 if constexpr (sizeof(VecType) == 32 && std::is_same_v<ScalarType, float>) {
216 if constexpr (sizeof(VecType) == 32) {
217 // Use 'set1' which the compiler usually optimizes to a broadcast
218 __m256 v_a = _mm256_loadu_ps((const float*)&a);
219 __m256 v_res = _mm256_mul_ps(v_a, _mm256_set1_ps(scalar));
220 _mm256_storeu_ps((float*)&result, v_res);
221 return result;
222 }
223 }
224 #endif
225 for (size_t i = 0; i < vector_traits<VecType>::size; ++i) {
226 result[i] = a[i] * scalar;
227 }
228 return result;
229 }
230
231 static AVS_FORCEINLINE VecType left_shift(const VecType& a, int shift) {
232 VecType result;
233 for (size_t i = 0; i < vector_traits<VecType>::size; ++i) {
234 result[i] = a[i] << shift;
235 }
236 return result;
237 }
238
239 static AVS_FORCEINLINE VecType right_shift(const VecType& a, int shift) {
240 VecType result;
241 for (size_t i = 0; i < vector_traits<VecType>::size; ++i) {
242 result[i] = a[i] >> shift;
243 }
244 return result;
245 }
246 };
247 #endif
248
249 // Template for vector class wrapper
250 // Passing "Derived" allows syntax for assigning a wrapper class to a derived class.
251 template <typename Derived, typename VecType>
252 class VectorWrapper {
253 private:
254 using ElementType = typename vector_traits<VecType>::element_type;
255 static constexpr size_t N = vector_traits<VecType>::size;
256 VecType data;
257
258 public:
259
260 // Default constructor
261 ✗ AVS_FORCEINLINE VectorWrapper() {
262 #if HAS_VECTOR_ATTRIBUTE
263 ✗ data = VecType{}; // empty initializer
264 #else
265 data.fill(ElementType{});
266 #endif
267 ✗ }
268
269 // Constructor from scalar
270 ✗ AVS_FORCEINLINE explicit VectorWrapper(ElementType val) {
271 #if HAS_VECTOR_ATTRIBUTE
272 ✗ data = VecType{};
273 ✗ for (size_t i = 0; i < N; ++i) {
274 ✗ data[i] = val;
275 }
276 #else
277 #if defined(_MSC_VER) && !defined(__clang__) && defined(INTEL_INTRINSICS)
278 if constexpr (sizeof(VecType) == 32 && std::is_same_v<ElementType, float>) {
279 #if defined(__AVX__)
280 if (val == 0.0f) {
281 __m256 zero = _mm256_setzero_ps();
282 _mm256_storeu_ps((float*)&data, zero);
283 return;
284 }
285 __m256 v = _mm256_set1_ps(val);
286 _mm256_storeu_ps((float*)&data, v);
287 return;
288 #else
289 // __m128 path
290 if (val == 0.0f) {
291 __m128 zero = _mm_setzero_ps();
292 _mm_storeu_ps((float*)&data, zero); // 2x128 = 32 bytes, so we can store 16 bytes at a time with
293 _mm_storeu_ps((float*)((char*)&data + 16), zero);
294 return;
295 }
296 __m128 v = _mm_set1_ps(val); // possibly optimized to a broadcast
297 _mm_storeu_ps((float*)&data, v);
298 _mm_storeu_ps((float*)((char*)&data + 16), v);
299 return;
300 #endif
301 } // 32 bytes
302 if constexpr (sizeof(VecType) == 16 && std::is_same_v<ElementType, float>) {
303 if (val == 0.0f) {
304 __m128 zero = _mm_setzero_ps();
305 _mm_storeu_ps((float*)&data, zero);
306 return;
307 }
308 __m128 v = _mm_set1_ps(val); // possibly optimized to a broadcast
309 _mm_storeu_ps((float*)&data, v);
310 return;
311 } // 16 bytes
312 #endif
313 data.fill(val);
314 #endif
315 ✗ }
316
317 // Constructor from raw vector type
318 ✗ AVS_FORCEINLINE explicit VectorWrapper(const VecType& vec) {
319 #if HAS_VECTOR_ATTRIBUTE
320 ✗ data = vec; // native vector assign, optimal
321 #else
322 memcpy(&data, &vec, sizeof(data)); // fixed-size, MSVC inlines to movq/movdqu
323 #endif
324 ✗ }
325 // Conversion operator, allowing implicit conversion to the derived type.
326 // Supporting syntax like: Int32x4 = src * coeff; where the right side is a VectorWrapper<Int32x4, int32_vec4_t>
327 AVS_FORCEINLINE operator Derived() const {
328 ✗ return static_cast<const Derived>(data);
329 }
330
331 // Copy constructor
332 // Note: There is no need to define assignment operators for move or copy, as the underlying 'data' type
333 // (either array or vector attribute) already provides support for these operations.
334 // 2026: but yes, we need! For MSVC optimizer's sake at least.
335 #ifdef HAS_VECTOR_ATTRIBUTE
336 AVS_FORCEINLINE VectorWrapper(const VectorWrapper& other) : data(other.data) {}
337 #else
338 AVS_FORCEINLINE VectorWrapper(const VectorWrapper& other) {
339 memcpy(&data, &other.data, sizeof(data));
340 }
341 AVS_FORCEINLINE VectorWrapper& operator=(const VectorWrapper& other) {
342 memcpy(&data, &other.data, sizeof(data));
343 return *this;
344 }
345 #endif
346
347 // Constructors from initializer list, e.g. x = {32768, 0, 0, 0};
348
349 // Constexpr variadic template constructor for compile-time initialization.Thx AI :)
350 template <typename... Args,
351 typename = std::enable_if_t<sizeof...(Args) + 1 == N>>
352 AVS_FORCEINLINE constexpr VectorWrapper(ElementType arg, Args... args) {
353 static_assert(sizeof...(args) + 1 == N, "Initializer list must have exactly N elements.");
354 ElementType values[] = { arg, static_cast<ElementType>(args)... };
355 for (size_t i = 0; i < N; ++i) {
356 data[i] = values[i];
357 }
358 }
359
360 // std::initializer_list constructor for runtime initialization
361 ✗ AVS_FORCEINLINE VectorWrapper(std::initializer_list<ElementType> values) {
362 ✗ if (values.size() != N) {
363 ✗ throw std::logic_error("Initializer list must have exactly N elements");
364 }
365 ✗ size_t i = 0;
366 ✗ for (auto val : values) {
367 ✗ data[i++] = val;
368 }
369 ✗ }
370
371 // Load only the lower half of the vector (e.g., 8 bytes for a 16-byte vector)
372 AVS_FORCEINLINE void load_lo(const ElementType* ptr) {
373 #if HAS_VECTOR_ATTRIBUTE
374 ✗ for (size_t i = 0; i < N / 2; ++i) {
375 ✗ data[i] = ptr[i];
376 }
377 ✗ for (size_t i = N / 2; i < N; ++i) {
378 ✗ data[i] = ElementType{}; // Zero out the upper half
379 }
380 #else
381 memcpy(data.v, ptr, sizeof(data.v) / 2); // avoid std::copy!!!at least for MSVC
382 memset(data.v + N / 2, 0, sizeof(data.v) / 2); // zero upper half
383 #endif
384 ✗ }
385
386 // Conversions from other types as long as their size matches
387 // Full and half size
388
389 // Full: Widening or narrowing without range checks, e.g., no saturation when converting down.
390 // This one must be used as src=Int32x4::convert_from(src16) and NOT as src32.convert_from(src16).
391 template <typename OtherDerived, typename OtherVecType>
392 AVS_FORCEINLINE static VectorWrapper convert_from(const VectorWrapper<OtherDerived, OtherVecType>& other) {
393 static_assert(vector_traits<VecType>::size == vector_traits<OtherVecType>::size,
394 "Vector sizes must match for convert_from.");
395 #if HAS_VECTOR_ATTRIBUTE
396 VecType result_vec;
397 ✗ result_vec = __builtin_convertvector(other.raw(), VecType);
398 ✗ return VectorWrapper(result_vec);
399 #else
400 VecType result;
401 for (size_t i = 0; i < vector_traits<VecType>::size; ++i) {
402 result[i] = static_cast<typename vector_traits<VecType>::element_type>(other[i]);
403 }
404 return VectorWrapper(result);
405 #endif
406
407 }
408
409 // Static factory: Widening or narrowing the lower half of a larger vector
410 template <typename OtherDerived, typename OtherVecType>
411 AVS_FORCEINLINE static VectorWrapper convert_from_lo(const VectorWrapper<OtherDerived, OtherVecType>& other) {
412 static_assert(vector_traits<VecType>::size == vector_traits<OtherVecType>::size / 2,
413 "Source vector must have twice the elements of target for convert_from_lo.");
414
415 VectorWrapper result;
416 ✗ for (size_t i = 0; i < vector_traits<VecType>::size; ++i) {
417 // Access via operator[] which is public
418 ✗ result.raw()[i] = static_cast<ElementType>(other[i]);
419 }
420 ✗ return result;
421 }
422
423 // Static factory: Widening or narrowing the upper half of a larger vector
424 template <typename OtherDerived, typename OtherVecType>
425 AVS_FORCEINLINE static VectorWrapper convert_from_hi(const VectorWrapper<OtherDerived, OtherVecType>& other) {
426 static_assert(vector_traits<VecType>::size == vector_traits<OtherVecType>::size / 2,
427 "Source vector must have twice the elements of target for convert_from_hi.");
428
429 constexpr auto half_size = vector_traits<OtherVecType>::size / 2;
430 VectorWrapper result;
431 for (size_t i = 0; i < vector_traits<VecType>::size; ++i) {
432 result.raw()[i] = static_cast<ElementType>(other[half_size + i]);
433 }
434 return result;
435 }
436
437 // Fill an integer vector/array from integer pointer source.
438 // Versions for uint8_t, int16_t (short), uint16_t and int32_t
439
440 AVS_FORCEINLINE void load_from_any_intptr(const uint8_t* ptr) {
441 static_assert(
442 std::is_same_v<typename vector_traits<VecType>::element_type, int32_t> ||
443 std::is_same_v<typename vector_traits<VecType>::element_type, int16_t> ||
444 std::is_same_v<typename vector_traits<VecType>::element_type, uint16_t> ||
445 std::is_same_v<typename vector_traits<VecType>::element_type, uint8_t>,
446 "load_from_any_intptr is only valid for int32_t, int16_t, uint16_t or uint8_t target element types."
447 );
448
449 ✗ for (size_t i = 0; i < vector_traits<VecType>::size; ++i) {
450 ✗ data[i] = static_cast<typename vector_traits<VecType>::element_type>(ptr[i]);
451 }
452 ✗ }
453
454 AVS_FORCEINLINE void load_from_any_intptr(const short* ptr) {
455 static_assert(
456 std::is_same_v<typename vector_traits<VecType>::element_type, int32_t> ||
457 std::is_same_v<typename vector_traits<VecType>::element_type, int16_t> ||
458 std::is_same_v<typename vector_traits<VecType>::element_type, uint16_t> ||
459 std::is_same_v<typename vector_traits<VecType>::element_type, uint8_t>,
460 "load_from_any_intptr is only valid for int32_t, int16_t, uint16_t or uint8_t target element types."
461 );
462
463 ✗ for (size_t i = 0; i < vector_traits<VecType>::size; ++i) {
464 ✗ data[i] = static_cast<typename vector_traits<VecType>::element_type>(ptr[i]);
465 }
466 ✗ }
467
468 AVS_FORCEINLINE void load_from_any_intptr(const uint16_t* ptr) {
469 static_assert(
470 std::is_same_v<typename vector_traits<VecType>::element_type, int32_t> ||
471 std::is_same_v<typename vector_traits<VecType>::element_type, int16_t> ||
472 std::is_same_v<typename vector_traits<VecType>::element_type, uint16_t> ||
473 std::is_same_v<typename vector_traits<VecType>::element_type, uint8_t>,
474 "load_from_any_intptr is only valid for int32_t, int16_t, uint16_t or uint8_t target element types."
475 );
476
477 ✗ for (size_t i = 0; i < vector_traits<VecType>::size; ++i) {
478 ✗ data[i] = static_cast<typename vector_traits<VecType>::element_type>(ptr[i]);
479 }
480 ✗ }
481
482 AVS_FORCEINLINE void load_from_any_intptr(const int32_t* ptr) {
483 static_assert(
484 std::is_same_v<typename vector_traits<VecType>::element_type, int32_t> ||
485 std::is_same_v<typename vector_traits<VecType>::element_type, int16_t> ||
486 std::is_same_v<typename vector_traits<VecType>::element_type, uint16_t> ||
487 std::is_same_v<typename vector_traits<VecType>::element_type, uint8_t>,
488 "load_from_any_intptr is only valid for int32_t, int16_t, uint16_t or uint8_t target element types."
489 );
490
491 for (size_t i = 0; i < vector_traits<VecType>::size; ++i) {
492 data[i] = static_cast<typename vector_traits<VecType>::element_type>(ptr[i]);
493 }
494 }
495
496 // Syntax: variable.load(myPointer);
497 // Load from pointer (potentially unaligned)
498 AVS_FORCEINLINE void load(const ElementType* ptr) {
499 #if HAS_VECTOR_ATTRIBUTE
500 ✗ for (size_t i = 0; i < N; ++i) {
501 ✗ data[i] = ptr[i];
502 }
503 #else
504 memcpy(data.v, ptr, sizeof(data.v)); // avoid std::copy!!! At least for MSVC
505 #endif
506 ✗ }
507
508 // Syntax: variable = VecType::from_ptr(myPointer);
509 // Static factory method
510 AVS_FORCEINLINE static VectorWrapper from_ptr(const ElementType* ptr) {
511 VectorWrapper result;
512 #if HAS_VECTOR_ATTRIBUTE
513 result.load(ptr); // Reuse the instance method
514 #else
515 constexpr size_t byte_size = sizeof(result.data);
516
517 // Use _MSC_VER (one underscore) and check for actual size
518 #if defined(_MSC_VER) && !defined(__clang__) && defined(INTEL_INTRINSICS)
519 if constexpr (byte_size == 4) {
520 __m128i tmp = _mm_setzero_si128();
521 memcpy(&tmp, ptr, 4); // Coax MSVC into MOVD
522 memcpy(&result.data, &tmp, 4);
523 }
524 else if constexpr (byte_size == 8) {
525 __m128i tmp = _mm_setzero_si128();
526 memcpy(&tmp, ptr, 8); // Coax MSVC into MOVQ
527 memcpy(&result.data, &tmp, 8);
528 }
529 else if constexpr (byte_size == 16) {
530 __m128i tmp = _mm_loadu_si128(reinterpret_cast<const __m128i*>(ptr));
531 memcpy(&result.data, &tmp, 16);
532 }
533 else if constexpr (byte_size == 32) {
534 #if defined(__AVX__)
535 // Use a single 256-bit load. This stops the XMM-to-stack-back-to-YMM MSVC habit.
536 _mm256_storeu_ps((float*)&result.raw(), _mm256_loadu_ps(ptr));
537 #else
538 __m128i tmp1 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(ptr));
539 __m128i tmp1next16 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(ptr + 4));
540 memcpy(&result.data, &tmp1, 16);
541 memcpy(reinterpret_cast<char*>(&result.data) + 16, &tmp1next16, 16);
542 #endif
543 }
544 else {
545 // Fallback for 32+ byte or unusual sizes
546 memcpy(&result.data, ptr, byte_size);
547 }
548 #else
549 memcpy(&result.data, ptr, byte_size);
550 #endif
551 #endif
552 ✗ return result;
553 }
554
555 // Syntax: variable = VecType::from_ptr_lo(myPointer);
556 // Loads the lower half of the vector and zeros the upper half.
557 AVS_FORCEINLINE static VectorWrapper from_ptr_lo(const ElementType* ptr) {
558 VectorWrapper result;
559 #if HAS_VECTOR_ATTRIBUTE
560 // Standard attribute-based load
561 result.load_lo(ptr);
562 #else
563 constexpr size_t total_bytes = sizeof(result.data);
564
565 #if defined(_MSC_VER) && !defined(__clang__) && defined(INTEL_INTRINSICS)
566 if constexpr (total_bytes == 16) {
567 // Int16x8 or Int32x4 case: Load 8 bytes, zero top 8.
568 // _mm_loadl_epi64 generates MOVQ (XMM load 64-bit).
569 __m128i tmp = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(ptr));
570 memcpy(&result.data, &tmp, 16);
571 }
572 else if constexpr (total_bytes == 32) {
573 #if defined(__AVX__)
574 __m128i tmp = _mm_loadu_si128(reinterpret_cast<const __m128i*>(ptr)); // Load 16 bytes into low half, zero top 16.
575 __m256i tmp256 = _mm256_setzero_si256();
576 tmp256 = _mm256_insertf128_si256(tmp256, tmp, 0);
577 memcpy(&result.data, &tmp256, 32);
578 #else
579 // 256-bit case: Load 16 bytes into low half, zero top 16.
580 __m128i lo = _mm_loadu_si128(reinterpret_cast<const __m128i*>(ptr));
581 memcpy(&result.data, &lo, 16);
582 // clear high 16 bytes
583 memset(reinterpret_cast<char*>(&result.data) + 16, 0, 16);
584 #endif
585 }
586 else {
587 result.load_lo(ptr);
588 }
589 #else
590 result.load_lo(ptr);
591 #endif
592 #endif
593 ✗ return result;
594 }
595
596 // Store to pointer (potentially unaligned)
597 AVS_FORCEINLINE void store(ElementType* ptr) const {
598 #if HAS_VECTOR_ATTRIBUTE
599 ✗ for (size_t i = 0; i < N; ++i) {
600 ✗ ptr[i] = data[i];
601 }
602 #else
603 memcpy(ptr, data.v, sizeof(data.v)); // with Avs_SimdArray; or &data[0] with std::array
604 #endif
605 ✗ }
606
607 // Addition operator (+)
608 AVS_FORCEINLINE VectorWrapper operator+(const VectorWrapper& other) const {
609 #if HAS_VECTOR_ATTRIBUTE
610 ✗ return VectorWrapper(data + other.data);
611 #else
612 return VectorWrapper(VectorOps<VecType>::add(data, other.data));
613 #endif
614 }
615
616 // Subtraction operator (-)
617 AVS_FORCEINLINE VectorWrapper operator-(const VectorWrapper& other) const {
618 #if HAS_VECTOR_ATTRIBUTE
619 return VectorWrapper(data - other.data);
620 #else
621 return VectorWrapper(VectorOps<VecType>::subtract(data, other.data));
622 #endif
623 }
624
625 // Multiplication operator (*)
626 AVS_FORCEINLINE VectorWrapper operator*(const VectorWrapper& other) const {
627 #if HAS_VECTOR_ATTRIBUTE
628 ✗ return VectorWrapper(data * other.data);
629 #else
630 return VectorWrapper(VectorOps<VecType>::multiply(data, other.data));
631 #endif
632 }
633
634 // Scalar multiplication (*val)
635 AVS_FORCEINLINE VectorWrapper operator*(ElementType scalar) const {
636 #if HAS_VECTOR_ATTRIBUTE
637 ✗ return VectorWrapper(data * scalar);
638 #else
639 return VectorWrapper(VectorOps<VecType>::scalar_multiply(data, scalar));
640 #endif
641 }
642
643 // Left shift operator (<<)
644 VectorWrapper operator<<(int shift) const {
645 #if HAS_VECTOR_ATTRIBUTE
646 return VectorWrapper(data << shift);
647 #else
648 return VectorWrapper(VectorOps<VecType>::left_shift(data, shift));
649 #endif
650 }
651
652 // Right shift operator (arithmetic) (>>)
653 AVS_FORCEINLINE VectorWrapper operator>>(int shift) const {
654 #if HAS_VECTOR_ATTRIBUTE
655 return VectorWrapper(data >> shift);
656 #else
657 return VectorWrapper(VectorOps<VecType>::right_shift(data, shift));
658 #endif
659 }
660
661 // We do not define special operators for logical shift, but provide a function.
662 AVS_FORCEINLINE VectorWrapper shift_right_logical(int shift) const {
663 // Cast to unsigned type for logical shift
664 using UnsignedVecType = std::make_unsigned_t<typename vector_traits<VecType>::element_type>;
665 VecType result;
666 for (size_t i = 0; i < vector_traits<VecType>::size; ++i) {
667 result[i] = static_cast<UnsignedVecType>(data[i]) >> shift;
668 }
669 return VectorWrapper(result);
670 }
671
672 // Addition assignment operator (+=)
673 AVS_FORCEINLINE VectorWrapper& operator+=(const VectorWrapper& other) {
674 #if HAS_VECTOR_ATTRIBUTE
675 ✗ data += other.data;
676 #else
677 data = VectorOps<VecType>::add(data, other.data);
678 #endif
679 ✗ return *this;
680 }
681
682 // Subtract assignment operator (-=)
683 AVS_FORCEINLINE VectorWrapper& operator-=(const VectorWrapper& other) {
684 #if HAS_VECTOR_ATTRIBUTE
685 data -= other.data;
686 #else
687 data = VectorOps<VecType>::sub(data, other.data);
688 #endif
689 return *this;
690 }
691
692 // Multiplication assignment operator (*=)
693 AVS_FORCEINLINE VectorWrapper& operator*=(const VectorWrapper& other) {
694 #if HAS_VECTOR_ATTRIBUTE
695 data *= other.data;
696 #else
697 data = VectorOps<VecType>::mul(data, other.data);
698 #endif
699 return *this;
700 }
701
702 // right shift assignment operator (>>=)
703 AVS_FORCEINLINE VectorWrapper& operator>>=(int shift) {
704 #if HAS_VECTOR_ATTRIBUTE
705 ✗ data >>= shift;
706 #else
707 data = VectorOps<VecType>::right_shift(data, shift);
708 #endif
709 ✗ return *this;
710 }
711
712 // left shift assignment operator (<<=)
713 AVS_FORCEINLINE VectorWrapper& operator<<=(int shift) {
714 #if HAS_VECTOR_ATTRIBUTE
715 data <<= shift;
716 #else
717 data = VectorOps<VecType>::left_shift(data, shift);
718 #endif
719 return *this;
720 }
721
722 // Minimum operation: z = x.min(scalar)
723 AVS_FORCEINLINE VectorWrapper min(ElementType scalar) const {
724 #if HAS_VECTOR_ATTRIBUTE
725 VecType result;
726 for (size_t i = 0; i < N; ++i) {
727 result[i] = data[i] < scalar ? data[i] : scalar;
728 }
729 return VectorWrapper(result);
730 #else
731 VecType result;
732 for (size_t i = 0; i < N; ++i) {
733 result[i] = std::min(data[i], scalar);
734 }
735 return VectorWrapper(result);
736 #endif
737 }
738
739 // Maximum operation: z = x.max(scalar)
740 AVS_FORCEINLINE VectorWrapper max(ElementType scalar) const {
741 #if HAS_VECTOR_ATTRIBUTE
742 VecType result;
743 for (size_t i = 0; i < N; ++i) {
744 result[i] = data[i] > scalar ? data[i] : scalar;
745 }
746 return VectorWrapper(result);
747 #else
748 VecType result;
749 for (size_t i = 0; i < N; ++i) {
750 result[i] = std::max(data[i], scalar);
751 }
752 return VectorWrapper(result);
753 #endif
754 }
755
756 // Minimum operation (element-wise): z = x.min(y)
757 AVS_FORCEINLINE VectorWrapper min(const VectorWrapper& other) const {
758 #if HAS_VECTOR_ATTRIBUTE
759 // Use a ternary operator with a comparison to simulate min operation
760 VecType result;
761 for (size_t i = 0; i < N; ++i) {
762 result[i] = data[i] < other.data[i] ? data[i] : other.data[i];
763 }
764 return VectorWrapper(result);
765 #else
766 VecType result;
767 for (size_t i = 0; i < N; ++i) {
768 result[i] = std::min(data[i], other.data[i]);
769 }
770 return VectorWrapper(result);
771 #endif
772 }
773
774 // Maximum operation (element-wise) : z = x.max(y)
775 AVS_FORCEINLINE VectorWrapper max(const VectorWrapper& other) const {
776 #if HAS_VECTOR_ATTRIBUTE
777 // Use a ternary operator with a comparison to simulate max operation
778 VecType result;
779 for (size_t i = 0; i < N; ++i) {
780 result[i] = data[i] > other.data[i] ? data[i] : other.data[i];
781 }
782 return VectorWrapper(result);
783 #else
784 VecType result;
785 for (size_t i = 0; i < N; ++i) {
786 result[i] = std::max(data[i], other.data[i]);
787 }
788 return VectorWrapper(result);
789 #endif
790 }
791
792 // Static min functions similar to SIMD intrinsics: z = min(x, y)
793 AVS_FORCEINLINE static VectorWrapper min(const VectorWrapper& a, const VectorWrapper& b) {
794 return a.min(b);
795 }
796
797 // Static max function similar to SIMD intrinsics: z = max(x, y)
798 AVS_FORCEINLINE static VectorWrapper max(const VectorWrapper& a, const VectorWrapper& b) {
799 return a.max(b);
800 }
801
802 // Accessor for raw data
803 ✗ AVS_FORCEINLINE const VecType& raw() const { return data; }
804 ✗ AVS_FORCEINLINE VecType& raw() { return data; }
805
806 // Element access (const and non-const versions)
807 AVS_FORCEINLINE ElementType operator[](size_t idx) const {
808 ✗ return data[idx];
809 }
810
811 // a set method instead of non-const operator[]
812 // use: x.set(index, value) instead of x[index] = value
813 AVS_FORCEINLINE void set(size_t idx, ElementType value) {
814 ✗ data[idx] = value;
815 ✗ }
816
817 AVS_FORCEINLINE int32_t horiz_add_int32() const {
818 #if defined(_MSC_VER) && !defined(__clang__) && defined(INTEL_INTRINSICS)
819 static_assert(N * sizeof(ElementType) <= 16, "horiz_add_int32: vector too large for XMM");
820 __m128i v = _mm_setzero_si128(); // zero upper lanes for sub-16-byte vectors
821 memcpy(&v, &data, N * sizeof(ElementType));
822
823 // SSE2 only, hadd simulation with shuffles and adds
824 if constexpr (N * sizeof(ElementType) <= 4) {
825 return _mm_cvtsi128_si32(v);
826 }
827 else if constexpr (N * sizeof(ElementType) <= 8) {
828 __m128i hi32 = _mm_shuffle_epi32(v, _MM_SHUFFLE(1, 1, 1, 1));
829 return _mm_cvtsi128_si32(_mm_add_epi32(v, hi32));
830 }
831 else {
832 // 4x int32
833 // v = [D, C, B, A]
834 __m128i hi64 = _mm_unpackhi_epi64(v, v); // [B, A, D, C]
835 v = _mm_add_epi32(v, hi64); // [D+B, C+A, B+D, A+C]
836 __m128i hi32 = _mm_shuffle_epi32(v, _MM_SHUFFLE(1, 1, 1, 1)); // [C+A, C+A, C+A, C+A]
837 v = _mm_add_epi32(v, hi32); // [..., ..., ..., A+C+B+D]
838 return _mm_cvtsi128_si32(v);
839 }
840 #else
841 ✗ int32_t sum = 0;
842 ✗ for (size_t i = 0; i < N; ++i)
843 #if HAS_VECTOR_ATTRIBUTE
844 ✗ sum += data[i];
845 #else
846 sum += data.v[i];
847 #endif
848 ✗ return sum;
849 #endif
850 }
851
852 AVS_FORCEINLINE float horiz_add_float() const {
853 #if defined(_MSC_VER) && !defined(__clang__) && defined(INTEL_INTRINSICS)
854 static_assert(N * sizeof(ElementType) <= 16, "horiz_add_float: vector too large for XMM");
855 __m128 v = _mm_setzero_ps();
856 memcpy(&v, &data, N * sizeof(ElementType));
857 // SSE2 only, not haddps, only quick shuffles and adds
858 if constexpr (N <= 1) {
859 return _mm_cvtss_f32(v);
860 }
861 else if constexpr (N <= 2) {
862 __m128 hi32 = _mm_shuffle_ps(v, v, _MM_SHUFFLE(1, 1, 1, 1));
863 return _mm_cvtss_f32(_mm_add_ss(v, hi32));
864 }
865 else {
866 // v = [D, C, B, A]
867 __m128 hi64 = _mm_movehl_ps(v, v); // [B, A, D, C]
868 v = _mm_add_ps(v, hi64); // [D+B, C+A, B+D, A+C]
869 __m128 hi32 = _mm_shuffle_ps(v, v, _MM_SHUFFLE(1, 1, 1, 1)); // [C+A, C+A, C+A, C+A]
870 v = _mm_add_ss(v, hi32); // [..., ..., ..., A+C+B+D]
871 return _mm_cvtss_f32(v);
872 }
873 #else
874 ✗ float sum = 0.0f;
875 ✗ for (size_t i = 0; i < N; ++i)
876 #if HAS_VECTOR_ATTRIBUTE
877 ✗ sum += static_cast<float>(data[i]);
878 #else
879 sum += static_cast<float>(data.v[i]);
880 #endif
881 ✗ return sum;
882 #endif
883 }
884
885 static constexpr size_t size() { return N; }
886 };
887
888 // Scalar multiplication (scalar on left)
889 template <typename Derived, typename VecType>
890 AVS_FORCEINLINE VectorWrapper<Derived, VecType> operator*(
891 typename vector_traits<VecType>::element_type scalar,
892 const VectorWrapper<Derived, VecType>& vec) {
893 return vec * scalar;
894 }
895
896 // Stream output operator
897 template <typename Derived, typename VecType>
898 std::ostream& operator<<(std::ostream& os, const VectorWrapper<Derived, VecType>& vec) {
899 os << "[";
900 for (size_t i = 0; i < vector_traits<VecType>::size; ++i) {
901 os << vec[i];
902 if (i < vector_traits<VecType>::size - 1) os << ", ";
903 }
904 os << "]";
905 return os;
906 }
907
908 // Concrete class definitions for 8, 16 and 32 bytes vectors
909 class Uint8x32 : public VectorWrapper<Uint8x32, uint8_vec32_t> {
910 public:
911 using Base = VectorWrapper<Uint8x32, uint8_vec32_t>;
912 using Base::Base;
913 using Base::operator=;
914 };
915
916 class Uint8x16 : public VectorWrapper<Uint8x16, uint8_vec16_t> {
917 public:
918 using Base = VectorWrapper<Uint8x16, uint8_vec16_t>;
919 using Base::Base;
920 using Base::operator=;
921 };
922
923 class Uint8x8 : public VectorWrapper<Uint8x8, uint8_vec8_t> {
924 public:
925 using Base = VectorWrapper<Uint8x8, uint8_vec8_t>;
926 ✗ using Base::Base;
927 using Base::operator=;
928 };
929
930 class Uint8x4 : public VectorWrapper<Uint8x4, uint8_vec4_t> {
931 public:
932 using Base = VectorWrapper<Uint8x4, uint8_vec4_t>;
933 using Base::Base;
934 using Base::operator=;
935 };
936
937 class Int16x16 : public VectorWrapper<Int16x16, int16_vec16_t> {
938 public:
939 using Base = VectorWrapper<Int16x16, int16_vec16_t>;
940 using Base::Base;
941 using Base::operator=;
942 };
943
944 class Int16x8 : public VectorWrapper<Int16x8, int16_vec8_t> {
945 public:
946 using Base = VectorWrapper<Int16x8, int16_vec8_t>;
947 ✗ using Base::Base;
948 using Base::operator=;
949 };
950
951 class Int16x4 : public VectorWrapper<Int16x4, int16_vec4_t> {
952 public:
953 using Base = VectorWrapper<Int16x4, int16_vec4_t>;
954 using Base::Base;
955 using Base::operator=;
956 };
957
958 class Uint16x16 : public VectorWrapper<Uint16x16, uint16_vec16_t> {
959 public:
960 using Base = VectorWrapper<Uint16x16, uint16_vec16_t>;
961 using Base::Base;
962 using Base::operator=;
963 };
964
965 class Uint16x8 : public VectorWrapper<Uint16x8, uint16_vec8_t> {
966 public:
967 using Base = VectorWrapper<Uint16x8, uint16_vec8_t>;
968 ✗ using Base::Base;
969 using Base::operator=;
970 };
971
972 class Uint16x4 : public VectorWrapper<Uint16x4, uint16_vec4_t> {
973 public:
974 using Base = VectorWrapper<Uint16x4, uint16_vec4_t>;
975 using Base::Base;
976 using Base::operator=;
977 };
978
979 class Int32x8 : public VectorWrapper<Int32x8, int32_vec8_t> {
980 public:
981 using Base = VectorWrapper<Int32x8, int32_vec8_t>;
982 using Base::Base;
983 using Base::operator=;
984 };
985
986 class Int32x4 : public VectorWrapper<Int32x4, int32_vec4_t> {
987 public:
988 using Base = VectorWrapper<Int32x4, int32_vec4_t>;
989 ✗ using Base::Base;
990 using Base::operator=;
991 };
992
993 class Int32x2 : public VectorWrapper<Int32x2, int32_vec2_t> {
994 public:
995 using Base = VectorWrapper<Int32x2, int32_vec2_t>;
996 using Base::Base;
997 using Base::operator=;
998 };
999
1000 class Float8 : public VectorWrapper<Float8, float_vec8_t> {
1001 public:
1002 using Base = VectorWrapper<Float8, float_vec8_t>;
1003 ✗ using Base::Base;
1004 using Base::operator=;
1005 };
1006
1007 class Float4 : public VectorWrapper<Float4, float_vec4_t> {
1008 public:
1009 using Base = VectorWrapper<Float4, float_vec4_t>;
1010 ✗ using Base::Base;
1011 using Base::operator=;
1012 };
1013
1014 class Float2 : public VectorWrapper<Float2, float_vec2_t> {
1015 public:
1016 using Base = VectorWrapper<Float2, float_vec2_t>;
1017 using Base::Base;
1018 using Base::operator=;
1019 };
1020
1021 // Saturated narrowing integer conversion functions.
1022
1023 // 8x int32_t -> 8x uint8_t
1024 // Guaranteed to stay in XMM/YMM registers
1025 [[maybe_unused]] static AVS_FORCEINLINE void convert_and_saturate_int32x8_to_uint8x8(const Int32x8& src1, Uint8x8& result) {
1026 #if defined(_MSC_VER) && !defined(__clang__) && defined(INTEL_INTRINSICS)
1027 // Assuming Int32x8 is composed of two __m128i (low/high) or one __m256i
1028 // If using 128-bit wrappers:
1029 __m128i src1_lo;
1030 __m128i src1_hi;
1031 memcpy(&src1_lo, &src1, 16); // first 4 ints
1032 memcpy(&src1_hi, reinterpret_cast<const char*>(&src1) + 16, 16); // second 4 ints
1033
1034 // Step 1: Pack 32-bit to 16-bit (Signed Saturation)
1035 __m128i v16 = _mm_packs_epi32(src1_lo, src1_hi);
1036
1037 // Step 2: Pack 16-bit to 8-bit (Unsigned Saturation)
1038 // We use v16 twice because pack instructions always process 128-bits into 128-bits
1039 __m128i v8 = _mm_packus_epi16(v16, v16);
1040
1041 // Step 3: Store only the low 8 bytes (64-bits)
1042 _mm_storel_epi64(reinterpret_cast<__m128i*>(&result), v8);
1043 #else
1044 // Fallback for LLVM (which handles the loop perfectly)
1045 for (size_t i = 0; i < 8; ++i) {
1046 result.set(i, static_cast<uint8_t>(std::min(std::max(src1[i], 0), 255)));
1047 }
1048 #endif
1049 }
1050
1051
1052 // 4x int32_t -> 4x uint8_t
1053 // Prevents the "Extract-to-GPR" penalty seen in your assembly
1054 [[maybe_unused]] static AVS_FORCEINLINE void convert_and_saturate_int32x4_to_uint8x4(const Int32x4& src1, Uint8x4& result) {
1055 #if defined(_MSC_VER) && !defined(__clang__) && defined(INTEL_INTRINSICS)
1056 __m128i a;
1057 memcpy(&a, &src1, 16);
1058
1059 // Step 1: 32-bit signed -> 16-bit signed saturation
1060 // [i32, i32, i32, i32] -> [i16, i16, i16, i16, i16, i16, i16, i16]
1061 // We duplicate 'a' to fill the 128-bit output
1062 __m128i v16 = _mm_packs_epi32(a, a);
1063
1064 // Step 2: 16-bit signed -> 8-bit unsigned saturation
1065 // This clamps everything to [0, 255]
1066 __m128i v8 = _mm_packus_epi16(v16, v16);
1067
1068 // Step 3: Store only the low 32 bits (4 bytes)
1069 // This replaces those 4 'mov byte ptr' instructions with one 'movd'
1070 int packed_pixels = _mm_cvtsi128_si32(v8);
1071 memcpy(&result, &packed_pixels, 4);
1072 #else
1073 // LLVM/Clang handles the loop fine, as you noted
1074 ✗ for (size_t i = 0; i < 4; ++i) {
1075 ✗ result.set(i, static_cast<uint8_t>(std::min(std::max(src1[i], 0), 255)));
1076 }
1077 #endif
1078 ✗ }
1079
1080 // 8x int32_t -> 8x uint16_t
1081 [[maybe_unused]] static AVS_FORCEINLINE void convert_and_saturate_int32x8_to_uint16x8(const Int32x8& src1, Uint16x8& result) {
1082 for (size_t i = 0; i < 8; ++i) {
1083 result.set(i, static_cast<uint16_t>(std::min(std::max(src1[i], 0), 65535)));
1084 }
1085 }
1086
1087
1088 // 4x int32_t -> 4x uint16_t with 0 <= x <= limit
1089 [[maybe_unused]] static AVS_FORCEINLINE void convert_and_saturate_int32x4_to_uint16x4_limit(const Int32x4& src1, Uint16x4& result, const uint16_t limit) {
1090 #if defined(_MSC_VER) && !defined(__clang__) && defined(INTEL_INTRINSICS)
1091 __m128i a;
1092 memcpy(&a, &src1, 16);
1093
1094 #if defined(__AVX__) || defined(__SSE4_1__)
1095 __m128i packed = _mm_packus_epi32(a, a);
1096 __m128i vlimit = _mm_set1_epi16(static_cast<short>(limit));
1097 __m128i clamped = _mm_min_epu16(packed, vlimit);
1098 _mm_storel_epi64(reinterpret_cast<__m128i*>(&result), clamped);
1099 #else
1100 // SSE2 Fallback: Bias trick + manual unsigned min
1101 const __m128i bias = _mm_set1_epi32(32768);
1102 const __m128i bias16 = _mm_set1_epi16(-32768);
1103 __m128i biased = _mm_sub_epi32(a, bias);
1104 __m128i packed = _mm_packs_epi32(biased, biased);
1105 __m128i unbiased = _mm_sub_epi16(packed, bias16);
1106
1107 __m128i vlimit = _mm_set1_epi16(static_cast<short>(limit));
1108 const __m128i sign_bias = _mm_set1_epi16(-32768);
1109 __m128i u_shifted = _mm_add_epi16(unbiased, sign_bias);
1110 __m128i l_shifted = _mm_add_epi16(vlimit, sign_bias);
1111 __m128i mask = _mm_cmpgt_epi16(u_shifted, l_shifted);
1112 __m128i final = _mm_or_si128(_mm_and_si128(mask, vlimit), _mm_andnot_si128(mask, unbiased));
1113 _mm_storel_epi64(reinterpret_cast<__m128i*>(&result), final);
1114 #endif
1115
1116 #else
1117 ✗ for (size_t i = 0; i < 4; ++i) {
1118 ✗ result.set(i, static_cast<uint16_t>(std::min(std::max(src1[i], 0), (int)limit)));
1119 }
1120 #endif
1121 ✗ }
1122
1123
1124 // 4x int32_t -> 4x uint16_t
1125 [[maybe_unused]] static AVS_FORCEINLINE void convert_and_saturate_int32x4_to_uint16x4(const Int32x4& src1, Uint16x4& result) {
1126 #if defined(_MSC_VER) && !defined(__clang__) && defined(INTEL_INTRINSICS)
1127 __m128i a;
1128 memcpy(&a, &src1, 16);
1129
1130 #if defined(__AVX__) || defined(__SSE4_1__)
1131 // Packs 4+4 lanes into 8 lanes of uint16. We use 'a' twice.
1132 __m128i packed = _mm_packus_epi32(a, a);
1133 // Store only the low 64 bits (4x uint16)
1134 _mm_storel_epi64(reinterpret_cast<__m128i*>(&result), packed);
1135 #else
1136 // SSE2 Fallback: The Bias Trick
1137 const __m128i bias = _mm_set1_epi32(32768);
1138 const __m128i bias16 = _mm_set1_epi16(-32768);
1139 __m128i biased = _mm_sub_epi32(a, bias);
1140 __m128i packed = _mm_packs_epi32(biased, biased);
1141 __m128i final = _mm_sub_epi16(packed, bias16);
1142 _mm_storel_epi64(reinterpret_cast<__m128i*>(&result), final);
1143 #endif
1144
1145 #else
1146 ✗ for (size_t i = 0; i < 4; ++i) {
1147 ✗ result.set(i, static_cast<uint16_t>(std::min(std::max(src1[i], 0), 65535)));
1148 }
1149 #endif
1150 ✗ }
1151
1152 // Combined horizontal add of 4 Int32x4 vectors into one Int32x4 of sums
1153 [[maybe_unused]]
1154 AVS_FORCEINLINE Int32x4 make_from_horiz_sums(
1155 const Int32x4& r0, const Int32x4& r1,
1156 const Int32x4& r2, const Int32x4& r3) {
1157 #if defined(_MSC_VER) && !defined(__clang__) && defined(INTEL_INTRINSICS)
1158 __m128i a, b, c, d;
1159 memcpy(&a, &r0, 16); memcpy(&b, &r1, 16);
1160 memcpy(&c, &r2, 16); memcpy(&d, &r3, 16);
1161
1162 // 1. Interleave to pair up elements (SSE2)
1163 __m128i t0 = _mm_unpacklo_epi32(a, b); // [a0, b0, a1, b1]
1164 __m128i t1 = _mm_unpackhi_epi32(a, b); // [a2, b2, a3, b3]
1165 __m128i t2 = _mm_unpacklo_epi32(c, d); // [c0, d0, c1, d1]
1166 __m128i t3 = _mm_unpackhi_epi32(c, d); // [c2, d2, c3, d3]
1167
1168 // 2. First vertical add (Partial sums)
1169 __m128i sum_ab = _mm_add_epi32(t0, t1); // [a0+a2, b0+b2, a1+a3, b1+b3]
1170 __m128i sum_cd = _mm_add_epi32(t2, t3); // [c0+c2, d0+d2, c1+c3, d1+d3]
1171
1172 // 3. Final pairing using 64-bit unpacks (SSE2)
1173 __m128i res_lo = _mm_unpacklo_epi64(sum_ab, sum_cd); // [a0+a2, b0+b2, c0+c2, d0+d2]
1174 __m128i res_hi = _mm_unpackhi_epi64(sum_ab, sum_cd); // [a1+a3, b1+b3, c1+c3, d1+d3]
1175
1176 // 4. Final vertical add
1177 __m128i res = _mm_add_epi32(res_lo, res_hi); // [sumA, sumB, sumC, sumD]
1178
1179 Int32x4 out;
1180 memcpy(&out, &res, 16);
1181 return out;
1182 #else
1183 ✗ Int32x4 out;
1184 ✗ out.set(0, r0.horiz_add_int32());
1185 ✗ out.set(1, r1.horiz_add_int32());
1186 ✗ out.set(2, r2.horiz_add_int32());
1187 ✗ out.set(3, r3.horiz_add_int32());
1188 ✗ return out;
1189 #endif
1190 }
1191
1192 // two 4x int32_t -> 8x uint16_t
1193 // like an _mm_packus_epi32
1194 [[maybe_unused]]
1195 static AVS_FORCEINLINE void convert_and_saturate_int32x4x2_to_uint16x8(const Int32x4& src1, const Int32x4& src2, Uint16x8& result) {
1196 #if defined(_MSC_VER) && !defined(__clang__) && defined(INTEL_INTRINSICS)
1197 __m128i a, b;
1198 memcpy(&a, &src1, 16);
1199 memcpy(&b, &src2, 16);
1200 // Though MSVC does not have sse4.1 arch granularity
1201 #if defined(__AVX__) || defined(__SSE4_1__)
1202 // SSE4.1+: direct unsigned pack
1203 __m128i packed = _mm_packus_epi32(a, b);
1204 memcpy(&result, &packed, 16);
1205 #else
1206 // SSE2 fallback: bias trick
1207 // _mm_packs_epi32 does signed saturation, so bias into signed range first
1208 // subtract 32768, pack as signed, then the result is offset by 32768
1209 const __m128i bias = _mm_set1_epi32(32768);
1210 const __m128i bias16 = _mm_set1_epi16(-32768); // same bit pattern, adds back
1211 __m128i a_biased = _mm_sub_epi32(a, bias);
1212 __m128i b_biased = _mm_sub_epi32(b, bias);
1213 __m128i packed = _mm_packs_epi32(a_biased, b_biased); // signed sat to int16
1214 __m128i unbiased = _mm_sub_epi16(packed, bias16); // restore: wraps correctly
1215 memcpy(&result, &unbiased, 16);
1216 #endif
1217 #else
1218 ✗ for (size_t i = 0; i < 4; ++i) {
1219 ✗ result.set(i, static_cast<uint16_t>(std::min(std::max(src1[i], 0), 65535)));
1220 ✗ result.set(i + 4, static_cast<uint16_t>(std::min(std::max(src2[i], 0), 65535)));
1221 }
1222 #endif
1223 ✗ }
1224
1225 // two 4x int32_t -> 8x uint16_t
1226 // like a _mm_packus_epi32 followed by a max(x,limit)
1227 [[maybe_unused]]
1228 static AVS_FORCEINLINE void convert_and_saturate_int32x4x2_to_uint16x8_limit(const Int32x4& src1, const Int32x4& src2, Uint16x8& result, const uint16_t limit) {
1229 #if defined(_MSC_VER) && !defined(__clang__) && defined(INTEL_INTRINSICS)
1230 __m128i a, b;
1231 memcpy(&a, &src1, 16);
1232 memcpy(&b, &src2, 16);
1233 #if defined(__AVX__) || defined(__SSE4_1__)
1234 __m128i packed = _mm_packus_epi32(a, b);
1235 __m128i vlimit = _mm_set1_epi16(static_cast<short>(limit));
1236 __m128i clamped = _mm_min_epu16(packed, vlimit);
1237 memcpy(&result, &clamped, 16);
1238 #else
1239 // SSE2: same bias trick for pack, then scalar min (limit path is rare/short)
1240 const __m128i bias = _mm_set1_epi32(32768);
1241 const __m128i bias16 = _mm_set1_epi16(-32768);
1242 __m128i a_biased = _mm_sub_epi32(a, bias);
1243 __m128i b_biased = _mm_sub_epi32(b, bias);
1244 __m128i packed = _mm_packs_epi32(a_biased, b_biased);
1245 __m128i unbiased = _mm_sub_epi16(packed, bias16);
1246 // SSE2 has no _mm_min_epu16, emulate with signed comparison bias
1247 __m128i vlimit = _mm_set1_epi16(static_cast<short>(limit));
1248 // unsigned min: a < b unsigned <=> (a - 32768) < (b - 32768) signed
1249 const __m128i sign_bias = _mm_set1_epi16(-32768);
1250 __m128i u_shifted = _mm_add_epi16(unbiased, sign_bias);
1251 __m128i l_shifted = _mm_add_epi16(vlimit, sign_bias);
1252 __m128i mask = _mm_cmpgt_epi16(u_shifted, l_shifted); // 0xFFFF where unbiased > limit
1253 __m128i clamped = _mm_or_si128(_mm_and_si128(mask, vlimit),
1254 _mm_andnot_si128(mask, unbiased));
1255 memcpy(&result, &clamped, 16);
1256 #endif
1257 #else
1258 ✗ for (size_t i = 0; i < 4; ++i) {
1259 ✗ result.set(i, static_cast<uint16_t>(std::min(std::max(src1[i], 0), (int)limit)));
1260 ✗ result.set(i + 4, static_cast<uint16_t>(std::min(std::max(src2[i], 0), (int)limit)));
1261 }
1262 #endif
1263 ✗ }
1264
1265 // two 4x int32_t -> 8x uint8_t
1266 // like an _mm_packs_epi32 followed by _mm_packus_epi16
1267 [[maybe_unused]]
1268 static AVS_FORCEINLINE void convert_and_saturate_int32x4x2_to_uint8x8(const Int32x4& src1, const Int32x4& src2, Uint8x8& result) {
1269 #if defined(_MSC_VER) && !defined(__clang__) && defined(INTEL_INTRINSICS)
1270 __m128i a, b;
1271 memcpy(&a, &src1, 16);
1272 memcpy(&b, &src2, 16);
1273 __m128i packed16 = _mm_packs_epi32(a, b); // [s1_0..s1_3, s2_0..s2_3] as int16 with signed sat
1274 __m128i packed8 = _mm_packus_epi16(packed16, packed16); // uint8 with unsigned sat, result in low 8 bytes
1275 memcpy(&result, &packed8, 8); // movq equivalent
1276 #else
1277 ✗ for (size_t i = 0; i < 4; ++i) {
1278 ✗ result.set(i, static_cast<uint8_t>(std::min(std::max(src1[i], 0), 255)));
1279 ✗ result.set(i + 4, static_cast<uint8_t>(std::min(std::max(src2[i], 0), 255)));
1280 }
1281 // This one is optimized with LLVM, but not with gcc or MSVC
1282 /* bravo LLVM x86-64 icx 2025.1.0, using return Int32x4
1283
1284 movdqu xmm0, xmmword ptr [rdi]
1285 packssdw xmm0, xmm0
1286 packuswb xmm0, xmm0
1287 movdqu xmm1, xmmword ptr [rsi + 16]
1288 packssdw xmm1, xmm1
1289 packuswb xmm1, xmm1
1290 movd eax, xmm0
1291 movd ecx, xmm1
1292 shl rcx, 32
1293 or rax, rcx
1294 ret
1295 */
1296 #endif
1297 ✗ }
1298
1299 // int16 * int16->int32 with partial result addition.
1300 // Simulates the _mm_madd_epi16 x86 SIMD intrinsic function.
1301 // returns [a0*b0 + a1*b1, a2*b2 + a3*b3, a4*b4 + a5*b5, a6*b6 + a7*b7]
1302 // When the pre-addition order is not important use the reduce_add version,
1303 // which may be easier to optimize on non-x86 architectures.
1304 // (only LLVM was able to optimize the x86 version and compile into real madd)
1305 [[maybe_unused]]
1306 static AVS_FORCEINLINE Int32x4 simul_madd_epi16(const Int16x8& a, const Int16x8& b) {
1307 Int32x4 result;
1308
1309 // pairwise multiplication and horizontal addition
1310 // Yes, I know c++ integer promotion rules, but maybe the compiler
1311 // will see our forced static_casting the already short type.
1312 // Otherwise, yes, it makes no sense.
1313 for (size_t i = 0; i < 4; ++i) {
1314 size_t idx = i * 2;
1315 int32_t sum =
1316 static_cast<short>(a[idx]) * static_cast<short>(b[idx]) +
1317 static_cast<short>(a[idx + 1]) * static_cast<short>(b[idx + 1]);
1318
1319 result.set(i, sum);
1320 }
1321 return result;
1322 }
1323
1324
1325 // int16 * int16 -> int32 with partial result addition.
1326 // Unlike the classic x86 SIMD madd_epi16, the sum of multiplications
1327 // is not horizontally adjacent, instead, the lanes are added together.
1328 // returns [a0*b0 + a4*b4, a1*b1 + a5*b5, a2*b2 + a6*b6, a3*b3 + a7*b7]
1329 // However, since the order is not always important, this approach is
1330 // beneficial for non-x64 processor architectures when the compiler
1331 // recognizes the widening multiplication pattern.
1332 [[maybe_unused]]
1333 static AVS_FORCEINLINE Int32x4 mul16x16_reduce_to_Int32x4(const Int16x8 &a, const Int16x8 &b) {
1334 ✗ Int32x4 result_lo, result_hi;
1335
1336 ✗ for (size_t i = 0; i < 4; ++i) {
1337 ✗ result_lo.set(i, static_cast<short>(a[i]) * static_cast<short>(b[i]));
1338 }
1339 ✗ for (size_t i = 0; i < 4; ++i) {
1340 ✗ result_hi.set(i, static_cast<short>(a[i + 4]) * static_cast<short>(b[i + 4]));
1341 }
1342 ✗ Int32x4 result = result_lo + result_hi;
1343 ✗ return result;
1344 }
1345
1346 [[maybe_unused]]
1347 static AVS_FORCEINLINE Int32x4 mul16x16_to_Int32x4(const Int16x4& a, const Int16x4& b) {
1348 Int32x4 result;
1349
1350 for (size_t i = 0; i < 4; ++i) {
1351 result.set(i, static_cast<short>(a[i]) * static_cast<short>(b[i]));
1352 }
1353 return result;
1354 }
1355
1356 #endif // __AVS_SIMD_C_H__
1357