GCC Code Coverage Report

Directory: avs_core/
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 0.0% 0 / 0 / 2703
Functions: 0.0% 0 / 0 / 78
Branches: 0.0% 0 / 0 / 1221
filters/intel/resample_avx512b.cpp
Line Branch Exec Source
1 // Avisynth v2.5. Copyright 2002 Ben Rudiak-Gould et al.
2 // http://avisynth.nl
3
4 // This program is free software; you can redistribute it and/or modify
5 // it under the terms of the GNU General Public License as published by
6 // the Free Software Foundation; either version 2 of the License, or
7 // (at your option) any later version.
8 //
9 // This program is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 // GNU General Public License for more details.
13 //
14 // You should have received a copy of the GNU General Public License
15 // along with this program; if not, write to the Free Software
16 // Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA, or visit
17 // http://www.gnu.org/copyleft/gpl.html .
18 //
19 // Linking Avisynth statically or dynamically with other modules is making a
20 // combined work based on Avisynth. Thus, the terms and conditions of the GNU
21 // General Public License cover the whole combination.
22 //
23 // As a special exception, the copyright holders of Avisynth give you
24 // permission to link Avisynth with independent modules that communicate with
25 // Avisynth solely through the interfaces defined in avisynth.h, regardless of the license
26 // terms of these independent modules, and to copy and distribute the
27 // resulting combined work under terms of your choice, provided that
28 // every copy of the combined work is accompanied by a complete copy of
29 // the source code of Avisynth (the version of Avisynth used to produce the
30 // combined work), being distributed under the terms of the GNU General
31 // Public License plus this exception. An independent module is a module
32 // which is not derived from or based on Avisynth, such as 3rd-party filters,
33 // import and export plugins, or graphical user interfaces.
34
35 #include <avs/config.h>
36 #include "../core/internal.h"
37
38 #include <avs/alignment.h>
39 #include <avs/minmax.h>
40
41 #include "check_avx512.h" // compiler avx512 directives check
42 #include "resample_avx512.h"
43 #include <type_traits>
44
45 #include <immintrin.h> // Includes AVX-512 intrinsics
46
47 #include "resample_avx512.hpp"
48
49 // FIXME!!! Use constexpr, compiler BUG in v143/v145! if (!lessthan16bit) generates bad code.
50 // https://developercommunity.visualstudio.com/t/Silent-Bad-CodeGen:-Regression-in-Lambda/11030256
51 // Fixed In: 18.4.0
52
53 /**
54 * Simulates _mm256_dpwssd_epi32 for CPUs without AVX512_VNNI (e.g., Xeon 613x).
55 * Logic: For each 32-bit lane, it treats the inputs as pairs of 16-bit signed ints,
56 * multiplies the pairs, adds them together, and adds the result to the accumulator.
57 */
58 static AVS_FORCEINLINE __m256i _MM256_DPWSSD_EPI32_SIMUL(__m256i acc, __m256i a, __m256i b) {
59 #if defined(__AVX512VNNI__) || defined(__AVX_VNNI__)
60 return _mm256_dpwssd_epi32(acc, a, b);
61 #else
62 // vpmaddwd: (a_even * b_even) + (a_odd * b_odd) for each 32-bit slot
63 ✗ __m256i product = _mm256_madd_epi16(a, b);
64 // vpaddd: add the products to the existing accumulator
65 ✗ return _mm256_add_epi32(acc, product);
66 #endif
67 }
68
69 /**
70 * SIMD Optimization Strategy for Horizontal Resampling Kernel (Filter) Execution.
71 *
72 * This section details the performance optimization strategy based on the filter's kernel size,
73 * ensuring high throughput across common resampling scenarios (upscaling/downscaling).
74 *
75 * I. Typical Kernel Sizes (Taps):
76 * ----------------------------------------------------------------------------------------------------------------
77 * Small (Best Case/Upscale): Taps = 4 (Bilinear/Spline36), 8 (Spline64/Lanczos 4).
78 * Medium (Mild Downscale): Taps = 12 (Lanczos 3, 2x downscale).
79 * Large (Worst Case/Downscale): Taps = 16 to 32 (Spline64/Lanczos 4, 2x to 4x downscale).
80 * ----------------------------------------------------------------------------------------------------------------
81 *
82 * II. Optimization Tiers (Template Specialization):
83 *
84 * 1. Dedicated, Fully Unrolled Paths (FixedFilterSize = 4, 8, 12):
85 * - Purpose: Eliminate all loop overhead (setup/teardown, bounds checks).
86 * - SIMD Choice: Uses __m128i/__m256i for efficiency with small loads, accumulating into __m512i.
87 * - Performance Gain: Estimated 1.5x to 2.5x speedup over generic loops for small, common kernels.
88 * - Micro-arch Benefit (i7-11700): Reduces uOps (instruction count) and avoids unnecessary register pressure.
89 *
90 * 2. Aligned VNNI Paths (FixedFilterSize = 16, 32, 48, 64...):
91 * - Purpose: Maximize vector utilization for worst-case downscaling.
92 * - SIMD Choice: Uses __m512i (AVX-512) processing 32 taps per iteration.
93 * - VNNI Advantage: Uses _mm512_dpwssd_epi32 for fused 16-bit multiply + 32-bit accumulation.
94 *
95 * 3. Generic Path (FixedFilterSize = -1):
96 * - Purpose: Handles all remaining unaligned or uncommon kernel sizes. Slower, but safe fallback.
97 */
98
99 // Helper to reduce a ZMM (16x int32) to a scalar int32 sum
100 // -----------------------------------------------------------------------------------------
101 // Helper: Reduce ZMM (16x int32) to scalar int32
102 // -----------------------------------------------------------------------------------------
103 AVS_FORCEINLINE static int32_t _mm512_reduce_add_epi32_compat(__m512i v) {
104 /*
105 __m256i v256 = _mm256_add_epi32(_mm512_extracti64x4_epi64(v, 0), _mm512_extracti64x4_epi64(v, 1));
106 __m128i v128 = _mm_add_epi32(_mm256_castsi256_si128(v256), _mm256_extracti128_si256(v256, 1));
107 v128 = _mm_add_epi32(v128, _mm_shuffle_epi32(v128, _MM_SHUFFLE(1, 0, 3, 2)));
108 v128 = _mm_add_epi32(v128, _mm_shuffle_epi32(v128, _MM_SHUFFLE(0, 3, 0, 1)));
109 return _mm_cvtsi128_si32(v128);
110 */
111 return _mm512_reduce_add_epi32(v);
112 }
113
114 // These are the direct rewrite of the full-generic AVX2 h resamplers
115 // - resizer_h_avx512_generic_uint8_t and
116 // - resizer_h_avx512_generic_uint16_t<bool lessthan16bit>
117 // They are not any quicker than the AVX2 versions, but they serve as a base for further optimizations.
118 // The 512-bitness is not exploited, only have more registers, but they did not help. WIP.
119
120 // -----------------------------------------------------------------------------------------
121 // Core Processing: 4x16, 2x16 Taps
122 // -----------------------------------------------------------------------------------------
123 // taps16, 4 coeffs
124 template<typename pixel_t, bool lessthan16bit>
125 AVS_FORCEINLINE static void process_two_16pixels_core(const pixel_t * AVS_RESTRICT src, int begin1, int begin2, int i, const short* AVS_RESTRICT current_coeff, int filter_size, __m256i & result1, __m256i & result2, __m256i & shifttosigned) {
126 __m256i data_1, data_2;
127
128 if constexpr (sizeof(pixel_t) == 1) {
129 data_1 = _mm256_cvtepu8_epi16(_mm_loadu_si128(reinterpret_cast<const __m128i*>(src + begin1 + i)));
130 data_2 = _mm256_cvtepu8_epi16(_mm_loadu_si128(reinterpret_cast<const __m128i*>(src + begin2 + i)));
131 }
132 else {
133 data_1 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src + begin1 + i));
134 data_2 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src + begin2 + i));
135 if constexpr (!lessthan16bit) {
136 data_1 = _mm256_add_epi16(data_1, shifttosigned);
137 data_2 = _mm256_add_epi16(data_2, shifttosigned);
138 }
139 }
140
141 // Aligned load is not OK for coeffs if filter size is only 4
142 // Coeffs are aligned to 4 or 8 shorts, so alignment is 8 or 16 bytes, _m256i requires 32 bytes alignment.
143
144 // assume 16 alignment
145
146 __m256i coeff_1 = _mm256_load_si256(reinterpret_cast<const __m256i*>(current_coeff));
147 __m256i coeff_2 = _mm256_load_si256(reinterpret_cast<const __m256i*>(current_coeff + filter_size));
148
149 result1 = _MM256_DPWSSD_EPI32_SIMUL(result1, data_1, coeff_1); // vnni, not really a bottleneck here
150 result2 = _MM256_DPWSSD_EPI32_SIMUL(result2, data_2, coeff_2);
151 }
152
153 // taps16, 4 coeffs
154 template<typename pixel_t, bool lessthan16bit>
155 AVS_FORCEINLINE static void process_four_16pixels_core(const pixel_t* AVS_RESTRICT src,
156 int begin1, int begin2, int begin3, int begin4, int i, const short* AVS_RESTRICT current_coeff, int filter_size,
157 __m256i& result1, __m256i& result2, __m256i& result3, __m256i& result4, __m256i& shifttosigned) {
158 __m256i data_1, data_2, data_3, data_4;
159
160 if constexpr (sizeof(pixel_t) == 1) {
161 ✗ data_1 = _mm256_cvtepu8_epi16(_mm_loadu_si128(reinterpret_cast<const __m128i*>(src + begin1 + i)));
162 ✗ data_2 = _mm256_cvtepu8_epi16(_mm_loadu_si128(reinterpret_cast<const __m128i*>(src + begin2 + i)));
163 ✗ data_3 = _mm256_cvtepu8_epi16(_mm_loadu_si128(reinterpret_cast<const __m128i*>(src + begin3 + i)));
164 ✗ data_4 = _mm256_cvtepu8_epi16(_mm_loadu_si128(reinterpret_cast<const __m128i*>(src + begin4 + i)));
165 }
166 else {
167 ✗ data_1 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src + begin1 + i));
168 ✗ data_2 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src + begin2 + i));
169 ✗ data_3 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src + begin3 + i));
170 ✗ data_4 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src + begin4 + i));
171 if constexpr (!lessthan16bit) {
172 ✗ data_1 = _mm256_add_epi16(data_1, shifttosigned);
173 ✗ data_2 = _mm256_add_epi16(data_2, shifttosigned);
174 ✗ data_3 = _mm256_add_epi16(data_3, shifttosigned);
175 ✗ data_4 = _mm256_add_epi16(data_4, shifttosigned);
176 }
177 }
178
179 // Aligned load is not OK for coeffs if filter size is only 4
180 // Coeffs are aligned to 4 or 8 shorts, so alignment is 8 or 16 bytes, _m256i requires 32 bytes alignment.
181
182 // assume 16 alignment
183
184 ✗ __m256i coeff_1 = _mm256_load_si256(reinterpret_cast<const __m256i*>(current_coeff));
185 ✗ __m256i coeff_2 = _mm256_load_si256(reinterpret_cast<const __m256i*>(current_coeff + filter_size));
186 ✗ __m256i coeff_3 = _mm256_load_si256(reinterpret_cast<const __m256i*>(current_coeff + 2 * filter_size));
187 ✗ __m256i coeff_4 = _mm256_load_si256(reinterpret_cast<const __m256i*>(current_coeff + 3 * filter_size));
188
189 ✗ result1 = _MM256_DPWSSD_EPI32_SIMUL(result1, data_1, coeff_1);
190 ✗ result2 = _MM256_DPWSSD_EPI32_SIMUL(result2, data_2, coeff_2);
191 ✗ result3 = _MM256_DPWSSD_EPI32_SIMUL(result3, data_3, coeff_3);
192 ✗ result4 = _MM256_DPWSSD_EPI32_SIMUL(result4, data_4, coeff_4);
193 ✗ }
194
195 // -----------------------------------------------------------------------------------------
196 // Helper: Unrolled Partial Core (4 or 8 Taps) XMM would be enough
197 // -----------------------------------------------------------------------------------------
198 // filter_size must be the aligned size, better named as filter_coeff_stride
199 template<typename pixel_t, bool lessthan16bit, int Taps>
200 AVS_FORCEINLINE static void process_two_partial_unrolled(const pixel_t* src, int begin1, int begin2, int offset, const short* coeff, int filter_size, __m256i& result1, __m256i& result2, __m256i& shifttosigned) {
201 // Taps is 4 or 8.
202 __m128i d1, d2;
203
204 // Load Data
205 if constexpr (sizeof(pixel_t) == 1) {
206 if constexpr (Taps == 4) {
207 d1 = _mm_cvtepu8_epi16(_mm_cvtsi32_si128(*reinterpret_cast<const int*>(src + begin1 + offset)));
208 d2 = _mm_cvtepu8_epi16(_mm_cvtsi32_si128(*reinterpret_cast<const int*>(src + begin2 + offset)));
209 }
210 else { // 8
211 d1 = _mm_cvtepu8_epi16(_mm_loadl_epi64(reinterpret_cast<const __m128i*>(src + begin1 + offset)));
212 d2 = _mm_cvtepu8_epi16(_mm_loadl_epi64(reinterpret_cast<const __m128i*>(src + begin2 + offset)));
213 }
214 }
215 else {
216 if constexpr (Taps == 4) {
217 d1 = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(src + begin1 + offset));
218 d2 = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(src + begin2 + offset));
219 }
220 else { // 8
221 d1 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src + begin1 + offset));
222 d2 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src + begin2 + offset));
223 }
224 if constexpr (!lessthan16bit) {
225 d1 = _mm_add_epi16(d1, _mm256_castsi256_si128(shifttosigned));
226 d2 = _mm_add_epi16(d2, _mm256_castsi256_si128(shifttosigned));
227 }
228 }
229
230 // Load Coeffs (Need to handle offset)
231 __m128i c1, c2;
232 if constexpr (Taps == 4) {
233 c1 = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(coeff + offset));
234 c2 = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(coeff + filter_size + offset));
235 }
236 else {
237 c1 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(coeff + offset));
238 c2 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(coeff + filter_size + offset));
239 }
240
241 // Calc
242 /*
243 result1 = _mm512_zextsi128_si512(_mm_dpwssd_epi32(_mm512_castsi512_si128(result1), d1, c1));
244 result2 = _mm512_zextsi128_si512(_mm_dpwssd_epi32(_mm512_castsi512_si128(result2), d2, c2));
245 */
246 __m256i c1_256 = _mm256_zextsi128_si256(c1);
247 __m256i c2_256 = _mm256_zextsi128_si256(c2);
248 __m256i d1_256 = _mm256_zextsi128_si256(d1);
249 __m256i d2_256 = _mm256_zextsi128_si256(d2);
250
251 result1 = _MM256_DPWSSD_EPI32_SIMUL(result1, d1_256, c1_256);
252 result2 = _MM256_DPWSSD_EPI32_SIMUL(result2, d2_256, c2_256);
253
254 }
255
256 // -----------------------------------------------------------------------------------------
257 // Helper: Unrolled Partial Core (4 or 8 Taps) XMM would be enough
258 // -----------------------------------------------------------------------------------------
259 // filter_size must be the aligned size, better named as filter_coeff_stride
260 template<typename pixel_t, bool lessthan16bit, int Taps>
261 AVS_FORCEINLINE static void process_four_partial_unrolled(const pixel_t* src,
262 int begin1, int begin2, int begin3, int begin4, int offset,
263 const short* coeff, int filter_size,
264 __m256i& result1, __m256i& result2, __m256i& result3, __m256i& result4, __m256i& shifttosigned) {
265 // Taps is 4 or 8.
266 __m128i d1, d2, d3, d4;
267
268 // Load Data
269 if constexpr (sizeof(pixel_t) == 1) {
270 if constexpr (Taps == 4) {
271 ✗ d1 = _mm_cvtepu8_epi16(_mm_cvtsi32_si128(*reinterpret_cast<const int*>(src + begin1 + offset)));
272 ✗ d2 = _mm_cvtepu8_epi16(_mm_cvtsi32_si128(*reinterpret_cast<const int*>(src + begin2 + offset)));
273 ✗ d3 = _mm_cvtepu8_epi16(_mm_cvtsi32_si128(*reinterpret_cast<const int*>(src + begin3 + offset)));
274 ✗ d4 = _mm_cvtepu8_epi16(_mm_cvtsi32_si128(*reinterpret_cast<const int*>(src + begin4 + offset)));
275 }
276 else { // 8
277 ✗ d1 = _mm_cvtepu8_epi16(_mm_loadl_epi64(reinterpret_cast<const __m128i*>(src + begin1 + offset)));
278 ✗ d2 = _mm_cvtepu8_epi16(_mm_loadl_epi64(reinterpret_cast<const __m128i*>(src + begin2 + offset)));
279 ✗ d3 = _mm_cvtepu8_epi16(_mm_loadl_epi64(reinterpret_cast<const __m128i*>(src + begin3 + offset)));
280 ✗ d4 = _mm_cvtepu8_epi16(_mm_loadl_epi64(reinterpret_cast<const __m128i*>(src + begin4 + offset)));
281 }
282 }
283 else {
284 if constexpr (Taps == 4) {
285 ✗ d1 = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(src + begin1 + offset));
286 ✗ d2 = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(src + begin2 + offset));
287 ✗ d3 = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(src + begin3 + offset));
288 ✗ d4 = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(src + begin4 + offset));
289 }
290 else { // 8
291 ✗ d1 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src + begin1 + offset));
292 ✗ d2 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src + begin2 + offset));
293 ✗ d3 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src + begin3 + offset));
294 ✗ d4 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src + begin4 + offset));
295 }
296 if constexpr (!lessthan16bit) {
297 ✗ d1 = _mm_add_epi16(d1, _mm256_castsi256_si128(shifttosigned));
298 ✗ d2 = _mm_add_epi16(d2, _mm256_castsi256_si128(shifttosigned));
299 ✗ d3 = _mm_add_epi16(d3, _mm256_castsi256_si128(shifttosigned));
300 ✗ d4 = _mm_add_epi16(d4, _mm256_castsi256_si128(shifttosigned));
301 }
302 }
303
304 // Load Coeffs (Need to handle offset)
305 __m128i c1, c2, c3, c4;
306 if constexpr (Taps == 4) {
307 ✗ c1 = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(coeff + offset));
308 ✗ c2 = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(coeff + filter_size + offset));
309 ✗ c3 = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(coeff + 2 * filter_size + offset));
310 ✗ c4 = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(coeff + 3 * filter_size + offset));
311 }
312 else {
313 ✗ c1 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(coeff + offset));
314 ✗ c2 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(coeff + filter_size + offset));
315 ✗ c3 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(coeff + 2 * filter_size + offset));
316 ✗ c4 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(coeff + 3 * filter_size + offset));
317 }
318
319 // Calc
320 /*
321 result1 = _mm512_zextsi128_si512(_mm_dpwssd_epi32(_mm512_castsi512_si128(result1), d1, c1));
322 result2 = _mm512_zextsi128_si512(_mm_dpwssd_epi32(_mm512_castsi512_si128(result2), d2, c2));
323 */
324 ✗ __m256i c1_256 = _mm256_zextsi128_si256(c1);
325 ✗ __m256i c2_256 = _mm256_zextsi128_si256(c2);
326 ✗ __m256i c3_256 = _mm256_zextsi128_si256(c3);
327 ✗ __m256i c4_256 = _mm256_zextsi128_si256(c4);
328 ✗ __m256i d1_256 = _mm256_zextsi128_si256(d1);
329 ✗ __m256i d2_256 = _mm256_zextsi128_si256(d2);
330 ✗ __m256i d3_256 = _mm256_zextsi128_si256(d3);
331 ✗ __m256i d4_256 = _mm256_zextsi128_si256(d4);
332
333 ✗ result1 = _MM256_DPWSSD_EPI32_SIMUL(result1, d1_256, c1_256);
334 ✗ result2 = _MM256_DPWSSD_EPI32_SIMUL(result2, d2_256, c2_256);
335 ✗ result3 = _MM256_DPWSSD_EPI32_SIMUL(result3, d3_256, c3_256);
336 ✗ result4 = _MM256_DPWSSD_EPI32_SIMUL(result4, d4_256, c4_256);
337 ✗ }
338
339
340 // ---------------------------------------------------------------------------
341 // FULLY VECTORIZED TREE REDUCTION (AVX2/AVX-512VL)
342 // Input: 8x __m256i accumulators (r0 through r7)
343 // Output: __m256i with 8 final, rounded pixel sums (p0 through p7)
344 // ---------------------------------------------------------------------------
345
346 AVS_FORCEINLINE static __m256i reduce_8x256i_32i_tree(
347 __m256i r0, __m256i r1, __m256i r2, __m256i r3,
348 __m256i r4, __m256i r5, __m256i r6, __m256i r7,
349 int rounder_scalar)
350 {
351 // --- Round 1: Reduce pairs (8 elements -> 4 elements per vector) ---
352 // VPHADDD on each pair. The results are still interleaved within the 128-bit blocks.
353 ✗ __m256i sum01 = _mm256_hadd_epi32(r0, r1);
354 ✗ __m256i sum23 = _mm256_hadd_epi32(r2, r3);
355 ✗ __m256i sum45 = _mm256_hadd_epi32(r4, r5);
356 ✗ __m256i sum67 = _mm256_hadd_epi32(r6, r7);
357
358 // --- Round 2: Reduce quads (4 elements -> 2 elements per vector) ---
359 // The final pixel sum is now split between the low 128-bit half and the high 128-bit half.
360 ✗ __m256i sum0123 = _mm256_hadd_epi32(sum01, sum23);
361 ✗ __m256i sum4567 = _mm256_hadd_epi32(sum45, sum67);
362
363 // --- Round 3: Final Horizontal Reduction (Across 128-bit boundary) ---
364
365 // 1. Add the low 128-bit half to the high 128-bit half to finalize the sum for P0-P7.
366 // VPERM2I128 (0x01 swaps the 128-bit halves)
367 ✗ __m256i hi_add0123 = _mm256_permute2f128_si256(sum0123, sum0123, 0x01);
368 ✗ __m256i hi_add4567 = _mm256_permute2f128_si256(sum4567, sum4567, 0x01);
369
370 // The final sums for P0-P3 are now consolidated into the low 4 lanes of final0123.
371 ✗ __m256i final0123 = _mm256_add_epi32(sum0123, hi_add0123);
372 ✗ __m256i final4567 = _mm256_add_epi32(sum4567, hi_add4567);
373
374 // --- Round 4: Final Consolidation (P0-P7 into one __m256i) ---
375
376 // 1. Extract the low 128 bits (which contain the P0-P3 sums)
377 ✗ __m128i p0_p3 = _mm256_castsi256_si128(final0123);
378 ✗ __m128i p4_p7 = _mm256_castsi256_si128(final4567);
379
380 // 2. Assemble the two 128-bit blocks into the final 256-bit result (VINSERTI128).
381 ✗ __m256i result_8x_32 = _mm256_inserti128_si256(_mm256_castsi128_si256(p0_p3), p4_p7, 1);
382
383 // --- Final Vectorized Rounding ---
384
385 // Create the rounder vector
386 ✗ __m256i rounder_v = _mm256_set1_epi32(rounder_scalar);
387
388 // Apply rounding to all 8 pixels simultaneously (VPADDD)
389 ✗ return _mm256_add_epi32(result_8x_32, rounder_v);
390 }
391
392
393 AVS_FORCEINLINE static __m256i reduce_8x128i_to_8x32i(
394 __m128i r0, __m128i r1, __m128i r2, __m128i r3,
395 __m128i r4, __m128i r5, __m128i r6, __m128i r7,
396 int rounder_scalar)
397 {
398 // --- Round 1: Reduce pairs (8 elements -> 4 elements per vector) ---
399 // VPHADDD on each pair. The results are still interleaved within the 128-bit blocks.
400 __m128i sum01 = _mm_hadd_epi32(r0, r1);
401 __m128i sum23 = _mm_hadd_epi32(r2, r3);
402 __m128i sum45 = _mm_hadd_epi32(r4, r5);
403 __m128i sum67 = _mm_hadd_epi32(r6, r7);
404
405 // --- Round 2: Reduce quads (4 elements -> 2 elements per vector) ---
406 // The final pixel sum is now split between the low 128-bit half and the high 128-bit half.
407 __m128i sum0123 = _mm_hadd_epi32(sum01, sum23);
408 __m128i sum4567 = _mm_hadd_epi32(sum45, sum67);
409
410 // 2. Assemble the two 128-bit blocks into the final 256-bit result (VINSERTI128).
411 __m256i result_8x_32 = _mm256_inserti128_si256(_mm256_castsi128_si256(sum0123), sum4567, 1);
412
413 // --- Final Vectorized Rounding ---
414
415 // Create the rounder vector
416 __m256i rounder_v = _mm256_set1_epi32(rounder_scalar);
417
418 // Apply rounding to all 8 pixels simultaneously (VPADDD)
419 return _mm256_add_epi32(result_8x_32, rounder_v);
420 }
421
422
423 // -----------------------------------------------------------------------------------------
424 // Wrapper for Two-Pixel Processing
425 // -----------------------------------------------------------------------------------------
426 template<bool safe_aligned_mode, typename pixel_t, bool lessthan16bit, int FixedFilterSize>
427 AVS_FORCEINLINE static void process_two_pixels_h_avx512(const pixel_t * AVS_RESTRICT src_ptr, int begin1, int begin2, const short* AVS_RESTRICT current_coeff,
428 int filter_size, __m256i & result1, __m256i & result2, int kernel_size, __m256i & shifttosigned) {
429
430 // filter_size here is the stride for coeffs, kernel_size is the actual number of taps to process.
431
432 if constexpr (FixedFilterSize == 4) {
433 process_two_partial_unrolled<pixel_t, lessthan16bit, 4>(src_ptr, begin1, begin2, 0, current_coeff, filter_size, result1, result2, shifttosigned);
434 return;
435 }
436 if constexpr (FixedFilterSize == 8) {
437 process_two_partial_unrolled<pixel_t, lessthan16bit, 8>(src_ptr, begin1, begin2, 0, current_coeff, filter_size, result1, result2, shifttosigned);
438 return;
439 }
440 if constexpr (FixedFilterSize == 12) {
441 process_two_partial_unrolled<pixel_t, lessthan16bit, 8>(src_ptr, begin1, begin2, 0, current_coeff, filter_size, result1, result2, shifttosigned);
442 process_two_partial_unrolled<pixel_t, lessthan16bit, 4>(src_ptr, begin1, begin2, 8, current_coeff, filter_size, result1, result2, shifttosigned);
443 return;
444 }
445
446 // 2. Large Kernel Loop (16-tap blocks)
447 int i = 0;
448 // We can use the FixedFilterSize to cap the loop if it's large (like 48, 64)
449 int runtime_filter_size = (FixedFilterSize >= 1) ? FixedFilterSize : filter_size;
450 int ksmod16 = (safe_aligned_mode && FixedFilterSize >= 16) ? (FixedFilterSize / 16 * 16) : (kernel_size / 16 * 16);
451
452 for (; i < ksmod16; i += 16) {
453 process_two_16pixels_core<pixel_t, lessthan16bit>(src_ptr, begin1, begin2, i, current_coeff + i, filter_size, result1, result2, shifttosigned);
454 }
455
456 // 3. Tail Handling
457 // If we are in safe mode and FixedSize is a multiple of 32, we are done.
458 if constexpr (safe_aligned_mode && (FixedFilterSize % 16 == 0) && FixedFilterSize > 0) return;
459
460 int remaining = runtime_filter_size - i;
461 if (remaining <= 0) return;
462
463 // Process remaining using scalar fallbacks
464 // Unrolled helpers for chunks of 8/4/scalar.
465
466 // Chunk 8
467 if (remaining >= 8) {
468 process_two_partial_unrolled<pixel_t, lessthan16bit, 8>(src_ptr, begin1, begin2, i, current_coeff, filter_size, result1, result2, shifttosigned);
469 i += 8;
470 remaining -= 8;
471 }
472 if (remaining >= 4) {
473 process_two_partial_unrolled<pixel_t, lessthan16bit, 4>(src_ptr, begin1, begin2, i, current_coeff, filter_size, result1, result2, shifttosigned);
474 i += 4;
475 remaining -= 4;
476 }
477
478 // Final scalar tail (1-3 pixels)
479 while (remaining > 0) {
480 int val1, val2;
481 if constexpr (sizeof(pixel_t) == 1) {
482 val1 = src_ptr[begin1 + i];
483 val2 = src_ptr[begin2 + i];
484 }
485 else {
486 val1 = src_ptr[begin1 + i];
487 val2 = src_ptr[begin2 + i];
488 if constexpr (!lessthan16bit) { val1 -= 32768; val2 -= 32768; }
489 }
490 int c1 = current_coeff[i];
491 int c2 = current_coeff[filter_size + i];
492
493 // Add to the first lane of the accumulator
494 __m256i s1 = _mm256_zextsi128_si256(_mm_cvtsi32_si128(val1 * c1));
495 __m256i s2 = _mm256_zextsi128_si256(_mm_cvtsi32_si128(val2 * c2));
496
497 result1 = _mm256_add_epi32(result1, s1);
498 result2 = _mm256_add_epi32(result2, s2);
499
500 i++;
501 remaining--;
502 }
503 }
504
505 // -----------------------------------------------------------------------------------------
506 // Wrapper for Four-Pixel Processing
507 // -----------------------------------------------------------------------------------------
508 template<bool safe_aligned_mode, typename pixel_t, bool lessthan16bit, int FixedFilterSize>
509 AVS_FORCEINLINE static void process_four_pixels_h_avx512(const pixel_t* AVS_RESTRICT src_ptr,
510 int begin1, int begin2, int begin3, int begin4, const short* AVS_RESTRICT current_coeff, int filter_size,
511 __m256i& result1, __m256i& result2, __m256i& result3, __m256i& result4, int kernel_size, __m256i& shifttosigned) {
512
513 // filter_size here is the stride for coeffs, kernel_size is the actual number of taps to process.
514
515 if constexpr (FixedFilterSize == 4) {
516 process_four_partial_unrolled<pixel_t, lessthan16bit, 4>(src_ptr, begin1, begin2, begin3, begin4, 0, current_coeff, filter_size, result1, result2, result3, result4, shifttosigned);
517 ✗ return;
518 }
519 if constexpr (FixedFilterSize == 8) {
520 process_four_partial_unrolled<pixel_t, lessthan16bit, 8>(src_ptr, begin1, begin2, begin3, begin4, 0, current_coeff, filter_size, result1, result2, result3, result4, shifttosigned);
521 ✗ return;
522 }
523 if constexpr (FixedFilterSize == 12) {
524 process_four_partial_unrolled<pixel_t, lessthan16bit, 8>(src_ptr, begin1, begin2, begin3, begin4, 0, current_coeff, filter_size, result1, result2, result3, result4, shifttosigned);
525 process_four_partial_unrolled<pixel_t, lessthan16bit, 4>(src_ptr, begin1, begin2, begin3, begin4, 8, current_coeff, filter_size, result1, result2, result3, result4, shifttosigned);
526 ✗ return;
527 }
528
529 // 2. Large Kernel Loop (16-tap blocks)
530 ✗ int i = 0;
531 // We can use the FixedFilterSize to cap the loop if it's large (like 48, 64)
532 ✗ int runtime_filter_size = (FixedFilterSize >= 1) ? FixedFilterSize : filter_size;
533 ✗ int ksmod16 = (safe_aligned_mode && FixedFilterSize >= 16) ? (FixedFilterSize / 16 * 16) : (kernel_size / 16 * 16);
534
535 ✗ for (; i < ksmod16; i += 16) {
536 ✗ process_four_16pixels_core<pixel_t, lessthan16bit>(src_ptr, begin1, begin2, begin3, begin4, i, current_coeff + i, filter_size, result1, result2, result3, result4, shifttosigned);
537 }
538
539 // 3. Tail Handling
540 // If we are in safe mode and FixedSize is a multiple of 32, we are done.
541 ✗ if constexpr (safe_aligned_mode && (FixedFilterSize % 16 == 0) && FixedFilterSize > 0) return;
542
543 ✗ int remaining = runtime_filter_size - i;
544 ✗ if (remaining <= 0) return;
545
546 // Process remaining using scalar fallbacks
547 // Unrolled helpers for chunks of 8/4/scalar.
548
549 // Chunk 8
550 ✗ if (remaining >= 8) {
551 process_four_partial_unrolled<pixel_t, lessthan16bit, 8>(src_ptr, begin1, begin2, begin3, begin4, i, current_coeff, filter_size, result1, result2, result3, result4, shifttosigned);
552 ✗ i += 8;
553 ✗ remaining -= 8;
554 }
555 ✗ if (remaining >= 4) {
556 process_four_partial_unrolled<pixel_t, lessthan16bit, 4>(src_ptr, begin1, begin2, begin3, begin4, i, current_coeff, filter_size, result1, result2, result3, result4, shifttosigned);
557 ✗ i += 4;
558 ✗ remaining -= 4;
559 }
560
561 // Final scalar tail (1-3 pixels)
562 ✗ while (remaining > 0) {
563 int val1, val2, val3, val4;
564 if constexpr (sizeof(pixel_t) == 1) {
565 ✗ val1 = src_ptr[begin1 + i];
566 ✗ val2 = src_ptr[begin2 + i];
567 ✗ val3 = src_ptr[begin3 + i];
568 ✗ val4 = src_ptr[begin4 + i];
569 }
570 else {
571 ✗ val1 = src_ptr[begin1 + i];
572 ✗ val2 = src_ptr[begin2 + i];
573 ✗ val3 = src_ptr[begin3 + i];
574 ✗ val4 = src_ptr[begin4 + i];
575 ✗ if constexpr (!lessthan16bit) { val1 -= 32768; val2 -= 32768; val3 -= 32768; val4 -= 32768; }
576 }
577 ✗ int c1 = current_coeff[i];
578 ✗ int c2 = current_coeff[filter_size + i];
579 ✗ int c3 = current_coeff[2 * filter_size + i];
580 ✗ int c4 = current_coeff[3 * filter_size + i];
581
582 // Add to the first lane of the accumulator
583 ✗ __m256i s1 = _mm256_zextsi128_si256(_mm_cvtsi32_si128(val1 * c1));
584 ✗ __m256i s2 = _mm256_zextsi128_si256(_mm_cvtsi32_si128(val2 * c2));
585 ✗ __m256i s3 = _mm256_zextsi128_si256(_mm_cvtsi32_si128(val3 * c3));
586 ✗ __m256i s4 = _mm256_zextsi128_si256(_mm_cvtsi32_si128(val4 * c4));
587
588 ✗ result1 = _mm256_add_epi32(result1, s1);
589 ✗ result2 = _mm256_add_epi32(result2, s2);
590 ✗ result3 = _mm256_add_epi32(result3, s3);
591 ✗ result4 = _mm256_add_epi32(result4, s4);
592
593 ✗ i++;
594 ✗ remaining--;
595 }
596 }
597
598 // -----------------------------------------------------------------------------------------
599 // Main Block Processor, TWO_PIXELS_AT_ONCE for testing performance difference
600 // -----------------------------------------------------------------------------------------
601 template<bool is_safe, typename pixel_t, bool lessthan16bit, int FixedFilterSize>
602 AVS_FORCEINLINE static void process_sixteen_pixels_h_avx512(const pixel_t * src, int x, const short* current_coeff_base, int filter_size,
603 int rounder_scalar, __m256i& shifttosigned, __m256i& clamp_limit,
604 pixel_t* dst, ResamplingProgram* program)
605 {
606 ✗ int run_filter_size_stride = (FixedFilterSize >= 1) ? FixedFilterSize : filter_size; // quasi constexpr if templated
607 ✗ const short* AVS_RESTRICT current_coeff = current_coeff_base + x * run_filter_size_stride;
608 ✗ const int unaligned_kernel_size = program->filter_size_real;
609
610 #ifdef TWO_PIXELS_AT_ONCE
611 __m256i result0 = _mm256_setzero_si256();
612 __m256i result1 = _mm256_setzero_si256();
613 process_two_pixels_h_avx512<is_safe, pixel_t, lessthan16bit, FixedFilterSize>(src, program->pixel_offset[x], program->pixel_offset[x + 1], current_coeff, run_filter_size_stride, result0, result1, unaligned_kernel_size, shifttosigned);
614 current_coeff += 2 * run_filter_size_stride;
615
616 __m256i result2 = _mm256_setzero_si256();
617 __m256i result3 = _mm256_setzero_si256();
618 process_two_pixels_h_avx512<is_safe, pixel_t, lessthan16bit, FixedFilterSize>(src, program->pixel_offset[x + 2], program->pixel_offset[x + 3], current_coeff, run_filter_size_stride, result2, result3, unaligned_kernel_size, shifttosigned);
619 current_coeff += 2 * run_filter_size_stride;
620
621 __m256i result4 = _mm256_setzero_si256();
622 __m256i result5 = _mm256_setzero_si256();
623 process_two_pixels_h_avx512<is_safe, pixel_t, lessthan16bit, FixedFilterSize>(src, program->pixel_offset[x + 4], program->pixel_offset[x + 5], current_coeff, run_filter_size_stride, result4, result5, unaligned_kernel_size, shifttosigned);
624 current_coeff += 2 * run_filter_size_stride;
625
626 __m256i result6 = _mm256_setzero_si256();
627 __m256i result7 = _mm256_setzero_si256();
628 process_two_pixels_h_avx512<is_safe, pixel_t, lessthan16bit, FixedFilterSize>(src, program->pixel_offset[x + 6], program->pixel_offset[x + 7], current_coeff, run_filter_size_stride, result6, result7, unaligned_kernel_size, shifttosigned);
629 current_coeff += 2 * run_filter_size_stride;
630 #else
631 ✗ __m256i result0 = _mm256_setzero_si256();
632 ✗ __m256i result1 = _mm256_setzero_si256();
633 ✗ __m256i result2 = _mm256_setzero_si256();
634 ✗ __m256i result3 = _mm256_setzero_si256();
635 ✗ process_four_pixels_h_avx512 < is_safe, pixel_t, lessthan16bit, FixedFilterSize>(src,
636 ✗ program->pixel_offset[x], program->pixel_offset[x + 1],
637 ✗ program->pixel_offset[x + 2], program->pixel_offset[x + 3],
638 current_coeff, run_filter_size_stride,
639 result0, result1, result2, result3,
640 unaligned_kernel_size, shifttosigned);
641 ✗ current_coeff += 4 * run_filter_size_stride;
642 ✗ __m256i result4 = _mm256_setzero_si256();
643 ✗ __m256i result5 = _mm256_setzero_si256();
644 ✗ __m256i result6 = _mm256_setzero_si256();
645 ✗ __m256i result7 = _mm256_setzero_si256();
646 ✗ process_four_pixels_h_avx512 < is_safe, pixel_t, lessthan16bit, FixedFilterSize>(src,
647 ✗ program->pixel_offset[x + 4], program->pixel_offset[x + 5],
648 ✗ program->pixel_offset[x + 6], program->pixel_offset[x + 7],
649 current_coeff, run_filter_size_stride,
650 result4, result5, result6, result7,
651 unaligned_kernel_size, shifttosigned);
652 ✗ current_coeff += 4 * run_filter_size_stride;
653 #endif
654 ✗ __m256i result_8x_32_lo = reduce_8x256i_32i_tree(
655 result0, result1, result2, result3,
656 result4, result5, result6, result7,
657 rounder_scalar);
658
659 // same for pixels 8..15
660 #ifdef TWO_PIXELS_AT_ONCE
661 __m256i result8 = _mm256_setzero_si256();
662 __m256i result9 = _mm256_setzero_si256();
663 process_two_pixels_h_avx512<is_safe, pixel_t, lessthan16bit, FixedFilterSize>(src, program->pixel_offset[x + 8], program->pixel_offset[x + 9], current_coeff, run_filter_size_stride, result8, result9, unaligned_kernel_size, shifttosigned);
664 current_coeff += 2 * run_filter_size_stride;
665
666 __m256i result10 = _mm256_setzero_si256();
667 __m256i result11 = _mm256_setzero_si256();
668 process_two_pixels_h_avx512<is_safe, pixel_t, lessthan16bit, FixedFilterSize>(src, program->pixel_offset[x + 10], program->pixel_offset[x + 11], current_coeff, run_filter_size_stride, result10, result11, unaligned_kernel_size, shifttosigned);
669 current_coeff += 2 * run_filter_size_stride;
670
671 __m256i result12 = _mm256_setzero_si256();
672 __m256i result13 = _mm256_setzero_si256();
673 process_two_pixels_h_avx512<is_safe, pixel_t, lessthan16bit, FixedFilterSize>(src, program->pixel_offset[x + 12], program->pixel_offset[x + 13], current_coeff, run_filter_size_stride, result12, result13, unaligned_kernel_size, shifttosigned);
674 current_coeff += 2 * run_filter_size_stride;
675
676 __m256i result14 = _mm256_setzero_si256();
677 __m256i result15 = _mm256_setzero_si256();
678 process_two_pixels_h_avx512<is_safe, pixel_t, lessthan16bit, FixedFilterSize>(src, program->pixel_offset[x + 14], program->pixel_offset[x + 15], current_coeff, run_filter_size_stride, result14, result15, unaligned_kernel_size, shifttosigned);
679 // last one, no need:
680 // current_coeff += 2 * run_filter_size_stride;
681 #else
682 ✗ __m256i result8 = _mm256_setzero_si256();
683 ✗ __m256i result9 = _mm256_setzero_si256();
684 ✗ __m256i result10 = _mm256_setzero_si256();
685 ✗ __m256i result11 = _mm256_setzero_si256();
686 ✗ process_four_pixels_h_avx512 < is_safe, pixel_t, lessthan16bit, FixedFilterSize>(src,
687 ✗ program->pixel_offset[x + 8], program->pixel_offset[x + 9],
688 ✗ program->pixel_offset[x + 10], program->pixel_offset[x + 11],
689 current_coeff, run_filter_size_stride,
690 result8, result9, result10, result11,
691 unaligned_kernel_size, shifttosigned);
692 ✗ current_coeff += 4 * run_filter_size_stride;
693 ✗ __m256i result12 = _mm256_setzero_si256();
694 ✗ __m256i result13 = _mm256_setzero_si256();
695 ✗ __m256i result14 = _mm256_setzero_si256();
696 ✗ __m256i result15 = _mm256_setzero_si256();
697 ✗ process_four_pixels_h_avx512 < is_safe, pixel_t, lessthan16bit, FixedFilterSize>(src,
698 ✗ program->pixel_offset[x + 12], program->pixel_offset[x + 13],
699 ✗ program->pixel_offset[x + 14], program->pixel_offset[x + 15],
700 current_coeff, run_filter_size_stride,
701 result12, result13, result14, result15,
702 unaligned_kernel_size, shifttosigned);
703 #endif
704
705 ✗ __m256i result_8x_32_hi = reduce_8x256i_32i_tree(
706 result8, result9, result10, result11,
707 result12, result13, result14, result15,
708 rounder_scalar);
709
710 //
711 if constexpr (sizeof(pixel_t) == 2 && !lessthan16bit) {
712 ✗ const __m256i shiftfromsigned = _mm256_set1_epi32(+32768 << FPScale16bits);
713 ✗ result_8x_32_lo = _mm256_add_epi32(result_8x_32_lo, shiftfromsigned);
714 ✗ result_8x_32_hi = _mm256_add_epi32(result_8x_32_hi, shiftfromsigned);
715 }
716
717 // ... scale/pack ...
718 ✗ const int shift = (sizeof(pixel_t) == 1) ? FPScale8bits : FPScale16bits;
719 ✗ __m256i scaled_lo = _mm256_srai_epi32(result_8x_32_lo, shift);
720 ✗ __m256i scaled_hi = _mm256_srai_epi32(result_8x_32_hi, shift);
721
722 // integer 32->unsigned 16 bit, the usual and quick way
723 ✗ __m256i result_16 = _mm256_packus_epi32(scaled_lo, scaled_hi);
724
725 // we have 8x16 bit unsigned pixels now in result_16
726 if constexpr (sizeof(pixel_t) == 2 && lessthan16bit) {
727 ✗ result_16 = _mm256_min_epu16(result_16, clamp_limit);
728 }
729
730 ✗ result_16 = _mm256_permute4x64_epi64(result_16, (0 << 0) | (2 << 2) | (1 << 4) | (3 << 6));
731
732 if constexpr (sizeof(pixel_t) == 1) {
733 ✗ __m128i result_8 = _mm_packus_epi16(_mm256_castsi256_si128(result_16), _mm256_extracti128_si256(result_16, 1));
734 ✗ _mm_stream_si128(reinterpret_cast<__m128i*>(dst + x), result_8); // 16x 8bit pixels
735 }
736 else {
737 ✗ _mm256_stream_si256(reinterpret_cast<__m256i*>(dst + x), result_16);
738 }
739 ✗ }
740
741 // -----------------------------------------------------------------------------------------
742 // Dispatcher
743 // -----------------------------------------------------------------------------------------
744 template<typename pixel_t, bool lessthan16bit, int FixedFilterSize>
745 ✗ static void internal_resizer_h_avx512_generic(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) {
746 ✗ int current_fp_scale_bits = (sizeof(pixel_t) == 1) ? FPScale8bits : FPScale16bits;
747 ✗ int rounder_scalar = 1 << (current_fp_scale_bits - 1);
748
749 ✗ __m256i shifttosigned = _mm256_set1_epi16(-32768);
750 ✗ __m256i clamp_limit = _mm256_set1_epi16((short)((1 << bits_per_pixel) - 1));
751
752 ✗ const pixel_t* src = reinterpret_cast<const pixel_t*>(src8);
753 ✗ pixel_t* dst = reinterpret_cast<pixel_t*>(dst8);
754 ✗ dst_pitch /= sizeof(pixel_t);
755 ✗ src_pitch /= sizeof(pixel_t);
756
757 ✗ const int PIXELS_AT_A_TIME = 16;
758
759 ✗ const int filter_size = (FixedFilterSize >= 1) ? FixedFilterSize : program->filter_size; // aligned coeff stride
760 ✗ const int w_safe = (program->safelimit_filter_size_aligned.overread_possible ? program->safelimit_filter_size_aligned.source_overread_beyond_targetx : width) / PIXELS_AT_A_TIME * PIXELS_AT_A_TIME;
761
762 ✗ for (int y = 0; y < height; y++) {
763 ✗ for (int x = 0; x < w_safe; x += PIXELS_AT_A_TIME) {
764 ✗ process_sixteen_pixels_h_avx512<true, pixel_t, lessthan16bit, FixedFilterSize>(src, x, program->pixel_coefficient, filter_size, rounder_scalar, shifttosigned, clamp_limit, dst, program);
765 }
766 ✗ for (int x = w_safe; x < width; x += PIXELS_AT_A_TIME) {
767 ✗ process_sixteen_pixels_h_avx512<false, pixel_t, lessthan16bit, FixedFilterSize>(src, x, program->pixel_coefficient, filter_size, rounder_scalar, shifttosigned, clamp_limit, dst, program);
768 }
769 ✗ dst += dst_pitch;
770 ✗ src += src_pitch;
771 }
772 ✗ }
773
774 // Entry Points: Map filter sizes to optimized templates
775 ✗ void resizer_h_avx512_generic_uint8_t(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) {
776 ✗ int fs = program->filter_size; // aligned coeff stride
777 // Dispatch to optimized templates based on filter size
778 ✗ if (fs == 4) internal_resizer_h_avx512_generic<uint8_t, true, 4>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
779 ✗ else if (fs == 8) internal_resizer_h_avx512_generic<uint8_t, true, 8>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
780 ✗ else if (fs == 12) internal_resizer_h_avx512_generic<uint8_t, true, 12>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
781 ✗ else if (fs == 16) internal_resizer_h_avx512_generic<uint8_t, true, 16>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
782 ✗ else if (fs == 32) internal_resizer_h_avx512_generic<uint8_t, true, 32>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
783 ✗ else if (fs == 48) internal_resizer_h_avx512_generic<uint8_t, true, 48>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
784 ✗ else internal_resizer_h_avx512_generic<uint8_t, true, -1>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
785 ✗ }
786
787 // 16 bit Horizontal Dispatcher
788 // Handles both full 16-bit (lessthan16bit=false) and 10-14 bit (lessthan16bit=true)
789 template<bool lessthan16bit>
790 ✗ void resizer_h_avx512_generic_uint16_t(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) {
791 ✗ const int fs = program->filter_size; // aligned coeff stride
792 // Dispatch to optimized templates based on filter size
793 ✗ if (fs == 4) internal_resizer_h_avx512_generic<uint16_t, lessthan16bit, 4>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
794 ✗ else if (fs == 8) internal_resizer_h_avx512_generic<uint16_t, lessthan16bit, 8>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
795 ✗ else if (fs == 12) internal_resizer_h_avx512_generic<uint16_t, lessthan16bit, 12>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
796 ✗ else if (fs == 16) internal_resizer_h_avx512_generic<uint16_t, lessthan16bit, 16>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
797 ✗ else if (fs == 32) internal_resizer_h_avx512_generic<uint16_t, lessthan16bit, 32>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
798 ✗ else if (fs == 48) internal_resizer_h_avx512_generic<uint16_t, lessthan16bit, 48>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
799 ✗ else internal_resizer_h_avx512_generic<uint16_t, lessthan16bit, -1>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
800 ✗ }
801
802 // Explicit template instantiation
803 // AVX512 16-bit (unsigned, full range)
804 template void resizer_h_avx512_generic_uint16_t<false>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
805
806 // AVX512 10-14 bit (requires clamping)
807 template void resizer_h_avx512_generic_uint16_t<true>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
808
809
810 //------- 512 bit float Horizontals
811
812 // Safe quad lane partial load with AVX512 using masks.
813 // Replaces scalar set_ps sequences with hardware-masked loads.
814 // Requires AVX-512VL (standard on Rocket Lake, Zen 4/5).
815 AVS_FORCEINLINE static __m512 _mm512_load_partial_safe_4_m128(const float* src_ptr_offsetted1, const float* src_ptr_offsetted2, const float* src_ptr_offsetted3, const float* src_ptr_offsetted4, int floats_to_load) {
816 // Example: N=3 -> (1<<3)-1 = 7 (binary 0111) -> Loads floats 0, 1, 2.
817 ✗ const __mmask8 k = (1 << floats_to_load) - 1;
818
819 // perform masked loads.
820 // _mm_maskz_loadu_ps(k, ptr):
821 // - Loads 'valid_pixels' floats from memory.
822 // - Zeros out the remaining upper floats in the XMM register (z-masking).
823 // - FAULT SUPPRESSION: Hardware guarantees no page fault for masked-off elements.
824 // Thought Avisynth frame buffers are overallocated to avoid OOB reads, this adds extra safety.
825 ✗ __m128 s1 = _mm_maskz_loadu_ps(k, src_ptr_offsetted1);
826 ✗ __m128 s2 = _mm_maskz_loadu_ps(k, src_ptr_offsetted2);
827 ✗ __m128 s3 = _mm_maskz_loadu_ps(k, src_ptr_offsetted3);
828 ✗ __m128 s4 = _mm_maskz_loadu_ps(k, src_ptr_offsetted4);
829
830 // Combine into ZMM
831 ✗ __m512 result = _mm512_castps128_ps512(s1); // Free (register aliasing)
832 ✗ result = _mm512_insertf32x4(result, s2, 1); // vinsertf32x4
833 ✗ result = _mm512_insertf32x4(result, s3, 2);
834 ✗ result = _mm512_insertf32x4(result, s4, 3);
835
836 ✗ return result;
837 }
838
839 AVS_FORCEINLINE static __m512 _mm512_load_partial_safe_2_m256(const float* src_ptr_offsetted1, const float* src_ptr_offsetted2, int floats_to_load) {
840 // Calculate the mask.
841 ✗ const __mmask8 k = (1U << floats_to_load) - 1;
842
843 // _mm256_maskz_loadu_ps provides fault suppression for masked-off elements,
844 // ensuring no page faults occur even if the pointer is near a boundary.
845 ✗ __m256 s1 = _mm256_maskz_loadu_ps(k, src_ptr_offsetted1);
846 ✗ __m256 s2 = _mm256_maskz_loadu_ps(k, src_ptr_offsetted2);
847
848 // Combine into ZMM.
849 ✗ __m512 result = _mm512_castps256_ps512(s1);
850 ✗ result = _mm512_insertf32x8(result, s2, 1);
851
852 ✗ return result;
853 }
854
855 #if 0
856 // Safe quad lane partial load with AVX512
857 // Read exactly N pixels (where N mod 4 is the template parameter), avoiding
858 // - reading beyond the end of the source buffer.
859 // - avoid NaN contamination by padding with zeros.
860 template <int Nmod4>
861 AVS_FORCEINLINE static __m512 _mm512_load_partial_safe_4_m128_avx2like(const float* src_ptr_offsetted1, const float* src_ptr_offsetted2, const float* src_ptr_offsetted3, const float* src_ptr_offsetted4) {
862 __m128 s1, s2, s3, s4;
863 switch (Nmod4) {
864 case 1:
865 s1 = _mm_set_ps(0.0f, 0.0f, 0.0f, src_ptr_offsetted1[0]);
866 s2 = _mm_set_ps(0.0f, 0.0f, 0.0f, src_ptr_offsetted2[0]);
867 s3 = _mm_set_ps(0.0f, 0.0f, 0.0f, src_ptr_offsetted3[0]);
868 s4 = _mm_set_ps(0.0f, 0.0f, 0.0f, src_ptr_offsetted4[0]);
869 // ideally: movss
870 break;
871 case 2:
872 s1 = _mm_set_ps(0.0f, 0.0f, src_ptr_offsetted1[1], src_ptr_offsetted1[0]);
873 s2 = _mm_set_ps(0.0f, 0.0f, src_ptr_offsetted2[1], src_ptr_offsetted2[0]);
874 s3 = _mm_set_ps(0.0f, 0.0f, src_ptr_offsetted3[1], src_ptr_offsetted3[0]);
875 s4 = _mm_set_ps(0.0f, 0.0f, src_ptr_offsetted4[1], src_ptr_offsetted4[0]);
876 // ideally: movsd
877 break;
878 case 3:
879 s1 = _mm_set_ps(0.0f, src_ptr_offsetted1[2], src_ptr_offsetted1[1], src_ptr_offsetted1[0]);
880 s2 = _mm_set_ps(0.0f, src_ptr_offsetted2[2], src_ptr_offsetted2[1], src_ptr_offsetted2[0]);
881 s3 = _mm_set_ps(0.0f, src_ptr_offsetted3[2], src_ptr_offsetted3[1], src_ptr_offsetted3[0]);
882 s4 = _mm_set_ps(0.0f, src_ptr_offsetted4[2], src_ptr_offsetted4[1], src_ptr_offsetted4[0]);
883 // ideally: movss + movsd + shuffle or movsd + insert
884 break;
885 case 0:
886 s1 = _mm_set_ps(src_ptr_offsetted1[3], src_ptr_offsetted1[2], src_ptr_offsetted1[1], src_ptr_offsetted1[0]);
887 s2 = _mm_set_ps(src_ptr_offsetted2[3], src_ptr_offsetted2[2], src_ptr_offsetted2[1], src_ptr_offsetted2[0]);
888 s3 = _mm_set_ps(src_ptr_offsetted3[3], src_ptr_offsetted3[2], src_ptr_offsetted3[1], src_ptr_offsetted3[0]);
889 s4 = _mm_set_ps(src_ptr_offsetted4[3], src_ptr_offsetted4[2], src_ptr_offsetted4[1], src_ptr_offsetted4[0]);
890 // ideally: movups
891 break;
892 default:
893 s1 = _mm_setzero_ps(); // n/a cannot happen
894 s2 = _mm_setzero_ps();
895 s3 = _mm_setzero_ps();
896 s4 = _mm_setzero_ps();
897 }
898 __m512 result = _mm512_castps128_ps512(s1); // Cast the first __m128 to __m512
899 result = _mm512_insertf32x4(result, s2, 1); // Insert the second __m128 at position 1
900 result = _mm512_insertf32x4(result, s3, 2); // Insert the third __m128 at position 2
901 result = _mm512_insertf32x4(result, s4, 3); // Insert the fourth __m128 at position 3
902 return result;
903 }
904 #endif
905
906 // Processes a horizontal resampling kernel of up to four coefficients for float pixel types.
907 // Supports BilinearResize, BicubicResize, or sinc with up to 2 taps (filter size <= 4).
908 // AVX512 optimization loads and processes four float coefficients and sixteen pixels simultaneously.
909 // This AVX512 requires only filter_size_alignment of 4.
910 ✗ void resize_h_planar_float_avx512_transpose_vstripe_ks4(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) {
911
912 ✗ const int filter_size = program->filter_size; // aligned, practically the coeff table stride
913 // needed for partial load
914 ✗ const int Nmod4 = program->filter_size_real % 4;
915 ✗ const int floats_to_load = (Nmod4 == 0) ? 4 : Nmod4;
916
917 ✗ src_pitch /= sizeof(float);
918 ✗ dst_pitch /= sizeof(float);
919
920 ✗ float* src = (float*)src8;
921 ✗ float* dst = (float*)dst8;
922
923 ✗ constexpr int PIXELS_AT_A_TIME = 16; // Process sixteen pixels in parallel using AVX512 (4x4 using m128 lanes)
924
925 // 'source_overread_beyond_targetx' indicates if the filter kernel can read beyond the target width.
926 // Even if the filter alignment allows larger reads, our safety boundary for unaligned loads starts at 4 pixels back
927 // from the target width, as we load 4 floats at once conceptually with our safe load.
928 ✗ const int width_safe_mod = (program->safelimit_4_pixels.overread_possible ? program->safelimit_4_pixels.source_overread_beyond_targetx : width) / PIXELS_AT_A_TIME * PIXELS_AT_A_TIME;
929
930 // Preconditions:
931 ✗ assert(program->filter_size_real <= 4);
932 ✗ assert(program->target_size_alignment >= 16); // Must align for 16 pixel offsets
933 ✗ assert(program->filter_size_alignment >= 4);
934 assert(FRAME_ALIGN >= 64); // Adjusted for 16 pixels AviSynth+ default
935
936 // Vertical stripe alignment
937 ✗ constexpr int STRIPE_ALIGN = 16;
938
939 ✗ int max_scanlines = program->max_scanlines / STRIPE_ALIGN * STRIPE_ALIGN;
940 ✗ if (max_scanlines < STRIPE_ALIGN) max_scanlines = STRIPE_ALIGN;
941
942 // --- outer loop: vertical stripes ---
943 ✗ for (auto y_from = 0; y_from < height; y_from += max_scanlines) {
944 ✗ size_t y_to = std::min(y_from + max_scanlines, height);
945
946 // Reset current_coeff for the start of the stripe
947 ✗ const float* AVS_RESTRICT current_coeff = program->pixel_coefficient_float;
948
949 ✗ size_t x = 0;
950
951 // Lambda for 512-bit Core
952 ✗ auto do_h_float_core = [&](auto partial_load) {
953
954 // load 4x4 sets of coefficients (16 pixels total)
955 // at once before the height loop.
956 // Coefficients for the source pixel offset (for src_ptr + begin1 [0..3], begin5 [0..3], begin9 [0..3], begin13 [0..3])
957 ✗ __m512 coef_1_5_9_13 = _mm512_load_4_m128(current_coeff + filter_size * 0, current_coeff + filter_size * 4, current_coeff + filter_size * 8, current_coeff + filter_size * 12);
958 ✗ __m512 coef_2_6_10_14 = _mm512_load_4_m128(current_coeff + filter_size * 1, current_coeff + filter_size * 5, current_coeff + filter_size * 9, current_coeff + filter_size * 13);
959 ✗ __m512 coef_3_7_11_15 = _mm512_load_4_m128(current_coeff + filter_size * 2, current_coeff + filter_size * 6, current_coeff + filter_size * 10, current_coeff + filter_size * 14);
960 ✗ __m512 coef_4_8_12_16 = _mm512_load_4_m128(current_coeff + filter_size * 3, current_coeff + filter_size * 7, current_coeff + filter_size * 11, current_coeff + filter_size * 15);
961
962 _MM_TRANSPOSE16_LANE4_PS(coef_1_5_9_13, coef_2_6_10_14, coef_3_7_11_15, coef_4_8_12_16);
963
964 // Pixel offsets for the current target x-positions.
965 // Even for x >= width, these offsets are guaranteed to be within the allocated 'target_size_alignment'.
966 ✗ const int begin1 = program->pixel_offset[x + 0];
967 ✗ const int begin2 = program->pixel_offset[x + 1];
968 ✗ const int begin3 = program->pixel_offset[x + 2];
969 ✗ const int begin4 = program->pixel_offset[x + 3];
970 ✗ const int begin5 = program->pixel_offset[x + 4];
971 ✗ const int begin6 = program->pixel_offset[x + 5];
972 ✗ const int begin7 = program->pixel_offset[x + 6];
973 ✗ const int begin8 = program->pixel_offset[x + 7];
974 ✗ const int begin9 = program->pixel_offset[x + 8];
975 ✗ const int begin10 = program->pixel_offset[x + 9];
976 ✗ const int begin11 = program->pixel_offset[x + 10];
977 ✗ const int begin12 = program->pixel_offset[x + 11];
978 ✗ const int begin13 = program->pixel_offset[x + 12];
979 ✗ const int begin14 = program->pixel_offset[x + 13];
980 ✗ const int begin15 = program->pixel_offset[x + 14];
981 ✗ const int begin16 = program->pixel_offset[x + 15];
982
983 ✗ int y = y_from;
984
985 // Calculate pointers ONCE before the inner loop (Optimization from AVX2 version)
986 ✗ float* AVS_RESTRICT dst_ptr = dst + y * dst_pitch + x;
987 ✗ const float* src_ptr = src + y * src_pitch;
988
989 // Inner loop: vertical processing. unroll 2 tested, no benefit
990 ✗ for (; y < y_to; ++y) {
991
992 __m512 data_1_5_9_13;
993 __m512 data_2_6_10_14;
994 __m512 data_3_7_11_15;
995 __m512 data_4_8_12_16;
996
997 if constexpr (partial_load) {
998 // In the potentially unsafe zone (near the right edge of the image), we use a safe loading function
999 // to prevent reading beyond the allocated source scanline.
1000 ✗ data_1_5_9_13 = _mm512_load_partial_safe_4_m128(src_ptr + begin1, src_ptr + begin5, src_ptr + begin9, src_ptr + begin13, floats_to_load);
1001 ✗ data_2_6_10_14 = _mm512_load_partial_safe_4_m128(src_ptr + begin2, src_ptr + begin6, src_ptr + begin10, src_ptr + begin14, floats_to_load);
1002 ✗ data_3_7_11_15 = _mm512_load_partial_safe_4_m128(src_ptr + begin3, src_ptr + begin7, src_ptr + begin11, src_ptr + begin15, floats_to_load);
1003 ✗ data_4_8_12_16 = _mm512_load_partial_safe_4_m128(src_ptr + begin4, src_ptr + begin8, src_ptr + begin12, src_ptr + begin16, floats_to_load);
1004 }
1005 else {
1006 // In the safe zone, we can directly load 4 source pixels at a time for each of the 16 lanes.
1007 ✗ data_1_5_9_13 = _mm512_loadu_4_m128(src_ptr + begin1, src_ptr + begin5, src_ptr + begin9, src_ptr + begin13);
1008 ✗ data_2_6_10_14 = _mm512_loadu_4_m128(src_ptr + begin2, src_ptr + begin6, src_ptr + begin10, src_ptr + begin14);
1009 ✗ data_3_7_11_15 = _mm512_loadu_4_m128(src_ptr + begin3, src_ptr + begin7, src_ptr + begin11, src_ptr + begin15);
1010 ✗ data_4_8_12_16 = _mm512_loadu_4_m128(src_ptr + begin4, src_ptr + begin8, src_ptr + begin12, src_ptr + begin16);
1011 }
1012
1013 _MM_TRANSPOSE16_LANE4_PS(data_1_5_9_13, data_2_6_10_14, data_3_7_11_15, data_4_8_12_16);
1014
1015 // two sets, hint for the compiler to allow parallel fma's
1016 ✗ __m512 result_0 = _mm512_mul_ps(data_1_5_9_13, coef_1_5_9_13);
1017 ✗ __m512 result_1 = _mm512_mul_ps(data_2_6_10_14, coef_2_6_10_14);
1018 ✗ result_0 = _mm512_fmadd_ps(data_3_7_11_15, coef_3_7_11_15, result_0);
1019 ✗ result_1 = _mm512_fmadd_ps(data_4_8_12_16, coef_4_8_12_16, result_1);
1020 ✗ _mm512_stream_ps(dst_ptr, _mm512_add_ps(result_0, result_1));
1021
1022 ✗ dst_ptr += dst_pitch;
1023 ✗ src_ptr += src_pitch;
1024 } // y
1025
1026 // Move to the next set of coefficients for the next 16 output pixels
1027 ✗ current_coeff += filter_size * 16;
1028 ✗ };
1029
1030 // Process the 'safe zone' where direct full unaligned loads are acceptable.
1031 ✗ for (; x < width_safe_mod; x += PIXELS_AT_A_TIME)
1032 {
1033 ✗ do_h_float_core(std::false_type{}); // partial_load == false, use direct _mm512_loadu_ps
1034 }
1035
1036 // Process the potentially 'unsafe zone' near the image edge, using safe loading.
1037 ✗ for (; x < width; x += PIXELS_AT_A_TIME)
1038 {
1039 ✗ do_h_float_core(std::true_type{}); // partial_load == true, use the safer _mm512_load_partial_safe_4_m128
1040 }
1041 }
1042 ✗ }
1043
1044
1045 ✗ void resize_h_planar_float_avx512_transpose_vstripe_ks8(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) {
1046
1047 ✗ const int filter_size = program->filter_size; // aligned, practically the coeff table stride
1048
1049 ✗ src_pitch /= sizeof(float);
1050 ✗ dst_pitch /= sizeof(float);
1051
1052 ✗ float* src = (float*)src8;
1053 ✗ float* dst = (float*)dst8;
1054
1055 ✗ constexpr int PIXELS_AT_A_TIME = 16; // Process sixteen pixels in parallel using AVX512 (4x4 using m128 lanes)
1056
1057 // 'source_overread_beyond_targetx' indicates if the filter kernel can read beyond the target width.
1058 // Even if the filter alignment allows larger reads, our safety boundary for unaligned loads starts at 4 pixels back
1059 // from the target width, as we load 4 floats at once conceptually with our safe load.
1060 ✗ const int width_safe_mod = (program->safelimit_4_pixels.overread_possible ? program->safelimit_4_pixels.source_overread_beyond_targetx : width) / PIXELS_AT_A_TIME * PIXELS_AT_A_TIME;
1061
1062 // Preconditions:
1063 ✗ assert(program->filter_size_real <= 8);
1064 ✗ assert(program->target_size_alignment >= 16); // Must align for 16 pixel offsets
1065 ✗ assert(program->filter_size_alignment >= 8);
1066 assert(FRAME_ALIGN >= 64); // Adjusted for 16 pixels AviSynth+ default
1067
1068 // Vertical stripe alignment
1069 ✗ constexpr int STRIPE_ALIGN = 16; // this must be multiple of PIXELS_AT_A_TIME
1070
1071 ✗ int max_scanlines = program->max_scanlines / STRIPE_ALIGN * STRIPE_ALIGN;
1072
1073 ✗ if (max_scanlines < STRIPE_ALIGN) max_scanlines = STRIPE_ALIGN;
1074
1075 // --- outer loop: vertical stripes ---
1076 ✗ for (auto y_from = 0; y_from < height; y_from += max_scanlines) {
1077 ✗ size_t y_to = std::min(y_from + max_scanlines, height);
1078
1079 // Reset current_coeff for the start of the stripe
1080 ✗ const float* AVS_RESTRICT current_coeff = program->pixel_coefficient_float;
1081
1082 ✗ size_t x = 0;
1083
1084 // Lambda for 512-bit Core
1085 ✗ auto do_h_float_core = [&](auto partial_load) {
1086
1087 // Load to 8 coefficients per source pixel at once before the height loop.
1088 // Pre-loading and transposing coefficients keeps register usage efficient.
1089 // Assumes 'filter_size_aligned' is at least 8.
1090
1091 ✗ __m256 coef0 = _mm256_loadu_ps(current_coeff + filter_size * 0);
1092 ✗ __m256 coef1 = _mm256_loadu_ps(current_coeff + filter_size * 1);
1093 ✗ __m512 coef01 = _mm512_insert_2_m256(coef0, coef1);
1094
1095 ✗ __m256 coef2 = _mm256_loadu_ps(current_coeff + filter_size * 2);
1096 ✗ __m256 coef3 = _mm256_loadu_ps(current_coeff + filter_size * 3);
1097 ✗ __m512 coef23 = _mm512_insert_2_m256(coef2, coef3);
1098
1099 ✗ __m256 coef4 = _mm256_loadu_ps(current_coeff + filter_size * 4);
1100 ✗ __m256 coef5 = _mm256_loadu_ps(current_coeff + filter_size * 5);
1101 ✗ __m512 coef45 = _mm512_insert_2_m256(coef4, coef5);
1102
1103 ✗ __m256 coef6 = _mm256_loadu_ps(current_coeff + filter_size * 6);
1104 ✗ __m256 coef7 = _mm256_loadu_ps(current_coeff + filter_size * 7);
1105 ✗ __m512 coef67 = _mm512_insert_2_m256(coef6, coef7);
1106
1107 ✗ __m256 coef8 = _mm256_loadu_ps(current_coeff + filter_size * 8);
1108 ✗ __m256 coef9 = _mm256_loadu_ps(current_coeff + filter_size * 9);
1109 ✗ __m512 coef89 = _mm512_insert_2_m256(coef8, coef9);
1110
1111 ✗ __m256 coef10 = _mm256_loadu_ps(current_coeff + filter_size * 10);
1112 ✗ __m256 coef11 = _mm256_loadu_ps(current_coeff + filter_size * 11);
1113 ✗ __m512 coef1011 = _mm512_insert_2_m256(coef10, coef11);
1114
1115 ✗ __m256 coef12 = _mm256_loadu_ps(current_coeff + filter_size * 12);
1116 ✗ __m256 coef13 = _mm256_loadu_ps(current_coeff + filter_size * 13);
1117 ✗ __m512 coef1213 = _mm512_insert_2_m256(coef12, coef13);
1118
1119 ✗ __m256 coef14 = _mm256_loadu_ps(current_coeff + filter_size * 14);
1120 ✗ __m256 coef15 = _mm256_loadu_ps(current_coeff + filter_size * 15);
1121 ✗ __m512 coef1415 = _mm512_insert_2_m256(coef14, coef15);
1122
1123 // Before: 8x_m512 as 8x2x_m256 holding 16*8 coefficients each for 16 pixels
1124 // coef01: 0.0 ... 0.7 | 1.0 ... 1.7
1125 // coef23: 2.0 ... 2.7 | 3.0 ... 3.7
1126 // coef45: 4.0 ... 4.7 | 5.0 ... 5.7
1127 // coef67: 6.0 ... 6.7 | 7.0 ... 7.7
1128 // coef89: 8.0 ... 8.7 | 9.0 ... 9.7
1129 // coef1011: A.0 ... A.7 | B.0 ... B.7
1130 // coef1213: C.0 ... C.7 | D.0 ... D.7
1131 // coef1415: E.0 ... E.7 | F.0 ... F.7
1132
1133 _MM_TRANSPOSE8x16_PS(coef01, coef23, coef45, coef67, coef89, coef1011, coef1213, coef1415);
1134
1135 // After: 8x _m512 holding 16 coefficients, each for 16 pixels
1136 // result0, as old_coef01 : 0.0 .. 7.0 | 8.0 .. F.0
1137 // result1, as old_coef23 : 0.1 .. 7.1 | 8.1 .. F.1
1138 // result2, as old_coef45 : 0.2 .. 7.2 | 8.2 .. F.2
1139 // result3, as old_coef67 : 0.3 .. 7.3 | 8.3 .. F.3
1140 // result4, as old_coef89 : 0.4 .. 7.4 | 8.4 .. F.4
1141 // result5, as old_coef1011: 0.5 .. 7.5 | 8.5 .. F.5
1142 // result6, as old_coef1213: 0.6 .. 7.6 | 8.6 .. F.6
1143 // result7, as old_coef1415: 0.7 .. 7.7 | 8.7 .. F.7
1144
1145 // Pixel offsets for the current target x-positions.
1146 // Even for x >= width, these offsets are guaranteed to be within the allocated 'target_size_alignment'.
1147 ✗ const int begin1 = program->pixel_offset[x + 0];
1148 ✗ const int begin2 = program->pixel_offset[x + 1];
1149 ✗ const int begin3 = program->pixel_offset[x + 2];
1150 ✗ const int begin4 = program->pixel_offset[x + 3];
1151 ✗ const int begin5 = program->pixel_offset[x + 4];
1152 ✗ const int begin6 = program->pixel_offset[x + 5];
1153 ✗ const int begin7 = program->pixel_offset[x + 6];
1154 ✗ const int begin8 = program->pixel_offset[x + 7];
1155 ✗ const int begin9 = program->pixel_offset[x + 8];
1156 ✗ const int begin10 = program->pixel_offset[x + 9];
1157 ✗ const int begin11 = program->pixel_offset[x + 10];
1158 ✗ const int begin12 = program->pixel_offset[x + 11];
1159 ✗ const int begin13 = program->pixel_offset[x + 12];
1160 ✗ const int begin14 = program->pixel_offset[x + 13];
1161 ✗ const int begin15 = program->pixel_offset[x + 14];
1162 ✗ const int begin16 = program->pixel_offset[x + 15];
1163
1164 ✗ int y = y_from;
1165
1166 // Calculate pointers ONCE before the inner loop (Optimization from AVX2 version)
1167 ✗ float* AVS_RESTRICT dst_ptr = dst + y * dst_pitch + x;
1168 ✗ const float* src_ptr = src + y * src_pitch;
1169
1170 // only needed for partial load
1171 ✗ const int Nmod8 = program->filter_size_real % 8;
1172 ✗ const int floats_to_load = Nmod8 == 0 ? 8 : Nmod8;
1173
1174 ✗ for (; y < y_to; ++y) {
1175
1176 __m512 data01, data23, data45, data67, data89, data1011, data1213, data1415;
1177
1178 if constexpr (partial_load) {
1179 // In the potentially unsafe zone (near the right edge of the image), we use a safe loading function
1180 // to prevent reading beyond the allocated source scanline.
1181 ✗ data01 = _mm512_load_partial_safe_2_m256(src_ptr + begin1, src_ptr + begin2, floats_to_load);
1182 ✗ data23 = _mm512_load_partial_safe_2_m256(src_ptr + begin3, src_ptr + begin4, floats_to_load);
1183 ✗ data45 = _mm512_load_partial_safe_2_m256(src_ptr + begin5, src_ptr + begin6, floats_to_load);
1184 ✗ data67 = _mm512_load_partial_safe_2_m256(src_ptr + begin7, src_ptr + begin8, floats_to_load);
1185 ✗ data89 = _mm512_load_partial_safe_2_m256(src_ptr + begin9, src_ptr + begin10, floats_to_load);
1186 ✗ data1011 = _mm512_load_partial_safe_2_m256(src_ptr + begin11, src_ptr + begin12, floats_to_load);
1187 ✗ data1213 = _mm512_load_partial_safe_2_m256(src_ptr + begin13, src_ptr + begin14, floats_to_load);
1188 ✗ data1415 = _mm512_load_partial_safe_2_m256(src_ptr + begin15, src_ptr + begin16, floats_to_load);
1189 }
1190 else {
1191 // In the safe zone, we can directly load 8 source pixels at a time for each of the 16 lanes.
1192 ✗ data01 = _mm512_insert_2_m256(_mm256_loadu_ps(src_ptr + begin1), _mm256_loadu_ps(src_ptr + begin2));
1193 ✗ data23 = _mm512_insert_2_m256(_mm256_loadu_ps(src_ptr + begin3), _mm256_loadu_ps(src_ptr + begin4));
1194 ✗ data45 = _mm512_insert_2_m256(_mm256_loadu_ps(src_ptr + begin5), _mm256_loadu_ps(src_ptr + begin6));
1195 ✗ data67 = _mm512_insert_2_m256(_mm256_loadu_ps(src_ptr + begin7), _mm256_loadu_ps(src_ptr + begin8));
1196 ✗ data89 = _mm512_insert_2_m256(_mm256_loadu_ps(src_ptr + begin9), _mm256_loadu_ps(src_ptr + begin10));
1197 ✗ data1011 = _mm512_insert_2_m256(_mm256_loadu_ps(src_ptr + begin11), _mm256_loadu_ps(src_ptr + begin12));
1198 ✗ data1213 = _mm512_insert_2_m256(_mm256_loadu_ps(src_ptr + begin13), _mm256_loadu_ps(src_ptr + begin14));
1199 ✗ data1415 = _mm512_insert_2_m256(_mm256_loadu_ps(src_ptr + begin15), _mm256_loadu_ps(src_ptr + begin16));
1200 }
1201
1202 _MM_TRANSPOSE8x16_PS(data01, data23, data45, data67, data89, data1011, data1213, data1415);
1203
1204 // two sets, hint for the compiler to allow parallel fma's
1205 ✗ __m512 result_0 = _mm512_mul_ps(data01, coef01);
1206 ✗ __m512 result_1 = _mm512_mul_ps(data23, coef23);
1207 ✗ result_0 = _mm512_fmadd_ps(data45, coef45, result_0);
1208 ✗ result_1 = _mm512_fmadd_ps(data67, coef67, result_1);
1209 ✗ result_0 = _mm512_fmadd_ps(data89, coef89, result_0);
1210 ✗ result_1 = _mm512_fmadd_ps(data1011, coef1011, result_1);
1211 ✗ result_0 = _mm512_fmadd_ps(data1213, coef1213, result_0);
1212 ✗ result_1 = _mm512_fmadd_ps(data1415, coef1415, result_1);
1213
1214 ✗ _mm512_stream_ps(dst_ptr, _mm512_add_ps(result_0, result_1));
1215
1216 ✗ dst_ptr += dst_pitch;
1217 ✗ src_ptr += src_pitch;
1218 } // y
1219
1220 // Move to the next set of coefficients for the next 16 output pixels
1221 ✗ current_coeff += filter_size * 16;
1222 ✗ }; // lambda
1223
1224 // Process the 'safe zone' where direct full unaligned loads are acceptable.
1225 ✗ for (; x < width_safe_mod; x += PIXELS_AT_A_TIME)
1226 {
1227 ✗ do_h_float_core(std::false_type{}); // partial_load == false, use direct _mm512_loadu_ps
1228 }
1229
1230 // Process the potentially 'unsafe zone' near the image edge, using safe loading.
1231 ✗ for (; x < width; x += PIXELS_AT_A_TIME)
1232 {
1233 ✗ do_h_float_core(std::true_type{}); // partial_load == true, use the safer _mm512_load_partial_safe_2_m256
1234 }
1235 }
1236 ✗ }
1237
1238 // Similar to AVX2 resize_h_planar_float_avx512_permutex_vstripe_ks4
1239 // but doing 16 pixels at a time with AVX512 permutex instructions.
1240 ✗ void resize_h_planar_float_avx512_permutex_vstripe_ks4(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel)
1241 {
1242 ✗ const int filter_size = program->filter_size; // aligned, practically the coeff table stride
1243
1244 ✗ src_pitch /= sizeof(float);
1245 ✗ dst_pitch /= sizeof(float);
1246
1247 ✗ float* src = (float*)src8;
1248 ✗ float* dst = (float*)dst8;
1249
1250 ✗ constexpr int PIXELS_AT_A_TIME = 16; // Process sixteen pixels in parallel using AVX512 (4x4 using m128 lanes)
1251
1252 // 'source_overread_beyond_targetx' indicates if the filter kernel can read beyond the target width.
1253 ✗ const int width_safe_mod = (program->safelimit_4_pixels.overread_possible ? program->safelimit_4_pixels.source_overread_beyond_targetx : width) / PIXELS_AT_A_TIME * PIXELS_AT_A_TIME;
1254
1255 // Preconditions:
1256 ✗ assert(program->filter_size_real <= 4); // We preload all relevant coefficients (up to 4) before the height loop.
1257
1258 // 'target_size_alignment' ensures we can safely access coefficients using offsets like
1259 // 'filter_size * 15' when processing 16 H pixels at a time
1260 ✗ assert(program->target_size_alignment >= 16); // Adjusted for 16 pixels
1261 assert(FRAME_ALIGN >= 64); // Adjusted for 16 pixels AviSynth+ default
1262
1263 // Ensure that coefficient loading beyond the valid target size is safe for 4x4 float loads.
1264 ✗ assert(program->filter_size_alignment >= 4);
1265
1266 ✗ const int max_scanlines = program->max_scanlines;
1267
1268 // Vertical stripe loop for L2 cache optimization
1269 ✗ for (int y_from = 0; y_from < height; y_from += max_scanlines)
1270 {
1271 ✗ int y_to = std::min(y_from + max_scanlines, height);
1272
1273 // Reset current_coeff for the start of the stripe (points to start of row's coeffs)
1274 ✗ const float* AVS_RESTRICT current_coeff = (const float* AVS_RESTRICT)program->pixel_coefficient_float;
1275
1276 ✗ int x = 0;
1277
1278 // Lambda to handle both safe (fast) and unsafe (masked/partial) loading paths
1279 ✗ auto do_h_float_core = [&](auto partial_load) {
1280
1281 // prepare coefs in transposed V-form
1282 // We load 4 coefficients sets (for 4 pixels) into 4 lanes of a zmm register
1283 ✗ __m512 coef_r0 = _mm512_load_4_m128(current_coeff + filter_size * 0, current_coeff + filter_size * 4, current_coeff + filter_size * 8, current_coeff + filter_size * 12);
1284 ✗ __m512 coef_r1 = _mm512_load_4_m128(current_coeff + filter_size * 1, current_coeff + filter_size * 5, current_coeff + filter_size * 9, current_coeff + filter_size * 13);
1285 ✗ __m512 coef_r2 = _mm512_load_4_m128(current_coeff + filter_size * 2, current_coeff + filter_size * 6, current_coeff + filter_size * 10, current_coeff + filter_size * 14);
1286 ✗ __m512 coef_r3 = _mm512_load_4_m128(current_coeff + filter_size * 3, current_coeff + filter_size * 7, current_coeff + filter_size * 11, current_coeff + filter_size * 15);
1287
1288 _MM_TRANSPOSE16_LANE4_PS(coef_r0, coef_r1, coef_r2, coef_r3);
1289
1290 // convert resampling program in H-form into permuting indexes for src transposition in V-form
1291 ✗ __m512i perm_0 = _mm512_loadu_si512((__m512i*)(&program->pixel_offset[x]));
1292 ✗ int iStart = program->pixel_offset[x];
1293 ✗ perm_0 = _mm512_sub_epi32(perm_0, _mm512_set1_epi32(iStart));
1294 /* like this:
1295 __m512i perm_0 = _mm512_set_epi32(
1296 program->pixel_offset[x + 15] - iStart,
1297 ...
1298 program->pixel_offset[x + 0] - iStart);
1299 */
1300
1301 // Taps are contiguous (0, 1, 2, 3), so we increment perm indexes by 1.
1302 ✗ __m512i one_epi32 = _mm512_set1_epi32(1);
1303 ✗ __m512i perm_1 = _mm512_add_epi32(perm_0, one_epi32);
1304 ✗ __m512i perm_2 = _mm512_add_epi32(perm_1, one_epi32);
1305 ✗ __m512i perm_3 = _mm512_add_epi32(perm_2, one_epi32);
1306
1307 ✗ float* AVS_RESTRICT dst_ptr = dst + x + y_from * dst_pitch;
1308 ✗ const float* src_ptr = src + iStart + y_from * src_pitch; // all permute offsets relative to this start offset
1309
1310 // Calculate remaining pixels for bounds checking in partial_load mode
1311 ✗ const int remaining = program->source_size - iStart;
1312
1313 ✗ for (int y = y_from; y < y_to; y++)
1314 {
1315 __m512 data_src, data_src2;
1316
1317 if constexpr (partial_load) {
1318 // Safe masked loads for the image edge
1319 // Load first 16 floats
1320 ✗ int rem1 = std::max(0, std::min(16, remaining));
1321 ✗ __mmask16 k1 = (1U << rem1) - 1;
1322 ✗ data_src = _mm512_maskz_loadu_ps(k1, src_ptr);
1323
1324 // Load next 16 floats (offset by 16)
1325 ✗ int rem2 = std::max(0, std::min(16, remaining - 16));
1326 ✗ __mmask16 k2 = (1U << rem2) - 1;
1327 ✗ data_src2 = _mm512_maskz_loadu_ps(k2, src_ptr + 16);
1328 }
1329 else {
1330 // Fast unaligned loads for the safe zone
1331 ✗ data_src = _mm512_loadu_ps(src_ptr);
1332 ✗ data_src2 = _mm512_loadu_ps(src_ptr + 16);
1333 }
1334
1335 ✗ __m512 data_0 = _mm512_permutex2var_ps(data_src, perm_0, data_src2);
1336 ✗ __m512 data_1 = _mm512_permutex2var_ps(data_src, perm_1, data_src2);
1337 ✗ __m512 data_2 = _mm512_permutex2var_ps(data_src, perm_2, data_src2);
1338 ✗ __m512 data_3 = _mm512_permutex2var_ps(data_src, perm_3, data_src2);
1339
1340 ✗ __m512 result0 = _mm512_mul_ps(data_0, coef_r0);
1341 ✗ __m512 result1 = _mm512_mul_ps(data_2, coef_r2);
1342
1343 ✗ result0 = _mm512_fmadd_ps(data_1, coef_r1, result0);
1344 ✗ result1 = _mm512_fmadd_ps(data_3, coef_r3, result1);
1345
1346 ✗ _mm512_stream_ps(dst_ptr, _mm512_add_ps(result0, result1));
1347
1348 ✗ dst_ptr += dst_pitch;
1349 ✗ src_ptr += src_pitch;
1350 }
1351
1352 ✗ current_coeff += filter_size * 16;
1353 ✗ };
1354
1355 // Process the 'safe zone' where direct full unaligned loads are acceptable.
1356 ✗ for (; x < width_safe_mod; x += PIXELS_AT_A_TIME)
1357 {
1358 ✗ do_h_float_core(std::false_type{});
1359 }
1360
1361 // Process the potentially 'unsafe zone' near the image edge, using safe masked loading.
1362 ✗ for (; x < width; x += PIXELS_AT_A_TIME)
1363 {
1364 ✗ do_h_float_core(std::true_type{});
1365 }
1366 }
1367 ✗ }
1368
1369 // Similar to resize_h_planar_float_avx512_permutex_vstripe_ks4 but for kernel size up to 8
1370 // 16 target pixels at a time with AVX512 permutex instructions.
1371 ✗ void resize_h_planar_float_avx512_permutex_vstripe_ks8(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel)
1372 {
1373 ✗ const int filter_size = program->filter_size; // aligned, practically the coeff table stride
1374
1375 ✗ src_pitch /= sizeof(float);
1376 ✗ dst_pitch /= sizeof(float);
1377
1378 ✗ float* src = (float*)src8;
1379 ✗ float* dst = (float*)dst8;
1380
1381 ✗ constexpr int PIXELS_AT_A_TIME = 16; // Process sixteen pixels in parallel using AVX512 (4x4 using m128 lanes)
1382
1383 // 'source_overread_beyond_targetx' indicates if the filter kernel can read beyond the target width.
1384 ✗ const int width_safe_mod = (program->safelimit_8_pixels.overread_possible ? program->safelimit_8_pixels.source_overread_beyond_targetx : width) / PIXELS_AT_A_TIME * PIXELS_AT_A_TIME;
1385
1386 // Preconditions:
1387 ✗ assert(program->filter_size_real <= 8); // We preload all relevant coefficients (up to 8) before the height loop.
1388
1389 // 'target_size_alignment' ensures we can safely access coefficients using offsets like
1390 // 'filter_size * 15' when processing 16 H pixels at a time
1391 ✗ assert(program->target_size_alignment >= 16); // Adjusted for 16 pixels
1392 assert(FRAME_ALIGN >= 64); // Adjusted for 16 pixels AviSynth+ default
1393
1394 // Ensure that coefficient loading beyond the valid target size is safe for 4x8 float loads.
1395 // n/a: load once 8. align 4 is the same as 8 for the purpose.
1396 ✗ assert(program->filter_size_alignment >= 8);
1397
1398 ✗ const int max_scanlines = program->max_scanlines;
1399
1400 // Vertical stripe loop for L2 cache optimization
1401 ✗ for (int y_from = 0; y_from < height; y_from += max_scanlines)
1402 {
1403 ✗ int y_to = std::min(y_from + max_scanlines, height);
1404
1405 // Reset current_coeff for the start of the stripe (points to start of row's coeffs)
1406 ✗ const float* AVS_RESTRICT current_coeff = (const float* AVS_RESTRICT)program->pixel_coefficient_float;
1407
1408 ✗ int x = 0;
1409
1410 // Lambda to handle both safe (fast) and unsafe (masked/partial) loading paths
1411 ✗ auto do_h_float_core = [&](auto partial_load) {
1412
1413 // prepare coefs in transposed V-form
1414 // 4 coefficients sets (for 4 pixels) into 4 lanes of a zmm register
1415 ✗ __m512 coef_r0 = _mm512_load_4_m128(current_coeff + filter_size * 0, current_coeff + filter_size * 4, current_coeff + filter_size * 8, current_coeff + filter_size * 12);
1416 ✗ __m512 coef_r1 = _mm512_load_4_m128(current_coeff + filter_size * 1, current_coeff + filter_size * 5, current_coeff + filter_size * 9, current_coeff + filter_size * 13);
1417 ✗ __m512 coef_r2 = _mm512_load_4_m128(current_coeff + filter_size * 2, current_coeff + filter_size * 6, current_coeff + filter_size * 10, current_coeff + filter_size * 14);
1418 ✗ __m512 coef_r3 = _mm512_load_4_m128(current_coeff + filter_size * 3, current_coeff + filter_size * 7, current_coeff + filter_size * 11, current_coeff + filter_size * 15);
1419
1420 ✗ const float* AVS_RESTRICT current_coeff_47 = current_coeff + 4;
1421
1422 ✗ __m512 coef_r4 = _mm512_load_4_m128(current_coeff_47 + filter_size * 0, current_coeff_47 + filter_size * 4, current_coeff_47 + filter_size * 8, current_coeff_47 + filter_size * 12);
1423 ✗ __m512 coef_r5 = _mm512_load_4_m128(current_coeff_47 + filter_size * 1, current_coeff_47 + filter_size * 5, current_coeff_47 + filter_size * 9, current_coeff_47 + filter_size * 13);
1424 ✗ __m512 coef_r6 = _mm512_load_4_m128(current_coeff_47 + filter_size * 2, current_coeff_47 + filter_size * 6, current_coeff_47 + filter_size * 10, current_coeff_47 + filter_size * 14);
1425 ✗ __m512 coef_r7 = _mm512_load_4_m128(current_coeff_47 + filter_size * 3, current_coeff_47 + filter_size * 7, current_coeff_47 + filter_size * 11, current_coeff_47 + filter_size * 15);
1426
1427 _MM_TRANSPOSE16_LANE4_PS(coef_r0, coef_r1, coef_r2, coef_r3);
1428 _MM_TRANSPOSE16_LANE4_PS(coef_r4, coef_r5, coef_r6, coef_r7);
1429
1430 // convert resampling program in H-form into permuting indexes for src transposition in V-form
1431 ✗ __m512i perm_0 = _mm512_loadu_si512((__m512i*)(&program->pixel_offset[x]));
1432 ✗ int iStart = program->pixel_offset[x + 0];
1433 ✗ perm_0 = _mm512_sub_epi32(perm_0, _mm512_set1_epi32(iStart));
1434 /* like this:
1435 __m512i perm_0 = _mm512_set_epi32(
1436 program->pixel_offset[x + 15] - iStart,
1437 ...
1438 program->pixel_offset[x + 0] - iStart);
1439 */
1440
1441 // Taps are contiguous (0, 1, 2, 3 .. 7), so we increment perm indexes by 1.
1442 ✗ __m512i one_epi32 = _mm512_set1_epi32(1);
1443 ✗ __m512i perm_1 = _mm512_add_epi32(perm_0, one_epi32);
1444 ✗ __m512i perm_2 = _mm512_add_epi32(perm_1, one_epi32);
1445 ✗ __m512i perm_3 = _mm512_add_epi32(perm_2, one_epi32);
1446 ✗ __m512i perm_4 = _mm512_add_epi32(perm_3, one_epi32);
1447 ✗ __m512i perm_5 = _mm512_add_epi32(perm_4, one_epi32);
1448 ✗ __m512i perm_6 = _mm512_add_epi32(perm_5, one_epi32);
1449 ✗ __m512i perm_7 = _mm512_add_epi32(perm_6, one_epi32);
1450
1451 ✗ float* AVS_RESTRICT dst_ptr = dst + x + y_from * dst_pitch;
1452 ✗ const float* src_ptr = src + iStart + y_from * src_pitch; // all permute offsets relative to this start offset
1453
1454 // Calculate remaining pixels for bounds checking in partial_load mode
1455 ✗ const int remaining = program->source_size - iStart;
1456
1457 ✗ for (int y = y_from; y < y_to; y++)
1458 {
1459 __m512 data_src, data_src2;
1460
1461 if constexpr (partial_load) {
1462 // Safe masked loads for the image edge
1463 // Load first 16 floats
1464 ✗ int rem1 = std::max(0, std::min(16, remaining));
1465 ✗ __mmask16 k1 = (1U << rem1) - 1;
1466 ✗ data_src = _mm512_maskz_loadu_ps(k1, src_ptr);
1467
1468 // Load next 16 floats (offset by 16)
1469 ✗ int rem2 = std::max(0, std::min(16, remaining - 16));
1470 ✗ __mmask16 k2 = (1U << rem2) - 1;
1471 ✗ data_src2 = _mm512_maskz_loadu_ps(k2, src_ptr + 16);
1472 }
1473 else {
1474 // Fast unaligned loads for the safe zone
1475 ✗ data_src = _mm512_loadu_ps(src_ptr);
1476 ✗ data_src2 = _mm512_loadu_ps(src_ptr + 16);
1477 }
1478
1479 ✗ __m512 data_0 = _mm512_permutex2var_ps(data_src, perm_0, data_src2);
1480 ✗ __m512 data_1 = _mm512_permutex2var_ps(data_src, perm_1, data_src2);
1481 ✗ __m512 data_2 = _mm512_permutex2var_ps(data_src, perm_2, data_src2);
1482 ✗ __m512 data_3 = _mm512_permutex2var_ps(data_src, perm_3, data_src2);
1483 ✗ __m512 data_4 = _mm512_permutex2var_ps(data_src, perm_4, data_src2);
1484 ✗ __m512 data_5 = _mm512_permutex2var_ps(data_src, perm_5, data_src2);
1485 ✗ __m512 data_6 = _mm512_permutex2var_ps(data_src, perm_6, data_src2);
1486 ✗ __m512 data_7 = _mm512_permutex2var_ps(data_src, perm_7, data_src2);
1487
1488 ✗ __m512 result0 = _mm512_mul_ps(data_0, coef_r0);
1489 ✗ __m512 result1 = _mm512_mul_ps(data_2, coef_r2);
1490 ✗ __m512 result2 = _mm512_mul_ps(data_4, coef_r4);
1491 ✗ __m512 result3 = _mm512_mul_ps(data_6, coef_r6);
1492
1493 ✗ result0 = _mm512_fmadd_ps(data_1, coef_r1, result0);
1494 ✗ result1 = _mm512_fmadd_ps(data_3, coef_r3, result1);
1495 ✗ result2 = _mm512_fmadd_ps(data_5, coef_r5, result2);
1496 ✗ result3 = _mm512_fmadd_ps(data_7, coef_r7, result3);
1497
1498 ✗ __m512 result01 = _mm512_add_ps(result0, result1);
1499 ✗ __m512 result23 = _mm512_add_ps(result2, result3);
1500 ✗ __m512 result0123 = _mm512_add_ps(result01, result23);
1501 _mm512_stream_ps(dst_ptr, result0123);
1502
1503 ✗ dst_ptr += dst_pitch;
1504 ✗ src_ptr += src_pitch;
1505 }
1506
1507 ✗ current_coeff += filter_size * 16;
1508 ✗ };
1509
1510 // Process the 'safe zone' where direct full unaligned loads are acceptable.
1511 ✗ for (; x < width_safe_mod; x += PIXELS_AT_A_TIME)
1512 {
1513 ✗ do_h_float_core(std::false_type{});
1514 }
1515
1516 // Process the potentially 'unsafe zone' near the image edge, using safe masked loading.
1517 ✗ for (; x < width; x += PIXELS_AT_A_TIME)
1518 {
1519 ✗ do_h_float_core(std::true_type{});
1520 }
1521 }
1522 ✗ }
1523
1524
1525 // Similar to resize_h_planar_float_avx512_permutex_vstripe_ks4 but for kernel size up to
1526 // 16 target pixels at a time with AVX512 permutex instructions.
1527 // Uses 2 groups of 8 output samples processing by independednd gathering 2x32 contigous groups of sources to support more downscale ratios
1528 ✗ void resize_h_planar_float_avx512_permutex_vstripe_2s8_ks8(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel)
1529 {
1530 ✗ const int filter_size = program->filter_size; // aligned, practically the coeff table stride
1531
1532 ✗ src_pitch /= sizeof(float);
1533 ✗ dst_pitch /= sizeof(float);
1534
1535 ✗ float* src = (float*)src8;
1536 ✗ float* dst = (float*)dst8;
1537
1538 ✗ constexpr int PIXELS_AT_A_TIME = 16; // Process 2 groups of 8 pixels in parallel using AVX512 with wider source gathering for downsample
1539
1540 // 'source_overread_beyond_targetx' indicates if the filter kernel can read beyond the target width.
1541 ✗ const int width_safe_mod = (program->safelimit_8_pixels.overread_possible ? program->safelimit_8_pixels.source_overread_beyond_targetx : width) / PIXELS_AT_A_TIME * PIXELS_AT_A_TIME;
1542
1543 // Preconditions:
1544 ✗ assert(program->filter_size_real <= 8); // We preload all relevant coefficients (up to 8) before the height loop.
1545
1546 // 'target_size_alignment' ensures we can safely access coefficients using offsets like
1547 // 'filter_size * 15' when processing 16 H pixels at a time
1548 ✗ assert(program->target_size_alignment >= 16); // Adjusted for 16 pixels
1549 assert(FRAME_ALIGN >= 64); // Adjusted for 16 pixels AviSynth+ default
1550
1551 // Ensure that coefficient loading beyond the valid target size is safe for 4x8 float loads.
1552 ✗ assert(program->filter_size_alignment >= 8);
1553 // n/a: load once 8. align 4 is the same as 8 for the purpose.
1554
1555 ✗ const int max_scanlines = program->max_scanlines;
1556
1557 // Vertical stripe loop for L2 cache optimization
1558 ✗ for (int y_from = 0; y_from < height; y_from += max_scanlines)
1559 {
1560 ✗ int y_to = std::min(y_from + max_scanlines, height);
1561
1562 // Reset current_coeff for the start of the stripe (points to start of row's coeffs)
1563 ✗ const float* AVS_RESTRICT current_coeff = (const float* AVS_RESTRICT)program->pixel_coefficient_float;
1564
1565 ✗ int x = 0;
1566
1567 // Lambda to handle both safe (fast) and unsafe (masked/partial) loading paths
1568 ✗ auto do_h_float_core = [&](auto partial_load) {
1569
1570 // prepare coefs in transposed V-form
1571 // 4 coefficients sets (for 4 pixels) into 4 lanes of a zmm register
1572 ✗ __m512 coef_r0 = _mm512_load_4_m128(current_coeff + filter_size * 0, current_coeff + filter_size * 4, current_coeff + filter_size * 8, current_coeff + filter_size * 12);
1573 ✗ __m512 coef_r1 = _mm512_load_4_m128(current_coeff + filter_size * 1, current_coeff + filter_size * 5, current_coeff + filter_size * 9, current_coeff + filter_size * 13);
1574 ✗ __m512 coef_r2 = _mm512_load_4_m128(current_coeff + filter_size * 2, current_coeff + filter_size * 6, current_coeff + filter_size * 10, current_coeff + filter_size * 14);
1575 ✗ __m512 coef_r3 = _mm512_load_4_m128(current_coeff + filter_size * 3, current_coeff + filter_size * 7, current_coeff + filter_size * 11, current_coeff + filter_size * 15);
1576
1577 ✗ const float* AVS_RESTRICT current_coeff_47 = current_coeff + 4;
1578
1579 ✗ __m512 coef_r4 = _mm512_load_4_m128(current_coeff_47 + filter_size * 0, current_coeff_47 + filter_size * 4, current_coeff_47 + filter_size * 8, current_coeff_47 + filter_size * 12);
1580 ✗ __m512 coef_r5 = _mm512_load_4_m128(current_coeff_47 + filter_size * 1, current_coeff_47 + filter_size * 5, current_coeff_47 + filter_size * 9, current_coeff_47 + filter_size * 13);
1581 ✗ __m512 coef_r6 = _mm512_load_4_m128(current_coeff_47 + filter_size * 2, current_coeff_47 + filter_size * 6, current_coeff_47 + filter_size * 10, current_coeff_47 + filter_size * 14);
1582 ✗ __m512 coef_r7 = _mm512_load_4_m128(current_coeff_47 + filter_size * 3, current_coeff_47 + filter_size * 7, current_coeff_47 + filter_size * 11, current_coeff_47 + filter_size * 15);
1583
1584 _MM_TRANSPOSE16_LANE4_PS(coef_r0, coef_r1, coef_r2, coef_r3);
1585 _MM_TRANSPOSE16_LANE4_PS(coef_r4, coef_r5, coef_r6, coef_r7);
1586
1587 // convert resampling program in H-form into permuting indexes for src transposition in V-form
1588 // shorter SIMD-way - single memory load (hacky SIMD load from int vector ?)
1589 ✗ __m512i perm_0_low8 = _mm512_loadu_si512((__m512i*)(&program->pixel_offset[x]));
1590 ✗ int iStart_low8 = program->pixel_offset[x];
1591 ✗ perm_0_low8 = _mm512_sub_epi32(perm_0_low8, _mm512_set1_epi32(iStart_low8)); // vpbroadcastd zmm, r32
1592
1593 ✗ __m512i perm_0_high8 = _mm512_loadu_si512((__m512i*)(&program->pixel_offset[x + 8]));
1594 ✗ int iStart_high8 = program->pixel_offset[x + 8];
1595 ✗ perm_0_high8 = _mm512_sub_epi32(perm_0_high8, _mm512_set1_epi32(iStart_high8)); // vpbroadcastd zmm, r32
1596 ✗ perm_0_high8 = _mm512_inserti64x4(perm_0_high8, _mm512_castsi512_si256(perm_0_high8), 1);// shift low 8 epi32 to high 8
1597
1598 ✗ const __mmask16 k_high8 = 0xFF00;
1599 ✗ __m512i perm_0 = _mm512_mask_blend_epi32(k_high8, perm_0_low8, perm_0_high8);
1600
1601 ✗ float* AVS_RESTRICT dst_ptr = dst + x + y_from * dst_pitch;
1602 ✗ const float* src_ptr_low8 = src + iStart_low8 + y_from * src_pitch; // all permute offsets relative to this start offset
1603 ✗ const float* src_ptr_high8 = src + iStart_high8 + y_from * src_pitch; // all permute offsets relative to this start offset
1604
1605 // Calculate remaining pixels for bounds checking in partial_load mode
1606 ✗ const int remaining_low8 = program->source_size - iStart_low8;
1607 ✗ const int remaining_high8 = program->source_size - iStart_high8;
1608
1609 ✗ int rem1_low8 = std::max(0, std::min(16, remaining_low8));
1610 ✗ __mmask16 k1_low8 = (1U << rem1_low8) - 1;
1611 ✗ int rem2 = std::max(0, std::min(16, remaining_low8 - 16));
1612 ✗ __mmask16 k2_low8 = (1U << rem2) - 1;
1613 ✗ int rem1_high8 = std::max(0, std::min(16, remaining_high8));
1614 ✗ __mmask16 k1_high8 = (1U << rem1_high8) - 1;
1615 ✗ int rem2_high8 = std::max(0, std::min(16, remaining_high8 - 16));
1616 ✗ __mmask16 k2_high8 = (1U << rem2_high8) - 1;
1617
1618 // Taps are contiguous (0, 1, 2, 3 .. 7), so we increment perm indexes by 1.
1619 ✗ const __m512i one_epi32 = _mm512_set1_epi32(1);
1620 ✗ const __m512i perm_1 = _mm512_add_epi32(perm_0, one_epi32);
1621 ✗ const __m512i perm_2 = _mm512_add_epi32(perm_1, one_epi32);
1622 ✗ const __m512i perm_3 = _mm512_add_epi32(perm_2, one_epi32);
1623 ✗ const __m512i perm_4 = _mm512_add_epi32(perm_3, one_epi32);
1624 ✗ const __m512i perm_5 = _mm512_add_epi32(perm_4, one_epi32);
1625 ✗ const __m512i perm_6 = _mm512_add_epi32(perm_5, one_epi32);
1626 ✗ const __m512i perm_7 = _mm512_add_epi32(perm_6, one_epi32);
1627
1628 ✗ for (int y = y_from; y < y_to; y++)
1629 {
1630 __m512 data_src_low8, data_src2_low8;
1631 __m512 data_src_high8, data_src2_high8;
1632
1633 if constexpr (partial_load) {
1634 // Safe masked loads for the image edge
1635 // Load first 16 floats
1636 ✗ data_src_low8 = _mm512_maskz_loadu_ps(k1_low8, src_ptr_low8);
1637 // Load next 16 floats (offset by 16)
1638 ✗ data_src2_low8 = _mm512_maskz_loadu_ps(k2_low8, src_ptr_low8 + 16);
1639
1640 // high8
1641 // Safe masked loads for the image edge
1642 // Load first 16 floats
1643 ✗ data_src_high8 = _mm512_maskz_loadu_ps(k1_high8, src_ptr_high8);
1644 // Load next 16 floats (offset by 16)
1645 ✗ data_src2_high8 = _mm512_maskz_loadu_ps(k2_high8, src_ptr_high8 + 16);
1646 }
1647 else {
1648 // Fast unaligned loads for the safe zone
1649 ✗ data_src_low8 = _mm512_loadu_ps(src_ptr_low8);
1650 ✗ data_src2_low8 = _mm512_loadu_ps(src_ptr_low8 + 16);
1651 ✗ data_src_high8 = _mm512_loadu_ps(src_ptr_high8);
1652 ✗ data_src2_high8 = _mm512_loadu_ps(src_ptr_high8 + 16);
1653 }
1654
1655 /* __m512 data_0 = _mm512_permutex2var_ps(data_src_low8, perm_0, data_src2_low8);
1656 __m512 data_0_high8 = _mm512_permutex2var_ps(data_src_high8, perm_0, data_src2_high8);
1657 data_0 = _mm512_mask_blend_ps(k_high8, data_0, data_0_high8);*/
1658 ✗ __m512 data_0 = _mm512_mask_blend_ps(k_high8, _mm512_permutex2var_ps(data_src_low8, perm_0, data_src2_low8), _mm512_permutex2var_ps(data_src_high8, perm_0, data_src2_high8));
1659 ✗ __m512 data_1 = _mm512_mask_blend_ps(k_high8, _mm512_permutex2var_ps(data_src_low8, perm_1, data_src2_low8), _mm512_permutex2var_ps(data_src_high8, perm_1, data_src2_high8));
1660 ✗ __m512 data_2 = _mm512_mask_blend_ps(k_high8, _mm512_permutex2var_ps(data_src_low8, perm_2, data_src2_low8), _mm512_permutex2var_ps(data_src_high8, perm_2, data_src2_high8));
1661 ✗ __m512 data_3 = _mm512_mask_blend_ps(k_high8, _mm512_permutex2var_ps(data_src_low8, perm_3, data_src2_low8), _mm512_permutex2var_ps(data_src_high8, perm_3, data_src2_high8));
1662 ✗ __m512 data_4 = _mm512_mask_blend_ps(k_high8, _mm512_permutex2var_ps(data_src_low8, perm_4, data_src2_low8), _mm512_permutex2var_ps(data_src_high8, perm_4, data_src2_high8));
1663 ✗ __m512 data_5 = _mm512_mask_blend_ps(k_high8, _mm512_permutex2var_ps(data_src_low8, perm_5, data_src2_low8), _mm512_permutex2var_ps(data_src_high8, perm_5, data_src2_high8));
1664 ✗ __m512 data_6 = _mm512_mask_blend_ps(k_high8, _mm512_permutex2var_ps(data_src_low8, perm_6, data_src2_low8), _mm512_permutex2var_ps(data_src_high8, perm_6, data_src2_high8));
1665 ✗ __m512 data_7 = _mm512_mask_blend_ps(k_high8, _mm512_permutex2var_ps(data_src_low8, perm_7, data_src2_low8), _mm512_permutex2var_ps(data_src_high8, perm_7, data_src2_high8));
1666
1667 ✗ __m512 result0 = _mm512_mul_ps(data_0, coef_r0);
1668 ✗ __m512 result1 = _mm512_mul_ps(data_2, coef_r2);
1669 ✗ __m512 result2 = _mm512_mul_ps(data_4, coef_r4);
1670 ✗ __m512 result3 = _mm512_mul_ps(data_6, coef_r6);
1671
1672 ✗ result0 = _mm512_fmadd_ps(data_1, coef_r1, result0);
1673 ✗ result1 = _mm512_fmadd_ps(data_3, coef_r3, result1);
1674 ✗ result2 = _mm512_fmadd_ps(data_5, coef_r5, result2);
1675 ✗ result3 = _mm512_fmadd_ps(data_7, coef_r7, result3);
1676
1677 ✗ __m512 result01 = _mm512_add_ps(result0, result1);
1678 ✗ __m512 result23 = _mm512_add_ps(result2, result3);
1679 ✗ __m512 result0123 = _mm512_add_ps(result01, result23);
1680 _mm512_stream_ps(dst_ptr, result0123);
1681
1682 ✗ dst_ptr += dst_pitch;
1683 ✗ src_ptr_low8 += src_pitch;
1684 ✗ src_ptr_high8 += src_pitch;
1685 }
1686
1687 ✗ current_coeff += filter_size * PIXELS_AT_A_TIME;
1688 ✗ };
1689
1690 // Process the 'safe zone' where direct full unaligned loads are acceptable.
1691 ✗ for (; x < width_safe_mod; x += PIXELS_AT_A_TIME)
1692 {
1693 ✗ do_h_float_core(std::false_type{});
1694 }
1695
1696 // Process the potentially 'unsafe zone' near the image edge, using safe masked loading.
1697 ✗ for (; x < width; x += PIXELS_AT_A_TIME)
1698 {
1699 ✗ do_h_float_core(std::true_type{});
1700 }
1701 }
1702 ✗ }
1703
1704
1705
1706 // Similar to resize_h_planar_float_avx512_permutex_vstripe_ks4 but for kernel size up to 16
1707 // 16 target pixels at a time with AVX512 permutex instructions.
1708 ✗ void resize_h_planar_float_avx512_permutex_vstripe_ks16(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel)
1709 {
1710 ✗ src_pitch /= sizeof(float);
1711 ✗ dst_pitch /= sizeof(float);
1712
1713 ✗ float* src = (float*)src8;
1714 ✗ float* dst = (float*)dst8;
1715
1716 ✗ constexpr int PIXELS_AT_A_TIME = 16; // Process sixteen pixels in parallel using AVX512
1717
1718 // 'source_overread_beyond_targetx' indicates if the filter kernel can read beyond the target width.
1719 ✗ const int width_safe_mod = (program->safelimit_8_pixels.overread_possible ? program->safelimit_8_pixels.source_overread_beyond_targetx : width) / PIXELS_AT_A_TIME * PIXELS_AT_A_TIME;
1720
1721 // Preconditions:
1722 ✗ assert(program->filter_size_real <= 16); // We preload all relevant coefficients (up to 8) before the height loop.
1723
1724 // 'target_size_alignment' ensures we can safely access coefficients using offsets like
1725 // 'filter_size * 15' when processing 16 H pixels at a time
1726 ✗ assert(program->target_size_alignment >= 16); // Adjusted for 16 pixels
1727 assert(FRAME_ALIGN >= 64); // Adjusted for 16 pixels AviSynth+ default
1728
1729 // Ensure that coefficient loading beyond the valid target size is safe for 4x16 float loads.
1730 // We gather_load 'short' coeffs, from 0..15.
1731 // Load is unaligned, but we treat 16 coeffs.
1732 // - For kernel_size 9..16 we have valid entries until 16 when filter_size_alignment==8:
1733 // alignment 8 is enough for gathering with filter_size*15 offset.
1734 // - But for kernel_size 1..8 we need alignment 16 to be able to gather up to
1735 // filter_size*15 offset.
1736 // This is just an extra safety; this case is called in practice between kernel sizes 9..16.
1737 ✗ assert(
1738 (program->filter_size_real < 8 && program->filter_size_alignment >= 16)
1739 ||
1740 (program->filter_size_real >= 8 && program->filter_size_alignment >= 8)
1741 );
1742
1743 ✗ const int max_scanlines = program->max_scanlines;
1744
1745 // Vertical stripe loop for L2 cache optimization
1746 ✗ for (int y_from = 0; y_from < height; y_from += max_scanlines)
1747 {
1748 ✗ int y_to = std::min(y_from + max_scanlines, height);
1749
1750 // Reset current_coeff_t to pretransposed coefficient buffer (tap-major layout)
1751 ✗ const float* AVS_RESTRICT current_coeff_t = (const float* AVS_RESTRICT)program->pixel_coefficient_AVX512_float_H;
1752
1753 ✗ int x = 0;
1754
1755 // Lambda to handle both safe (fast) and unsafe (masked/partial) loading paths
1756 ✗ auto do_h_float_core = [&](auto partial_load) {
1757
1758 // Load pretransposed coefficients: 16 sequential aligned loads instead of 16 gathers.
1759 // Layout per x-group: [coef_r0[px0..15], coef_r1[px0..15], ..., coef_r15[px0..15]]
1760 ✗ const __m512 coef_r0 = _mm512_load_ps(current_coeff_t + 0 * 16);
1761 ✗ const __m512 coef_r1 = _mm512_load_ps(current_coeff_t + 1 * 16);
1762 ✗ const __m512 coef_r2 = _mm512_load_ps(current_coeff_t + 2 * 16);
1763 ✗ const __m512 coef_r3 = _mm512_load_ps(current_coeff_t + 3 * 16);
1764 ✗ const __m512 coef_r4 = _mm512_load_ps(current_coeff_t + 4 * 16);
1765 ✗ const __m512 coef_r5 = _mm512_load_ps(current_coeff_t + 5 * 16);
1766 ✗ const __m512 coef_r6 = _mm512_load_ps(current_coeff_t + 6 * 16);
1767 ✗ const __m512 coef_r7 = _mm512_load_ps(current_coeff_t + 7 * 16);
1768 ✗ const __m512 coef_r8 = _mm512_load_ps(current_coeff_t + 8 * 16);
1769 ✗ const __m512 coef_r9 = _mm512_load_ps(current_coeff_t + 9 * 16);
1770 ✗ const __m512 coef_r10 = _mm512_load_ps(current_coeff_t + 10 * 16);
1771 ✗ const __m512 coef_r11 = _mm512_load_ps(current_coeff_t + 11 * 16);
1772 ✗ const __m512 coef_r12 = _mm512_load_ps(current_coeff_t + 12 * 16);
1773 ✗ const __m512 coef_r13 = _mm512_load_ps(current_coeff_t + 13 * 16);
1774 ✗ const __m512 coef_r14 = _mm512_load_ps(current_coeff_t + 14 * 16);
1775 ✗ const __m512 coef_r15 = _mm512_load_ps(current_coeff_t + 15 * 16);
1776
1777 ✗ const __m512i one_epi32 = _mm512_set1_epi32(1);
1778
1779 // convert resampling program in H-form into permuting indexes for src transposition in V-form
1780 // shorter SIMD-way - single memory load (hacky SIMD load from int vector ?)
1781 ✗ __m512i perm_0 = _mm512_loadu_si512((__m512i*)(&program->pixel_offset[x]));
1782 ✗ int iStart = _mm256_extract_epi32(_mm512_castsi512_si256(perm_0), 0);
1783 ✗ perm_0 = _mm512_sub_epi32(perm_0, _mm512_set1_epi32(iStart)); // vpbroadcastd zmm, r32
1784
1785 ✗ float* AVS_RESTRICT dst_ptr = dst + x + y_from * dst_pitch;
1786 ✗ const float* src_ptr = src + iStart + y_from * src_pitch; // all permute offsets relative to this start offset
1787
1788 // Calculate remaining pixels for bounds checking in partial_load mode
1789 ✗ const int remaining = program->source_size - iStart;
1790
1791 ✗ int rem1 = std::max(0, std::min(16, remaining));
1792 ✗ __mmask16 k1 = (1U << rem1) - 1;
1793 ✗ int rem2 = std::max(0, std::min(16, remaining - 16));
1794 ✗ __mmask16 k2 = (1U << rem2) - 1;
1795
1796 ✗ for (int y = y_from; y < y_to; y++)
1797 {
1798 // Taps are contiguous (0, 1, 2, 3 .. 7), so we increment perm indexes by 1.
1799 // To save register usage in ks16 version - calculate offsets at runtime
1800 // working indexes, reloaded from constant perm_0
1801 ✗ __m512i perm_0w = perm_0;
1802 ✗ __m512i perm_1w = _mm512_add_epi32(perm_0, one_epi32);
1803
1804 ✗ const __m512i two_epi32 = _mm512_set1_epi32(2);
1805
1806 __m512 data_src, data_src2;
1807
1808 if constexpr (partial_load) {
1809 // Safe masked loads for the image edge
1810 // Load first 16 floats
1811 ✗ data_src = _mm512_maskz_loadu_ps(k1, src_ptr);
1812 // Load next 16 floats (offset by 16)
1813 ✗ data_src2 = _mm512_maskz_loadu_ps(k2, src_ptr + 16);
1814 }
1815 else {
1816 // Fast unaligned loads for the safe zone
1817 ✗ data_src = _mm512_loadu_ps(src_ptr);
1818 ✗ data_src2 = _mm512_loadu_ps(src_ptr + 16);
1819 }
1820
1821 ✗ __m512 data_0 = _mm512_permutex2var_ps(data_src, perm_0w, data_src2); // TODO: replace with shorter _mm512_mul_ps(_mm512_permutex2var_ps(data_src, perm_0w, data_src2), coef_r0);
1822 ✗ __m512 data_1 = _mm512_permutex2var_ps(data_src, perm_1w, data_src2);
1823
1824 ✗ __m512 result0 = _mm512_mul_ps(data_0, coef_r0);
1825 ✗ __m512 result1 = _mm512_mul_ps(data_1, coef_r1);
1826
1827 ✗ perm_0w = _mm512_add_epi32(perm_0w, two_epi32);
1828 ✗ perm_1w = _mm512_add_epi32(perm_1w, two_epi32);
1829
1830 ✗ __m512 data_2 = _mm512_permutex2var_ps(data_src, perm_0w, data_src2); // TODO: replace with shorter result0 = _mm512_fmadd_ps(_mm512_permutex2var_ps(data_src, perm_0w, data_src2), coef_r2, result0);
1831 ✗ __m512 data_3 = _mm512_permutex2var_ps(data_src, perm_1w, data_src2);
1832
1833 ✗ result0 = _mm512_fmadd_ps(data_2, coef_r2, result0);
1834 ✗ result1 = _mm512_fmadd_ps(data_3, coef_r3, result1);
1835
1836 ✗ perm_0w = _mm512_add_epi32(perm_0w, two_epi32);
1837 ✗ perm_1w = _mm512_add_epi32(perm_1w, two_epi32);
1838
1839 ✗ __m512 data_4 = _mm512_permutex2var_ps(data_src, perm_0w, data_src2);
1840 ✗ __m512 data_5 = _mm512_permutex2var_ps(data_src, perm_1w, data_src2);
1841
1842 ✗ result0 = _mm512_fmadd_ps(data_4, coef_r4, result0);
1843 ✗ result1 = _mm512_fmadd_ps(data_5, coef_r5, result1);
1844
1845 ✗ perm_0w = _mm512_add_epi32(perm_0w, two_epi32);
1846 ✗ perm_1w = _mm512_add_epi32(perm_1w, two_epi32);
1847
1848 ✗ __m512 data_6 = _mm512_permutex2var_ps(data_src, perm_0w, data_src2);
1849 ✗ __m512 data_7 = _mm512_permutex2var_ps(data_src, perm_1w, data_src2);
1850
1851 ✗ result0 = _mm512_fmadd_ps(data_6, coef_r6, result0);
1852 ✗ result1 = _mm512_fmadd_ps(data_7, coef_r7, result1);
1853
1854 ✗ perm_0w = _mm512_add_epi32(perm_0w, two_epi32);
1855 ✗ perm_1w = _mm512_add_epi32(perm_1w, two_epi32);
1856
1857 ✗ __m512 data_8 = _mm512_permutex2var_ps(data_src, perm_0w, data_src2);
1858 ✗ __m512 data_9 = _mm512_permutex2var_ps(data_src, perm_1w, data_src2);
1859
1860 ✗ result0 = _mm512_fmadd_ps(data_8, coef_r8, result0);
1861 ✗ result1 = _mm512_fmadd_ps(data_9, coef_r9, result1);
1862
1863 ✗ perm_0w = _mm512_add_epi32(perm_0w, two_epi32);
1864 ✗ perm_1w = _mm512_add_epi32(perm_1w, two_epi32);
1865
1866 ✗ __m512 data_10 = _mm512_permutex2var_ps(data_src, perm_0w, data_src2);
1867 ✗ __m512 data_11 = _mm512_permutex2var_ps(data_src, perm_1w, data_src2);
1868
1869 ✗ result0 = _mm512_fmadd_ps(data_10, coef_r10, result0);
1870 ✗ result1 = _mm512_fmadd_ps(data_11, coef_r11, result1);
1871
1872 ✗ perm_0w = _mm512_add_epi32(perm_0w, two_epi32);
1873 ✗ perm_1w = _mm512_add_epi32(perm_1w, two_epi32);
1874
1875 ✗ __m512 data_12 = _mm512_permutex2var_ps(data_src, perm_0w, data_src2);
1876 ✗ __m512 data_13 = _mm512_permutex2var_ps(data_src, perm_1w, data_src2);
1877
1878 ✗ result0 = _mm512_fmadd_ps(data_12, coef_r12, result0);
1879 ✗ result1 = _mm512_fmadd_ps(data_13, coef_r13, result1);
1880
1881 ✗ perm_0w = _mm512_add_epi32(perm_0w, two_epi32);
1882 ✗ perm_1w = _mm512_add_epi32(perm_1w, two_epi32);
1883
1884 ✗ __m512 data_14 = _mm512_permutex2var_ps(data_src, perm_0w, data_src2);
1885 ✗ __m512 data_15 = _mm512_permutex2var_ps(data_src, perm_1w, data_src2);
1886
1887 ✗ result0 = _mm512_fmadd_ps(data_14, coef_r14, result0);
1888 ✗ result1 = _mm512_fmadd_ps(data_15, coef_r15, result1);
1889
1890 ✗ _mm512_stream_ps(dst_ptr, _mm512_add_ps(result0, result1));
1891
1892 ✗ dst_ptr += dst_pitch;
1893 ✗ src_ptr += src_pitch;
1894 }
1895
1896 ✗ current_coeff_t += 16 * PIXELS_AT_A_TIME; // 256 floats per x-group (16 taps × 16 pixels)
1897 ✗ };
1898
1899 // Process the 'safe zone' where direct full unaligned loads are acceptable.
1900 ✗ for (; x < width_safe_mod; x += PIXELS_AT_A_TIME)
1901 {
1902 ✗ do_h_float_core(std::false_type{});
1903 }
1904
1905 // Process the potentially 'unsafe zone' near the image edge, using safe masked loading.
1906 ✗ for (; x < width; x += PIXELS_AT_A_TIME)
1907 {
1908 ✗ do_h_float_core(std::true_type{});
1909 }
1910 }
1911 ✗ }
1912
1913
1914 // Similar to resize_h_planar_float_avx512_permutex_vstripe_ks4 but for kernel size up to 16
1915 // 16 target pixels at a time with AVX512 permutex instructions.
1916 // Uses 2 groups of 8 output samples processing by independent gathering 2x32 contigous groups of sources to support more downscale ratios
1917 ✗ void resize_h_planar_float_avx512_permutex_vstripe_2s8_ks16(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel)
1918 {
1919 ✗ src_pitch /= sizeof(float);
1920 ✗ dst_pitch /= sizeof(float);
1921
1922 ✗ float* src = (float*)src8;
1923 ✗ float* dst = (float*)dst8;
1924
1925 ✗ constexpr int PIXELS_AT_A_TIME = 16; // Process 16 pixels in parallel using AVX512 for partial downsampling works
1926
1927 // 'source_overread_beyond_targetx' indicates if the filter kernel can read beyond the target width.
1928 ✗ const int width_safe_mod = (program->safelimit_8_pixels.overread_possible ? program->safelimit_8_pixels.source_overread_beyond_targetx : width) / PIXELS_AT_A_TIME * PIXELS_AT_A_TIME;
1929
1930 // Preconditions:
1931 ✗ assert(program->filter_size_real <= 16); // We preload all relevant coefficients (up to 8) before the height loop.
1932
1933 // 'target_size_alignment' ensures we can safely access coefficients using offsets like
1934 // 'filter_size * 15' when processing 16 H pixels at a time
1935 ✗ assert(program->target_size_alignment >= 16); // Adjusted for 16 pixels
1936 assert(FRAME_ALIGN >= 64); // Adjusted for 16 pixels AviSynth+ default
1937
1938 // Ensure that coefficient loading beyond the valid target size is safe for 4x16 float loads.
1939 // n/a: load once 16. align 8 is the same as 16 for the purpose.
1940 ✗ assert(program->filter_size_alignment >= 8);
1941
1942 ✗ const int max_scanlines = program->max_scanlines;
1943
1944 // Vertical stripe loop for L2 cache optimization
1945 ✗ for (int y_from = 0; y_from < height; y_from += max_scanlines)
1946 {
1947 ✗ int y_to = std::min(y_from + max_scanlines, height);
1948
1949 // Reset current_coeff_t to pretransposed coefficient buffer (tap-major layout)
1950 ✗ const float* AVS_RESTRICT current_coeff_t = (const float* AVS_RESTRICT)program->pixel_coefficient_AVX512_float_H;
1951
1952 ✗ int x = 0;
1953
1954 // Lambda to handle both safe (fast) and unsafe (masked/partial) loading paths
1955 ✗ auto do_h_float_core = [&](auto partial_load) {
1956
1957 // Load pretransposed coefficients: 16 sequential aligned loads instead of 16 gathers.
1958 // Layout per x-group: [coef_r0[px0..15], coef_r1[px0..15], ..., coef_r15[px0..15]]
1959 ✗ const __m512 coef_r0 = _mm512_load_ps(current_coeff_t + 0 * 16);
1960 ✗ const __m512 coef_r1 = _mm512_load_ps(current_coeff_t + 1 * 16);
1961 ✗ const __m512 coef_r2 = _mm512_load_ps(current_coeff_t + 2 * 16);
1962 ✗ const __m512 coef_r3 = _mm512_load_ps(current_coeff_t + 3 * 16);
1963 ✗ const __m512 coef_r4 = _mm512_load_ps(current_coeff_t + 4 * 16);
1964 ✗ const __m512 coef_r5 = _mm512_load_ps(current_coeff_t + 5 * 16);
1965 ✗ const __m512 coef_r6 = _mm512_load_ps(current_coeff_t + 6 * 16);
1966 ✗ const __m512 coef_r7 = _mm512_load_ps(current_coeff_t + 7 * 16);
1967 ✗ const __m512 coef_r8 = _mm512_load_ps(current_coeff_t + 8 * 16);
1968 ✗ const __m512 coef_r9 = _mm512_load_ps(current_coeff_t + 9 * 16);
1969 ✗ const __m512 coef_r10 = _mm512_load_ps(current_coeff_t + 10 * 16);
1970 ✗ const __m512 coef_r11 = _mm512_load_ps(current_coeff_t + 11 * 16);
1971 ✗ const __m512 coef_r12 = _mm512_load_ps(current_coeff_t + 12 * 16);
1972 ✗ const __m512 coef_r13 = _mm512_load_ps(current_coeff_t + 13 * 16);
1973 ✗ const __m512 coef_r14 = _mm512_load_ps(current_coeff_t + 14 * 16);
1974 ✗ const __m512 coef_r15 = _mm512_load_ps(current_coeff_t + 15 * 16);
1975
1976 ✗ const __m512i one_epi32 = _mm512_set1_epi32(1);
1977
1978 // convert resampling program in H-form into permuting indexes for src transposition in V-form
1979 // shorter SIMD-way - single memory load (hacky SIMD load from int vector ?)
1980 ✗ __m512i perm_0_low8 = _mm512_loadu_si512((__m512i*)(&program->pixel_offset[x]));
1981 ✗ int iStart_low8 = program->pixel_offset[x];
1982 ✗ perm_0_low8 = _mm512_sub_epi32(perm_0_low8, _mm512_set1_epi32(iStart_low8)); // vpbroadcastd zmm, r32
1983
1984 ✗ __m512i perm_0_high8 = _mm512_loadu_si512((__m512i*)(&program->pixel_offset[x + 8]));
1985 ✗ int iStart_high8 = program->pixel_offset[x + 8];
1986 ✗ perm_0_high8 = _mm512_sub_epi32(perm_0_high8, _mm512_set1_epi32(iStart_high8)); // vpbroadcastd zmm, r32
1987 ✗ perm_0_high8 = _mm512_inserti64x4(perm_0_high8, _mm512_castsi512_si256(perm_0_high8), 1);// shift low 8 epi32 to high 8
1988
1989 ✗ const __mmask16 k_high8 = 0xFF00;
1990 ✗ const __m512i perm_0 = _mm512_mask_blend_epi32(k_high8, perm_0_low8, perm_0_high8);
1991
1992 ✗ float* AVS_RESTRICT dst_ptr = dst + x + y_from * dst_pitch;
1993 ✗ const float* src_ptr_low8 = src + iStart_low8 + y_from * src_pitch; // all permute offsets in a first group relative to this start offset
1994 ✗ const float* src_ptr_high8 = src + iStart_high8 + y_from * src_pitch; // all permute offsets in a second group relative to this start offset
1995
1996 // Calculate remaining pixels for bounds checking in partial_load mode
1997 ✗ const int remaining_low8 = program->source_size - iStart_low8;
1998 ✗ const int remaining_high8 = program->source_size - iStart_high8;
1999
2000 ✗ int rem1_low8 = std::max(0, std::min(16, remaining_low8));
2001 ✗ __mmask16 k1_low8 = (1U << rem1_low8) - 1;
2002 ✗ int rem1_high8 = std::max(0, std::min(16, remaining_high8));
2003 ✗ __mmask16 k1_high8 = (1U << rem1_high8) - 1;
2004
2005 ✗ for (int y = y_from; y < y_to; y++)
2006 {
2007 // Taps are contiguous (0, 1, 2, 3 .. 7), so we increment perm indexes by 1.
2008 // To save register usage in ks16 version - calculate offsets at runtime
2009 // working indexes, reloaded from constant perm_0
2010 ✗ __m512i perm_0w = perm_0;
2011 ✗ __m512i perm_1w = _mm512_add_epi32(perm_0, one_epi32);
2012
2013 ✗ const __m512i two_epi32 = _mm512_set1_epi32(2);
2014
2015 __m512 data_src_low8, data_src2_low8;
2016 __m512 data_src_high8, data_src2_high8;
2017
2018 if constexpr (partial_load) {
2019 // Safe masked loads for the image edge
2020 // Load first 16 floats
2021 ✗ data_src_low8 = _mm512_maskz_loadu_ps(k1_low8, src_ptr_low8);
2022
2023 // Load next 16 floats (offset by 16)
2024 ✗ int rem2 = std::max(0, std::min(16, remaining_low8 - 16));
2025 ✗ __mmask16 k2_low8 = (1U << rem2) - 1;
2026 ✗ data_src2_low8 = _mm512_maskz_loadu_ps(k2_low8, src_ptr_low8 + 16);
2027
2028 // high8
2029 // Safe masked loads for the image edge
2030 // Load first 16 floats
2031 ✗ data_src_high8 = _mm512_maskz_loadu_ps(k1_high8, src_ptr_high8);
2032
2033 // Load next 16 floats (offset by 16)
2034 ✗ int rem2_high8 = std::max(0, std::min(16, remaining_high8 - 16));
2035 ✗ __mmask16 k2_high8 = (1U << rem2_high8) - 1;
2036 ✗ data_src2_high8 = _mm512_maskz_loadu_ps(k2_high8, src_ptr_high8 + 16);
2037 }
2038 else {
2039 // Fast unaligned loads for the safe zone
2040 ✗ data_src_low8 = _mm512_loadu_ps(src_ptr_low8);
2041 ✗ data_src2_low8 = _mm512_loadu_ps(src_ptr_low8 + 16);
2042 ✗ data_src_high8 = _mm512_loadu_ps(src_ptr_high8);
2043 ✗ data_src2_high8 = _mm512_loadu_ps(src_ptr_high8 + 16);
2044 }
2045
2046 ✗ __m512 data_0 = _mm512_mask_blend_ps(k_high8, _mm512_permutex2var_ps(data_src_low8, perm_0w, data_src2_low8), _mm512_permutex2var_ps(data_src_high8, perm_0w, data_src2_high8));
2047 ✗ __m512 data_1 = _mm512_mask_blend_ps(k_high8, _mm512_permutex2var_ps(data_src_low8, perm_1w, data_src2_low8), _mm512_permutex2var_ps(data_src_high8, perm_1w, data_src2_high8));
2048
2049 ✗ __m512 result0 = _mm512_mul_ps(data_0, coef_r0);
2050 ✗ __m512 result1 = _mm512_mul_ps(data_1, coef_r1);
2051
2052 ✗ perm_0w = _mm512_add_epi32(perm_0w, two_epi32);
2053 ✗ perm_1w = _mm512_add_epi32(perm_1w, two_epi32);
2054
2055 ✗ __m512 data_2 = _mm512_mask_blend_ps(k_high8, _mm512_permutex2var_ps(data_src_low8, perm_0w, data_src2_low8), _mm512_permutex2var_ps(data_src_high8, perm_0w, data_src2_high8));
2056 ✗ __m512 data_3 = _mm512_mask_blend_ps(k_high8, _mm512_permutex2var_ps(data_src_low8, perm_1w, data_src2_low8), _mm512_permutex2var_ps(data_src_high8, perm_1w, data_src2_high8));
2057
2058 ✗ result0 = _mm512_fmadd_ps(data_2, coef_r2, result0);
2059 ✗ result1 = _mm512_fmadd_ps(data_3, coef_r3, result1);
2060
2061 ✗ perm_0w = _mm512_add_epi32(perm_0w, two_epi32);
2062 ✗ perm_1w = _mm512_add_epi32(perm_1w, two_epi32);
2063
2064 ✗ __m512 data_4 = _mm512_mask_blend_ps(k_high8, _mm512_permutex2var_ps(data_src_low8, perm_0w, data_src2_low8), _mm512_permutex2var_ps(data_src_high8, perm_0w, data_src2_high8));
2065 ✗ __m512 data_5 = _mm512_mask_blend_ps(k_high8, _mm512_permutex2var_ps(data_src_low8, perm_1w, data_src2_low8), _mm512_permutex2var_ps(data_src_high8, perm_1w, data_src2_high8));
2066
2067 ✗ result0 = _mm512_fmadd_ps(data_4, coef_r4, result0);
2068 ✗ result1 = _mm512_fmadd_ps(data_5, coef_r5, result1);
2069
2070 ✗ perm_0w = _mm512_add_epi32(perm_0w, two_epi32);
2071 ✗ perm_1w = _mm512_add_epi32(perm_1w, two_epi32);
2072
2073 ✗ __m512 data_6 = _mm512_mask_blend_ps(k_high8, _mm512_permutex2var_ps(data_src_low8, perm_0w, data_src2_low8), _mm512_permutex2var_ps(data_src_high8, perm_0w, data_src2_high8));
2074 ✗ __m512 data_7 = _mm512_mask_blend_ps(k_high8, _mm512_permutex2var_ps(data_src_low8, perm_1w, data_src2_low8), _mm512_permutex2var_ps(data_src_high8, perm_1w, data_src2_high8));
2075
2076 ✗ result0 = _mm512_fmadd_ps(data_6, coef_r6, result0);
2077 ✗ result1 = _mm512_fmadd_ps(data_7, coef_r7, result1);
2078
2079 ✗ perm_0w = _mm512_add_epi32(perm_0w, two_epi32);
2080 ✗ perm_1w = _mm512_add_epi32(perm_1w, two_epi32);
2081
2082 ✗ __m512 data_8 = _mm512_mask_blend_ps(k_high8, _mm512_permutex2var_ps(data_src_low8, perm_0w, data_src2_low8), _mm512_permutex2var_ps(data_src_high8, perm_0w, data_src2_high8));
2083 ✗ __m512 data_9 = _mm512_mask_blend_ps(k_high8, _mm512_permutex2var_ps(data_src_low8, perm_1w, data_src2_low8), _mm512_permutex2var_ps(data_src_high8, perm_1w, data_src2_high8));
2084
2085 ✗ result0 = _mm512_fmadd_ps(data_8, coef_r8, result0);
2086 ✗ result1 = _mm512_fmadd_ps(data_9, coef_r9, result1);
2087
2088 ✗ perm_0w = _mm512_add_epi32(perm_0w, two_epi32);
2089 ✗ perm_1w = _mm512_add_epi32(perm_1w, two_epi32);
2090
2091 ✗ __m512 data_10 = _mm512_mask_blend_ps(k_high8, _mm512_permutex2var_ps(data_src_low8, perm_0w, data_src2_low8), _mm512_permutex2var_ps(data_src_high8, perm_0w, data_src2_high8));
2092 ✗ __m512 data_11 = _mm512_mask_blend_ps(k_high8, _mm512_permutex2var_ps(data_src_low8, perm_1w, data_src2_low8), _mm512_permutex2var_ps(data_src_high8, perm_1w, data_src2_high8));
2093
2094 ✗ result0 = _mm512_fmadd_ps(data_10, coef_r10, result0);
2095 ✗ result1 = _mm512_fmadd_ps(data_11, coef_r11, result1);
2096
2097 ✗ perm_0w = _mm512_add_epi32(perm_0w, two_epi32);
2098 ✗ perm_1w = _mm512_add_epi32(perm_1w, two_epi32);
2099
2100 ✗ __m512 data_12 = _mm512_mask_blend_ps(k_high8, _mm512_permutex2var_ps(data_src_low8, perm_0w, data_src2_low8), _mm512_permutex2var_ps(data_src_high8, perm_0w, data_src2_high8));
2101 ✗ __m512 data_13 = _mm512_mask_blend_ps(k_high8, _mm512_permutex2var_ps(data_src_low8, perm_1w, data_src2_low8), _mm512_permutex2var_ps(data_src_high8, perm_1w, data_src2_high8));
2102
2103 ✗ result0 = _mm512_fmadd_ps(data_12, coef_r12, result0);
2104 ✗ result1 = _mm512_fmadd_ps(data_13, coef_r13, result1);
2105
2106 ✗ perm_0w = _mm512_add_epi32(perm_0w, two_epi32);
2107 ✗ perm_1w = _mm512_add_epi32(perm_1w, two_epi32);
2108
2109 ✗ __m512 data_14 = _mm512_mask_blend_ps(k_high8, _mm512_permutex2var_ps(data_src_low8, perm_0w, data_src2_low8), _mm512_permutex2var_ps(data_src_high8, perm_0w, data_src2_high8));
2110 ✗ __m512 data_15 = _mm512_mask_blend_ps(k_high8, _mm512_permutex2var_ps(data_src_low8, perm_1w, data_src2_low8), _mm512_permutex2var_ps(data_src_high8, perm_1w, data_src2_high8));
2111
2112 ✗ result0 = _mm512_fmadd_ps(data_14, coef_r14, result0);
2113 ✗ result1 = _mm512_fmadd_ps(data_15, coef_r15, result1);
2114
2115 ✗ _mm512_stream_ps(dst_ptr, _mm512_add_ps(result0, result1));
2116
2117 ✗ dst_ptr += dst_pitch;
2118 ✗ src_ptr_low8 += src_pitch;
2119 ✗ src_ptr_high8 += src_pitch;
2120 }
2121
2122 ✗ current_coeff_t += 16 * PIXELS_AT_A_TIME; // 256 floats per x-group (16 taps × 16 pixels)
2123 ✗ };
2124
2125 // Process the 'safe zone' where direct full unaligned loads are acceptable.
2126 ✗ for (; x < width_safe_mod; x += PIXELS_AT_A_TIME)
2127 {
2128 ✗ do_h_float_core(std::false_type{});
2129 }
2130
2131 // Process the potentially 'unsafe zone' near the image edge, using safe masked loading.
2132 ✗ for (; x < width; x += PIXELS_AT_A_TIME)
2133 {
2134 ✗ do_h_float_core(std::true_type{});
2135 }
2136 }
2137 ✗ }
2138
2139
2140 // 16 target pixels at a time from 4 independent source groups of 4 output pixels each.
2141 // Handles heavy downscaling where 2s8_ks16 is not feasible (each 8-px group needs >32 source floats)
2142 // but each 4-px group still fits within a 32-float (2-ZMM) permutex window.
2143 ✗ void resize_h_planar_float_avx512_permutex_vstripe_4s4_ks16(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel)
2144 {
2145 ✗ src_pitch /= sizeof(float);
2146 ✗ dst_pitch /= sizeof(float);
2147
2148 ✗ float* src = (float*)src8;
2149 ✗ float* dst = (float*)dst8;
2150
2151 ✗ constexpr int PIXELS_AT_A_TIME = 16;
2152
2153 ✗ const int width_safe_mod = (program->safelimit_8_pixels.overread_possible ? program->safelimit_8_pixels.source_overread_beyond_targetx : width) / PIXELS_AT_A_TIME * PIXELS_AT_A_TIME;
2154
2155 ✗ assert(program->filter_size_real <= 16);
2156 ✗ assert(program->target_size_alignment >= 16);
2157 assert(FRAME_ALIGN >= 64);
2158 ✗ assert(program->filter_size_alignment >= 8);
2159
2160 ✗ const int max_scanlines = program->max_scanlines;
2161
2162 ✗ for (int y_from = 0; y_from < height; y_from += max_scanlines)
2163 {
2164 ✗ int y_to = std::min(y_from + max_scanlines, height);
2165 ✗ const float* AVS_RESTRICT current_coeff_t = (const float* AVS_RESTRICT)program->pixel_coefficient_AVX512_float_H;
2166 ✗ int x = 0;
2167
2168 ✗ auto do_h_float_core = [&](auto partial_load) {
2169 ✗ const __m512 coef_r0 = _mm512_load_ps(current_coeff_t + 0 * 16);
2170 ✗ const __m512 coef_r1 = _mm512_load_ps(current_coeff_t + 1 * 16);
2171 ✗ const __m512 coef_r2 = _mm512_load_ps(current_coeff_t + 2 * 16);
2172 ✗ const __m512 coef_r3 = _mm512_load_ps(current_coeff_t + 3 * 16);
2173 ✗ const __m512 coef_r4 = _mm512_load_ps(current_coeff_t + 4 * 16);
2174 ✗ const __m512 coef_r5 = _mm512_load_ps(current_coeff_t + 5 * 16);
2175 ✗ const __m512 coef_r6 = _mm512_load_ps(current_coeff_t + 6 * 16);
2176 ✗ const __m512 coef_r7 = _mm512_load_ps(current_coeff_t + 7 * 16);
2177 ✗ const __m512 coef_r8 = _mm512_load_ps(current_coeff_t + 8 * 16);
2178 ✗ const __m512 coef_r9 = _mm512_load_ps(current_coeff_t + 9 * 16);
2179 ✗ const __m512 coef_r10 = _mm512_load_ps(current_coeff_t + 10 * 16);
2180 ✗ const __m512 coef_r11 = _mm512_load_ps(current_coeff_t + 11 * 16);
2181 ✗ const __m512 coef_r12 = _mm512_load_ps(current_coeff_t + 12 * 16);
2182 ✗ const __m512 coef_r13 = _mm512_load_ps(current_coeff_t + 13 * 16);
2183 ✗ const __m512 coef_r14 = _mm512_load_ps(current_coeff_t + 14 * 16);
2184 ✗ const __m512 coef_r15 = _mm512_load_ps(current_coeff_t + 15 * 16);
2185
2186 ✗ const __m512i one_epi32 = _mm512_set1_epi32(1);
2187
2188 // 4 source groups, each covering 4 consecutive output pixels (x+0..3, x+4..7, x+8..11, x+12..15).
2189 // Each group's perm indices are stored in the corresponding quarter of perm_0 (128-bit lane).
2190 ✗ const int iStart_g0 = program->pixel_offset[x];
2191 ✗ const int iStart_g1 = program->pixel_offset[x + 4];
2192 ✗ const int iStart_g2 = program->pixel_offset[x + 8];
2193 ✗ const int iStart_g3 = program->pixel_offset[x + 12];
2194
2195 // Build combined perm vector: subtract each group's start from its own 4 offsets.
2196 // Positions 0..3 = g0-relative, 4..7 = g1-relative, 8..11 = g2-relative, 12..15 = g3-relative.
2197 ✗ const __m512i perm_0 = _mm512_sub_epi32(
2198 ✗ _mm512_loadu_si512((__m512i*)&program->pixel_offset[x]),
2199 _mm512_set_epi32(
2200 iStart_g3, iStart_g3, iStart_g3, iStart_g3,
2201 iStart_g2, iStart_g2, iStart_g2, iStart_g2,
2202 iStart_g1, iStart_g1, iStart_g1, iStart_g1,
2203 iStart_g0, iStart_g0, iStart_g0, iStart_g0
2204 )
2205 );
2206
2207 ✗ float* AVS_RESTRICT dst_ptr = dst + x + y_from * dst_pitch;
2208 ✗ const float* src_g0 = src + iStart_g0 + y_from * src_pitch;
2209 ✗ const float* src_g1 = src + iStart_g1 + y_from * src_pitch;
2210 ✗ const float* src_g2 = src + iStart_g2 + y_from * src_pitch;
2211 ✗ const float* src_g3 = src + iStart_g3 + y_from * src_pitch;
2212
2213 ✗ const int rem_g0 = program->source_size - iStart_g0;
2214 ✗ const int rem_g1 = program->source_size - iStart_g1;
2215 ✗ const int rem_g2 = program->source_size - iStart_g2;
2216 ✗ const int rem_g3 = program->source_size - iStart_g3;
2217
2218 ✗ const __mmask16 k1_g0 = (1U << std::max(0, std::min(16, rem_g0))) - 1;
2219 ✗ const __mmask16 k1_g1 = (1U << std::max(0, std::min(16, rem_g1))) - 1;
2220 ✗ const __mmask16 k1_g2 = (1U << std::max(0, std::min(16, rem_g2))) - 1;
2221 ✗ const __mmask16 k1_g3 = (1U << std::max(0, std::min(16, rem_g3))) - 1;
2222
2223 ✗ for (int y = y_from; y < y_to; y++)
2224 {
2225 __m512 sg0, sg0b, sg1, sg1b, sg2, sg2b, sg3, sg3b;
2226
2227 if constexpr (partial_load) {
2228 ✗ sg0 = _mm512_maskz_loadu_ps(k1_g0, src_g0);
2229 ✗ sg0b = _mm512_maskz_loadu_ps((1U << std::max(0, std::min(16, rem_g0 - 16))) - 1, src_g0 + 16);
2230 ✗ sg1 = _mm512_maskz_loadu_ps(k1_g1, src_g1);
2231 ✗ sg1b = _mm512_maskz_loadu_ps((1U << std::max(0, std::min(16, rem_g1 - 16))) - 1, src_g1 + 16);
2232 ✗ sg2 = _mm512_maskz_loadu_ps(k1_g2, src_g2);
2233 ✗ sg2b = _mm512_maskz_loadu_ps((1U << std::max(0, std::min(16, rem_g2 - 16))) - 1, src_g2 + 16);
2234 ✗ sg3 = _mm512_maskz_loadu_ps(k1_g3, src_g3);
2235 ✗ sg3b = _mm512_maskz_loadu_ps((1U << std::max(0, std::min(16, rem_g3 - 16))) - 1, src_g3 + 16);
2236 } else {
2237 ✗ sg0 = _mm512_loadu_ps(src_g0); sg0b = _mm512_loadu_ps(src_g0 + 16);
2238 ✗ sg1 = _mm512_loadu_ps(src_g1); sg1b = _mm512_loadu_ps(src_g1 + 16);
2239 ✗ sg2 = _mm512_loadu_ps(src_g2); sg2b = _mm512_loadu_ps(src_g2 + 16);
2240 ✗ sg3 = _mm512_loadu_ps(src_g3); sg3b = _mm512_loadu_ps(src_g3 + 16);
2241 }
2242
2243 ✗ __m512i perm_0w = perm_0;
2244 ✗ __m512i perm_1w = _mm512_add_epi32(perm_0, one_epi32);
2245 ✗ const __m512i two_epi32 = _mm512_set1_epi32(2);
2246
2247 // Gather from all 4 source groups using combined perm, then blend quarters.
2248 // Positions 0..3 from g0, 4..7 from g1, 8..11 from g2, 12..15 from g3.
2249 // Each permutex2var uses the same perm_w: the "wrong" quarters produce garbage but are discarded by blend.
2250 #define G4(p) _mm512_mask_blend_ps(0xF000, \
2251 _mm512_mask_blend_ps(0x0F00, \
2252 _mm512_mask_blend_ps(0x00F0, \
2253 _mm512_permutex2var_ps(sg0, (p), sg0b), \
2254 _mm512_permutex2var_ps(sg1, (p), sg1b)), \
2255 _mm512_permutex2var_ps(sg2, (p), sg2b)), \
2256 _mm512_permutex2var_ps(sg3, (p), sg3b))
2257
2258 ✗ __m512 result0 = _mm512_mul_ps(G4(perm_0w), coef_r0);
2259 ✗ __m512 result1 = _mm512_mul_ps(G4(perm_1w), coef_r1);
2260 ✗ perm_0w = _mm512_add_epi32(perm_0w, two_epi32);
2261 ✗ perm_1w = _mm512_add_epi32(perm_1w, two_epi32);
2262
2263 ✗ result0 = _mm512_fmadd_ps(G4(perm_0w), coef_r2, result0);
2264 ✗ result1 = _mm512_fmadd_ps(G4(perm_1w), coef_r3, result1);
2265 ✗ perm_0w = _mm512_add_epi32(perm_0w, two_epi32);
2266 ✗ perm_1w = _mm512_add_epi32(perm_1w, two_epi32);
2267
2268 ✗ result0 = _mm512_fmadd_ps(G4(perm_0w), coef_r4, result0);
2269 ✗ result1 = _mm512_fmadd_ps(G4(perm_1w), coef_r5, result1);
2270 ✗ perm_0w = _mm512_add_epi32(perm_0w, two_epi32);
2271 ✗ perm_1w = _mm512_add_epi32(perm_1w, two_epi32);
2272
2273 ✗ result0 = _mm512_fmadd_ps(G4(perm_0w), coef_r6, result0);
2274 ✗ result1 = _mm512_fmadd_ps(G4(perm_1w), coef_r7, result1);
2275 ✗ perm_0w = _mm512_add_epi32(perm_0w, two_epi32);
2276 ✗ perm_1w = _mm512_add_epi32(perm_1w, two_epi32);
2277
2278 ✗ result0 = _mm512_fmadd_ps(G4(perm_0w), coef_r8, result0);
2279 ✗ result1 = _mm512_fmadd_ps(G4(perm_1w), coef_r9, result1);
2280 ✗ perm_0w = _mm512_add_epi32(perm_0w, two_epi32);
2281 ✗ perm_1w = _mm512_add_epi32(perm_1w, two_epi32);
2282
2283 ✗ result0 = _mm512_fmadd_ps(G4(perm_0w), coef_r10, result0);
2284 ✗ result1 = _mm512_fmadd_ps(G4(perm_1w), coef_r11, result1);
2285 ✗ perm_0w = _mm512_add_epi32(perm_0w, two_epi32);
2286 ✗ perm_1w = _mm512_add_epi32(perm_1w, two_epi32);
2287
2288 ✗ result0 = _mm512_fmadd_ps(G4(perm_0w), coef_r12, result0);
2289 ✗ result1 = _mm512_fmadd_ps(G4(perm_1w), coef_r13, result1);
2290 ✗ perm_0w = _mm512_add_epi32(perm_0w, two_epi32);
2291 ✗ perm_1w = _mm512_add_epi32(perm_1w, two_epi32);
2292
2293 ✗ result0 = _mm512_fmadd_ps(G4(perm_0w), coef_r14, result0);
2294 ✗ result1 = _mm512_fmadd_ps(G4(perm_1w), coef_r15, result1);
2295
2296 #undef G4
2297
2298 ✗ _mm512_stream_ps(dst_ptr, _mm512_add_ps(result0, result1));
2299
2300 ✗ dst_ptr += dst_pitch;
2301 ✗ src_g0 += src_pitch; src_g1 += src_pitch;
2302 ✗ src_g2 += src_pitch; src_g3 += src_pitch;
2303 }
2304
2305 ✗ current_coeff_t += 16 * PIXELS_AT_A_TIME;
2306 ✗ };
2307
2308 ✗ for (; x < width_safe_mod; x += PIXELS_AT_A_TIME)
2309 ✗ do_h_float_core(std::false_type{});
2310 ✗ for (; x < width; x += PIXELS_AT_A_TIME)
2311 ✗ do_h_float_core(std::true_type{});
2312 }
2313 ✗ }
2314
2315
2316 //-------- 512 bit float Verticals
2317
2318 // base version, no horizontal unrolling
2319 ✗ void resize_v_avx512_planar_float(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int target_height, int bits_per_pixel)
2320 {
2321 AVS_UNUSED(bits_per_pixel);
2322
2323 ✗ const int filter_size = program->filter_size;
2324 ✗ const float* AVS_RESTRICT current_coeff = program->pixel_coefficient_float;
2325
2326 ✗ const float* src = (const float*)src8;
2327 ✗ float* AVS_RESTRICT dst = (float*)dst8;
2328 ✗ dst_pitch = dst_pitch / sizeof(float);
2329 ✗ src_pitch = src_pitch / sizeof(float);
2330
2331 ✗ const int kernel_size = program->filter_size_real; // not the aligned
2332 ✗ const int kernel_size_mod2 = (kernel_size / 2) * 2; // Process pairs of rows for better efficiency
2333 ✗ const bool notMod2 = kernel_size_mod2 < kernel_size;
2334
2335 ✗ for (int y = 0; y < target_height; y++) {
2336 ✗ int offset = program->pixel_offset[y];
2337 ✗ const float* src_ptr = src + offset * src_pitch;
2338
2339 // 64 byte 16 floats (AVX512 register holds 16 floats)
2340 // no need for wmod8, alignment is safe 32 bytes at least - is it safe for 64 bytes ?
2341 ✗ for (int x = 0; x < width; x += 16) {
2342 ✗ __m512 result_single = _mm512_setzero_ps();
2343 ✗ __m512 result_single_2 = _mm512_setzero_ps();
2344
2345 ✗ const float* AVS_RESTRICT src2_ptr = src_ptr + x; // __restrict here
2346
2347 // Process pairs of rows for better efficiency (2 coeffs/cycle)
2348 // two result variables for potential parallel operation
2349 ✗ int i = 0;
2350 ✗ for (; i < kernel_size_mod2; i += 2) {
2351 ✗ __m512 coeff_even = _mm512_set1_ps(current_coeff[i]);
2352 ✗ __m512 coeff_odd = _mm512_set1_ps(current_coeff[i + 1]);
2353
2354 ✗ __m512 src_even = _mm512_load_ps(src2_ptr);
2355 ✗ __m512 src_odd = _mm512_load_ps(src2_ptr + src_pitch);
2356
2357 ✗ result_single = _mm512_fmadd_ps(src_even, coeff_even, result_single);
2358 ✗ result_single_2 = _mm512_fmadd_ps(src_odd, coeff_odd, result_single_2);
2359
2360 ✗ src2_ptr += 2 * src_pitch;
2361 }
2362
2363 ✗ result_single = _mm512_add_ps(result_single, result_single_2);
2364
2365 // Process the last odd row if needed
2366 ✗ if (notMod2) {
2367 ✗ __m512 coeff = _mm512_set1_ps(current_coeff[i]);
2368 ✗ __m512 src_val = _mm512_load_ps(src2_ptr);
2369 ✗ result_single = _mm512_fmadd_ps(src_val, coeff, result_single);
2370 }
2371
2372 ✗ _mm512_stream_ps(dst + x, result_single);
2373 }
2374
2375 ✗ dst += dst_pitch;
2376 ✗ current_coeff += filter_size;
2377 }
2378 ✗ }
2379
2380 // memory-optimized version of resize_v_avx512_planar_float
2381 ✗ void resize_v_avx512_planar_float_w_sr(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int target_height, int bits_per_pixel)
2382 {
2383 AVS_UNUSED(bits_per_pixel);
2384
2385 ✗ const int filter_size = program->filter_size;
2386 ✗ const float* AVS_RESTRICT current_coeff = (const float* AVS_RESTRICT)program->pixel_coefficient_float;
2387
2388 ✗ const float* src = (const float*)src8;
2389 ✗ float* AVS_RESTRICT dst = (float*)dst8;
2390
2391 ✗ const int dst_stride_float = dst_pitch / sizeof(float);
2392 ✗ const int src_stride_float = src_pitch / sizeof(float);
2393
2394 ✗ const int kernel_size = program->filter_size_real;
2395 // Pre-calculate Mod2 size for the remainder loop
2396 ✗ const int kernel_size_mod2 = (kernel_size / 2) * 2;
2397 ✗ const bool notMod2 = kernel_size_mod2 < kernel_size;
2398
2399 ✗ for (int y = 0; y < target_height; y++) {
2400 ✗ int offset = program->pixel_offset[y];
2401 ✗ const float* src_row_start = src + offset * src_stride_float;
2402
2403 ✗ int x = 0;
2404
2405 // -----------------------------------------------------------------------
2406 // 128 pixels (512 bytes) per iteration
2407 // Uses ~17 ZMM registers. Safe for x64 (32 regs available).
2408 // Provides 8 independent dependency chains to hide FMA latency.
2409 // -----------------------------------------------------------------------
2410 ✗ const int width_mod128 = (width / 128) * 128;
2411 ✗ for (; x < width_mod128; x += 128) {
2412 ✗ __m512 result_1 = _mm512_setzero_ps();
2413 ✗ __m512 result_2 = _mm512_setzero_ps();
2414 ✗ __m512 result_3 = _mm512_setzero_ps();
2415 ✗ __m512 result_4 = _mm512_setzero_ps();
2416 ✗ __m512 result_5 = _mm512_setzero_ps();
2417 ✗ __m512 result_6 = _mm512_setzero_ps();
2418 ✗ __m512 result_7 = _mm512_setzero_ps();
2419 ✗ __m512 result_8 = _mm512_setzero_ps();
2420
2421 ✗ const float* AVS_RESTRICT src2_ptr = src_row_start + x;
2422
2423 ✗ for (int i = 0; i < kernel_size; i++) {
2424 ✗ __m512 coeff = _mm512_set1_ps(current_coeff[i]);
2425
2426 // Loading 512 bytes contiguous memory (8 cache lines)
2427 ✗ __m512 src_1 = _mm512_load_ps(src2_ptr); // 0..15
2428 ✗ __m512 src_2 = _mm512_load_ps(src2_ptr + 16); // 16..31 (offset in floats)
2429 ✗ __m512 src_3 = _mm512_load_ps(src2_ptr + 32);
2430 ✗ __m512 src_4 = _mm512_load_ps(src2_ptr + 48);
2431 ✗ __m512 src_5 = _mm512_load_ps(src2_ptr + 64);
2432 ✗ __m512 src_6 = _mm512_load_ps(src2_ptr + 80);
2433 ✗ __m512 src_7 = _mm512_load_ps(src2_ptr + 96);
2434 ✗ __m512 src_8 = _mm512_load_ps(src2_ptr + 112);
2435
2436 ✗ result_1 = _mm512_fmadd_ps(src_1, coeff, result_1);
2437 ✗ result_2 = _mm512_fmadd_ps(src_2, coeff, result_2);
2438 ✗ result_3 = _mm512_fmadd_ps(src_3, coeff, result_3);
2439 ✗ result_4 = _mm512_fmadd_ps(src_4, coeff, result_4);
2440 ✗ result_5 = _mm512_fmadd_ps(src_5, coeff, result_5);
2441 ✗ result_6 = _mm512_fmadd_ps(src_6, coeff, result_6);
2442 ✗ result_7 = _mm512_fmadd_ps(src_7, coeff, result_7);
2443 ✗ result_8 = _mm512_fmadd_ps(src_8, coeff, result_8);
2444
2445 ✗ src2_ptr += src_stride_float;
2446 }
2447
2448 ✗ _mm512_stream_ps(dst + x, result_1);
2449 ✗ _mm512_stream_ps(dst + x + 16, result_2);
2450 ✗ _mm512_stream_ps(dst + x + 32, result_3);
2451 ✗ _mm512_stream_ps(dst + x + 48, result_4);
2452 ✗ _mm512_stream_ps(dst + x + 64, result_5);
2453 ✗ _mm512_stream_ps(dst + x + 80, result_6);
2454 ✗ _mm512_stream_ps(dst + x + 96, result_7);
2455 ✗ _mm512_stream_ps(dst + x + 112, result_8);
2456 }
2457
2458 // -----------------------------------------------------------------------
2459 // 64 pixels per iteration
2460 // -----------------------------------------------------------------------
2461 ✗ const int width_mod64 = (width / 64) * 64;
2462 ✗ for (; x < width_mod64; x += 64) {
2463 ✗ __m512 result_1 = _mm512_setzero_ps();
2464 ✗ __m512 result_2 = _mm512_setzero_ps();
2465 ✗ __m512 result_3 = _mm512_setzero_ps();
2466 ✗ __m512 result_4 = _mm512_setzero_ps();
2467
2468 ✗ const float* AVS_RESTRICT src2_ptr = src_row_start + x;
2469
2470 ✗ for (int i = 0; i < kernel_size; i++) {
2471 ✗ __m512 coeff = _mm512_set1_ps(current_coeff[i]);
2472
2473 ✗ __m512 src_1 = _mm512_load_ps(src2_ptr);
2474 ✗ __m512 src_2 = _mm512_load_ps(src2_ptr + 16);
2475 ✗ __m512 src_3 = _mm512_load_ps(src2_ptr + 32);
2476 ✗ __m512 src_4 = _mm512_load_ps(src2_ptr + 48);
2477
2478 ✗ result_1 = _mm512_fmadd_ps(src_1, coeff, result_1);
2479 ✗ result_2 = _mm512_fmadd_ps(src_2, coeff, result_2);
2480 ✗ result_3 = _mm512_fmadd_ps(src_3, coeff, result_3);
2481 ✗ result_4 = _mm512_fmadd_ps(src_4, coeff, result_4);
2482
2483 ✗ src2_ptr += src_stride_float;
2484 }
2485
2486 ✗ _mm512_stream_ps(dst + x, result_1);
2487 ✗ _mm512_stream_ps(dst + x + 16, result_2);
2488 ✗ _mm512_stream_ps(dst + x + 32, result_3);
2489 ✗ _mm512_stream_ps(dst + x + 48, result_4);
2490 }
2491
2492 // -----------------------------------------------------------------------
2493 // 32 pixels per iteration
2494 // -----------------------------------------------------------------------
2495 ✗ const int width_mod32 = (width / 32) * 32;
2496 ✗ for (; x < width_mod32; x += 32) {
2497 ✗ __m512 result_1 = _mm512_setzero_ps();
2498 ✗ __m512 result_2 = _mm512_setzero_ps();
2499
2500 ✗ const float* AVS_RESTRICT src2_ptr = src_row_start + x;
2501
2502 ✗ for (int i = 0; i < kernel_size; i++) {
2503 ✗ __m512 coeff = _mm512_set1_ps(current_coeff[i]);
2504
2505 ✗ __m512 src_1 = _mm512_load_ps(src2_ptr);
2506 ✗ __m512 src_2 = _mm512_load_ps(src2_ptr + 16);
2507
2508 ✗ result_1 = _mm512_fmadd_ps(src_1, coeff, result_1);
2509 ✗ result_2 = _mm512_fmadd_ps(src_2, coeff, result_2);
2510
2511 ✗ src2_ptr += src_stride_float;
2512 }
2513
2514 ✗ _mm512_stream_ps(dst + x, result_1);
2515 ✗ _mm512_stream_ps(dst + x + 16, result_2);
2516 }
2517
2518 // -----------------------------------------------------------------------
2519 // Remainder loop (16 pixels)
2520 // Uses vertical loop unrolling (pairs of taps) to hide FMA latency
2521 // because we don't have enough horizontal data to do it spatially.
2522 // -----------------------------------------------------------------------
2523 ✗ const int src_stride_2 = src_stride_float * 2;
2524
2525 ✗ for (; x < width; x += 16) {
2526 ✗ __m512 result_single = _mm512_setzero_ps();
2527 ✗ __m512 result_single_2 = _mm512_setzero_ps();
2528
2529 ✗ const float* AVS_RESTRICT src2_ptr = src_row_start + x;
2530 ✗ int i = 0;
2531
2532 // Process pairs of rows
2533 ✗ for (; i < kernel_size_mod2; i += 2) {
2534 ✗ __m512 coeff_even = _mm512_set1_ps(current_coeff[i]);
2535 ✗ __m512 coeff_odd = _mm512_set1_ps(current_coeff[i + 1]);
2536
2537 ✗ __m512 src_even = _mm512_load_ps(src2_ptr);
2538 ✗ __m512 src_odd = _mm512_load_ps(src2_ptr + src_stride_float);
2539
2540 ✗ result_single = _mm512_fmadd_ps(src_even, coeff_even, result_single);
2541 ✗ result_single_2 = _mm512_fmadd_ps(src_odd, coeff_odd, result_single_2);
2542
2543 ✗ src2_ptr += src_stride_2;
2544 }
2545
2546 ✗ result_single = _mm512_add_ps(result_single, result_single_2);
2547
2548 // Process the last odd row if needed
2549 ✗ if (notMod2) {
2550 ✗ __m512 coeff = _mm512_set1_ps(current_coeff[i]);
2551 ✗ __m512 src_val = _mm512_load_ps(src2_ptr);
2552 ✗ result_single = _mm512_fmadd_ps(src_val, coeff, result_single);
2553 }
2554
2555 ✗ _mm512_stream_ps(dst + x, result_single);
2556 }
2557
2558 ✗ dst += dst_stride_float;
2559 ✗ current_coeff += filter_size;
2560 }
2561 ✗ }
2562
2563 // uint8_t
2564 ✗ void resize_v_avx512_planar_uint8_t_w_sr(BYTE* AVS_RESTRICT dst, const BYTE* src, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int target_height, int bits_per_pixel)
2565 {
2566 AVS_UNUSED(bits_per_pixel);
2567 ✗ int filter_size = program->filter_size;
2568 ✗ const short* AVS_RESTRICT current_coeff = program->pixel_coefficient;
2569 ✗ __m512i rounder = _mm512_set1_epi32(1 << (FPScale8bits - 1));
2570 ✗ __m512i zero = _mm512_setzero_si512();
2571
2572 ✗ const int kernel_size = program->filter_size_real; // not the aligned
2573
2574 ✗ const int width_mod128 = (width / 128) * 128;
2575
2576 ✗ const __m512i perm_idx1 = _mm512_set_epi64(8 + 5, 8 + 4, 8 + 1, 8 + 0, 5, 4, 1, 0);
2577 ✗ const __m512i perm_idx2 = _mm512_set_epi64(8 + 7, 8 + 6, 8 + 3, 8 + 2, 7, 6, 3, 2);
2578
2579 ✗ for (int y = 0; y < target_height; y++) {
2580 ✗ int offset = program->pixel_offset[y];
2581 ✗ const BYTE* AVS_RESTRICT src_ptr = src + offset * src_pitch;
2582
2583 ✗ for (int x = 0; x < width_mod128; x += 128) {
2584
2585 ✗ __m512i result_lo = rounder;
2586 ✗ __m512i result_hi = rounder;
2587 ✗ __m512i result_lo2 = rounder;
2588 ✗ __m512i result_hi2 = rounder;
2589
2590 ✗ __m512i result_lo_2 = rounder;
2591 ✗ __m512i result_hi_2 = rounder;
2592 ✗ __m512i result_lo2_2 = rounder;
2593 ✗ __m512i result_hi2_2 = rounder;
2594
2595 ✗ const uint8_t* AVS_RESTRICT src2_ptr = src_ptr + x;
2596
2597 ✗ int i = 0;
2598 // 128 byte 128 pixel
2599 ✗ for (; i < kernel_size; i++) {
2600 // Broadcast a single coefficients
2601 ✗ __m512i coeff = _mm512_set1_epi16(*reinterpret_cast<const short*>(current_coeff + i)); // 0|co|0|co|0|co|0|co 0|co|0|co|0|co|0|co
2602
2603 ✗ __m512i src_1_1 = _mm512_cvtepu8_epi16(_mm256_load_si256(reinterpret_cast<const __m256i*>(src2_ptr))); // 32x 8->16bit pixels
2604 ✗ __m512i src_1_2 = _mm512_cvtepu8_epi16(_mm256_load_si256(reinterpret_cast<const __m256i*>(src2_ptr + 32))); // 32x 8->16bit pixels
2605 ✗ __m512i src_2_1 = _mm512_cvtepu8_epi16(_mm256_load_si256(reinterpret_cast<const __m256i*>(src2_ptr + 64))); // 32x 8->16bit pixels
2606 ✗ __m512i src_2_2 = _mm512_cvtepu8_epi16(_mm256_load_si256(reinterpret_cast<const __m256i*>(src2_ptr + 96))); // 32x 8->16bit pixels
2607
2608 ✗ __m512i src_lo = _mm512_unpacklo_epi16(src_1_1, zero);
2609 ✗ __m512i src_hi = _mm512_unpackhi_epi16(src_1_1, zero);
2610 ✗ __m512i src_lo2 = _mm512_unpacklo_epi16(src_1_2, zero);
2611 ✗ __m512i src_hi2 = _mm512_unpackhi_epi16(src_1_2, zero);
2612
2613 ✗ __m512i src_lo_2 = _mm512_unpacklo_epi16(src_2_1, zero);
2614 ✗ __m512i src_hi_2 = _mm512_unpackhi_epi16(src_2_1, zero);
2615 ✗ __m512i src_lo2_2 = _mm512_unpacklo_epi16(src_2_2, zero);
2616 ✗ __m512i src_hi2_2 = _mm512_unpackhi_epi16(src_2_2, zero);
2617
2618 ✗ result_lo = _mm512_add_epi32(result_lo, _mm512_madd_epi16(src_lo, coeff)); // a*b + c
2619 ✗ result_hi = _mm512_add_epi32(result_hi, _mm512_madd_epi16(src_hi, coeff)); // a*b + c
2620 ✗ result_lo2 = _mm512_add_epi32(result_lo2, _mm512_madd_epi16(src_lo2, coeff)); // a*b + c
2621 ✗ result_hi2 = _mm512_add_epi32(result_hi2, _mm512_madd_epi16(src_hi2, coeff)); // a*b + c
2622
2623 ✗ result_lo_2 = _mm512_add_epi32(result_lo_2, _mm512_madd_epi16(src_lo_2, coeff)); // a*b + c
2624 ✗ result_hi_2 = _mm512_add_epi32(result_hi_2, _mm512_madd_epi16(src_hi_2, coeff)); // a*b + c
2625 ✗ result_lo2_2 = _mm512_add_epi32(result_lo2_2, _mm512_madd_epi16(src_lo2_2, coeff)); // a*b + c
2626 ✗ result_hi2_2 = _mm512_add_epi32(result_hi2_2, _mm512_madd_epi16(src_hi2_2, coeff)); // a*b + c
2627
2628 ✗ src2_ptr += src_pitch;
2629
2630 }
2631
2632 // scale back, store
2633 // shift back integer arithmetic 14 bits precision
2634 ✗ result_lo = _mm512_srai_epi32(result_lo, FPScale8bits);
2635 ✗ result_hi = _mm512_srai_epi32(result_hi, FPScale8bits);
2636 ✗ result_lo2 = _mm512_srai_epi32(result_lo2, FPScale8bits);
2637 ✗ result_hi2 = _mm512_srai_epi32(result_hi2, FPScale8bits);
2638
2639 ✗ result_lo_2 = _mm512_srai_epi32(result_lo_2, FPScale8bits);
2640 ✗ result_hi_2 = _mm512_srai_epi32(result_hi_2, FPScale8bits);
2641 ✗ result_lo2_2 = _mm512_srai_epi32(result_lo2_2, FPScale8bits);
2642 ✗ result_hi2_2 = _mm512_srai_epi32(result_hi2_2, FPScale8bits);
2643
2644 ✗ __m512i result_2x8x_uint16 = _mm512_packus_epi32(result_lo, result_hi);
2645 ✗ __m512i result2_2x8x_uint16 = _mm512_packus_epi32(result_lo2, result_hi2);
2646
2647 ✗ __m512i result_2x8x_uint16_2 = _mm512_packus_epi32(result_lo_2, result_hi_2);
2648 ✗ __m512i result2_2x8x_uint16_2 = _mm512_packus_epi32(result_lo2_2, result_hi2_2);
2649
2650 ✗ __m512i pack_1 = _mm512_permutex2var_epi64(result_2x8x_uint16, perm_idx1, result2_2x8x_uint16);
2651 ✗ __m512i pack_2 = _mm512_permutex2var_epi64(result_2x8x_uint16, perm_idx2, result2_2x8x_uint16);
2652
2653 ✗ __m512i pack_1_2 = _mm512_permutex2var_epi64(result_2x8x_uint16_2, perm_idx1, result2_2x8x_uint16_2);
2654 ✗ __m512i pack_2_2 = _mm512_permutex2var_epi64(result_2x8x_uint16_2, perm_idx2, result2_2x8x_uint16_2);
2655
2656 ✗ __m512i res = _mm512_packus_epi16(pack_1, pack_2);
2657 ✗ __m512i res_2 = _mm512_packus_epi16(pack_1_2, pack_2_2);
2658
2659 ✗ _mm512_store_si512(reinterpret_cast<__m512i*>(dst + x), res);
2660 ✗ _mm512_store_si512(reinterpret_cast<__m512i*>(dst + x + 64), res_2);
2661
2662 }
2663
2664 // 64 byte 64 pixel
2665 // no need wmod16, alignment is safe at least 32
2666 ✗ for (int x = width_mod128; x < width; x += 64) {
2667
2668 ✗ __m512i result_lo = rounder;
2669 ✗ __m512i result_hi = rounder;
2670
2671 ✗ __m512i result_lo2 = rounder;
2672 ✗ __m512i result_hi2 = rounder;
2673
2674 ✗ const uint8_t* AVS_RESTRICT src2_ptr = src_ptr + x;
2675
2676 ✗ int i = 0;
2677 ✗ for (; i < kernel_size; i++) {
2678 // Broadcast a single coefficients
2679 ✗ __m512i coeff = _mm512_set1_epi16(*reinterpret_cast<const short*>(current_coeff + i)); // 0|co|0|co|0|co|0|co 0|co|0|co|0|co|0|co
2680
2681 ✗ __m512i src_1_1 = _mm512_cvtepu8_epi16(_mm256_load_si256(reinterpret_cast<const __m256i*>(src2_ptr))); // 32x 8->16bit pixels
2682 ✗ __m512i src_1_2 = _mm512_cvtepu8_epi16(_mm256_load_si256(reinterpret_cast<const __m256i*>(src2_ptr + 32))); // 32x 8->16bit pixels
2683
2684 ✗ __m512i src_lo = _mm512_unpacklo_epi16(src_1_1, zero);
2685 ✗ __m512i src_hi = _mm512_unpackhi_epi16(src_1_1, zero);
2686
2687 ✗ __m512i src_lo2 = _mm512_unpacklo_epi16(src_1_2, zero);
2688 ✗ __m512i src_hi2 = _mm512_unpackhi_epi16(src_1_2, zero);
2689
2690 ✗ result_lo = _mm512_add_epi32(result_lo, _mm512_madd_epi16(src_lo, coeff)); // a*b + c
2691 ✗ result_hi = _mm512_add_epi32(result_hi, _mm512_madd_epi16(src_hi, coeff)); // a*b + c
2692
2693 ✗ result_lo2 = _mm512_add_epi32(result_lo2, _mm512_madd_epi16(src_lo2, coeff)); // a*b + c
2694 ✗ result_hi2 = _mm512_add_epi32(result_hi2, _mm512_madd_epi16(src_hi2, coeff)); // a*b + c
2695
2696 ✗ src2_ptr += src_pitch;
2697
2698 }
2699
2700 // scale back, store
2701 // shift back integer arithmetic 14 bits precision
2702 ✗ result_lo = _mm512_srai_epi32(result_lo, FPScale8bits);
2703 ✗ result_hi = _mm512_srai_epi32(result_hi, FPScale8bits);
2704
2705 ✗ result_lo2 = _mm512_srai_epi32(result_lo2, FPScale8bits);
2706 ✗ result_hi2 = _mm512_srai_epi32(result_hi2, FPScale8bits);
2707
2708 ✗ __m512i result_2x8x_uint16 = _mm512_packus_epi32(result_lo, result_hi);
2709 ✗ __m512i result_2x8x_uint16_2 = _mm512_packus_epi32(result_lo2, result_hi2);
2710
2711 ✗ __m512i pack_1 = _mm512_permutex2var_epi64(result_2x8x_uint16, perm_idx1, result_2x8x_uint16_2);
2712 ✗ __m512i pack_2 = _mm512_permutex2var_epi64(result_2x8x_uint16, perm_idx2, result_2x8x_uint16_2);
2713
2714 ✗ __m512i res = _mm512_packus_epi16(pack_1, pack_2);
2715
2716 ✗ _mm512_stream_si512(reinterpret_cast<__m512i*>(dst + x), res);
2717
2718 }
2719
2720 ✗ dst += dst_pitch;
2721 ✗ current_coeff += filter_size;
2722 }
2723 ✗ }
2724
2725 //uint16_t
2726 template<bool lessthan16bit>
2727 ✗ void resize_v_avx512_planar_uint16_t_w_sr(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int target_height, int bits_per_pixel)
2728 {
2729 ✗ int filter_size = program->filter_size;
2730 ✗ const short* AVS_RESTRICT current_coeff = program->pixel_coefficient;
2731
2732 ✗ const __m512i zero = _mm512_setzero_si512();
2733
2734 ✗ const int width_mod64 = (width / 64) * 64;
2735
2736 // for 16 bits only
2737 ✗ const __m512i shifttosigned = _mm512_set1_epi16(-32768);
2738 ✗ const __m512i shiftfromsigned = _mm512_set1_epi32(32768 << FPScale16bits);
2739
2740 ✗ const __m512i rounder = _mm512_set1_epi32(1 << (FPScale16bits - 1));
2741
2742 ✗ const uint16_t* src = (uint16_t*)src8;
2743 ✗ uint16_t* AVS_RESTRICT dst = (uint16_t * AVS_RESTRICT)dst8;
2744 ✗ dst_pitch = dst_pitch / sizeof(uint16_t);
2745 ✗ src_pitch = src_pitch / sizeof(uint16_t);
2746
2747 ✗ const int kernel_size = program->filter_size_real; // not the aligned
2748
2749 ✗ const int limit = (1 << bits_per_pixel) - 1;
2750 ✗ __m512i clamp_limit = _mm512_set1_epi16((short)limit); // clamp limit for <16 bits
2751
2752 ✗ for (int y = 0; y < target_height; y++) {
2753 ✗ int offset = program->pixel_offset[y];
2754 ✗ const uint16_t* src_ptr = src + offset * src_pitch;
2755
2756 // 128 byte 32 word
2757 ✗ for (int x = 0; x < width_mod64; x += 64) {
2758
2759 ✗ __m512i result_lo = rounder;
2760 ✗ __m512i result_hi = rounder;
2761
2762 ✗ __m512i result_lo_2 = rounder;
2763 ✗ __m512i result_hi_2 = rounder;
2764
2765 ✗ const uint16_t* AVS_RESTRICT src2_ptr = src_ptr + x;
2766
2767 ✗ int i = 0;
2768 ✗ for (; i < kernel_size; i++) {
2769 // Broadcast a single coefficients
2770 ✗ __m512i coeff = _mm512_set1_epi16(current_coeff[i]); // 0|co|0|co|0|co|0|co 0|co|0|co|0|co|0|co
2771
2772 ✗ __m512i src = _mm512_load_si512(reinterpret_cast<const __m512i*>(src2_ptr)); // 32x 16bit pixels
2773 ✗ __m512i src_2 = _mm512_load_si512(reinterpret_cast<const __m512i*>(src2_ptr + 32)); // 32x 16bit pixels
2774
2775 if constexpr (!lessthan16bit) {
2776 ✗ src = _mm512_add_epi16(src, shifttosigned);
2777 ✗ src_2 = _mm512_add_epi16(src_2, shifttosigned);
2778 }
2779
2780 ✗ __m512i src_lo = _mm512_unpacklo_epi16(src, zero);
2781 ✗ __m512i src_hi = _mm512_unpackhi_epi16(src, zero);
2782
2783 ✗ __m512i src_lo_2 = _mm512_unpacklo_epi16(src_2, zero);
2784 ✗ __m512i src_hi_2 = _mm512_unpackhi_epi16(src_2, zero);
2785
2786 ✗ result_lo = _mm512_add_epi32(result_lo, _mm512_madd_epi16(src_lo, coeff)); // a*b + c
2787 ✗ result_hi = _mm512_add_epi32(result_hi, _mm512_madd_epi16(src_hi, coeff)); // a*b + c
2788
2789 ✗ result_lo_2 = _mm512_add_epi32(result_lo_2, _mm512_madd_epi16(src_lo_2, coeff)); // a*b + c
2790 ✗ result_hi_2 = _mm512_add_epi32(result_hi_2, _mm512_madd_epi16(src_hi_2, coeff)); // a*b + c
2791
2792 ✗ src2_ptr += src_pitch;
2793 }
2794
2795 if constexpr (!lessthan16bit) {
2796 ✗ result_lo = _mm512_add_epi32(result_lo, shiftfromsigned);
2797 ✗ result_hi = _mm512_add_epi32(result_hi, shiftfromsigned);
2798
2799 ✗ result_lo_2 = _mm512_add_epi32(result_lo_2, shiftfromsigned);
2800 ✗ result_hi_2 = _mm512_add_epi32(result_hi_2, shiftfromsigned);
2801
2802 }
2803 // shift back integer arithmetic 13 bits precision
2804 ✗ result_lo = _mm512_srai_epi32(result_lo, FPScale16bits);
2805 ✗ result_hi = _mm512_srai_epi32(result_hi, FPScale16bits);
2806
2807 ✗ result_lo_2 = _mm512_srai_epi32(result_lo_2, FPScale16bits);
2808 ✗ result_hi_2 = _mm512_srai_epi32(result_hi_2, FPScale16bits);
2809
2810 ✗ __m512i result_2x8x_uint16 = _mm512_packus_epi32(result_lo, result_hi);
2811 ✗ __m512i result_2x8x_uint16_2 = _mm512_packus_epi32(result_lo_2, result_hi_2);
2812 if constexpr (lessthan16bit) {
2813 ✗ result_2x8x_uint16 = _mm512_min_epu16(result_2x8x_uint16, clamp_limit); // extra clamp for 10-14 bit
2814 ✗ result_2x8x_uint16_2 = _mm512_min_epu16(result_2x8x_uint16_2, clamp_limit); // extra clamp for 10-14 bit
2815 }
2816 ✗ _mm512_store_si512(reinterpret_cast<__m512i*>(dst + x), result_2x8x_uint16);
2817 ✗ _mm512_store_si512(reinterpret_cast<__m512i*>(dst + x + 32), result_2x8x_uint16_2);
2818 }
2819
2820 // last 32
2821 // 64 byte 32 word
2822 ✗ for (int x = width_mod64; x < width; x += 32) {
2823
2824 ✗ __m512i result_lo = rounder;
2825 ✗ __m512i result_hi = rounder;
2826
2827 ✗ const uint16_t* AVS_RESTRICT src2_ptr = src_ptr + x;
2828
2829 ✗ int i = 0;
2830 ✗ for (; i < kernel_size; i++) {
2831 // Broadcast a single coefficients
2832 ✗ __m512i coeff = _mm512_set1_epi16(current_coeff[i]); // 0|co|0|co|0|co|0|co 0|co|0|co|0|co|0|co
2833
2834 ✗ __m512i src = _mm512_load_si512(reinterpret_cast<const __m512i*>(src2_ptr)); // 32x 16bit pixels
2835 if constexpr (!lessthan16bit) {
2836 ✗ src = _mm512_add_epi16(src, shifttosigned);
2837 }
2838 ✗ __m512i src_lo = _mm512_unpacklo_epi16(src, zero);
2839 ✗ __m512i src_hi = _mm512_unpackhi_epi16(src, zero);
2840 ✗ result_lo = _mm512_add_epi32(result_lo, _mm512_madd_epi16(src_lo, coeff)); // a*b + c
2841 ✗ result_hi = _mm512_add_epi32(result_hi, _mm512_madd_epi16(src_hi, coeff)); // a*b + c
2842
2843 ✗ src2_ptr += src_pitch;
2844 }
2845
2846 if constexpr (!lessthan16bit) {
2847 ✗ result_lo = _mm512_add_epi32(result_lo, shiftfromsigned);
2848 ✗ result_hi = _mm512_add_epi32(result_hi, shiftfromsigned);
2849 }
2850 // shift back integer arithmetic 13 bits precision
2851 ✗ result_lo = _mm512_srai_epi32(result_lo, FPScale16bits);
2852 ✗ result_hi = _mm512_srai_epi32(result_hi, FPScale16bits);
2853
2854 ✗ __m512i result_2x8x_uint16 = _mm512_packus_epi32(result_lo, result_hi);
2855 if constexpr (lessthan16bit) {
2856 ✗ result_2x8x_uint16 = _mm512_min_epu16(result_2x8x_uint16, clamp_limit); // extra clamp for 10-14 bit
2857 }
2858 ✗ _mm512_stream_si512(reinterpret_cast<__m512i*>(dst + x), result_2x8x_uint16);
2859
2860 }
2861
2862 ✗ dst += dst_pitch;
2863 ✗ current_coeff += filter_size;
2864 }
2865 ✗ }
2866
2867 // avx512 16
2868 template void resize_v_avx512_planar_uint16_t_w_sr<false>(BYTE* dst0, const BYTE* src0, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int target_height, int bits_per_pixel);
2869 // avx512 10-14bit
2870 template void resize_v_avx512_planar_uint16_t_w_sr<true>(BYTE* dst0, const BYTE* src0, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int target_height, int bits_per_pixel);
2871
2872 //----------------------- generic horizontal avx512 float
2873
2874 // AVX512 Horizontal float
2875
2876 // Three helpers, each for processing 4 target pixels from 16, 8 and 4 source pixel/coeff pairs.
2877
2878 // Helper, _mm256_zextps128_ps256 exists only in AVX512 VL
2879 // zero-extend 128-bit float vector to 256-bit float vector
2880 AVS_FORCEINLINE static __m256 _mm256_zextps128_ps256_simul_avx(__m128 a)
2881 {
2882 // Flags defines by MSVC at /AVX512 mode
2883 // other flags: __AVX512F__, __AVX512CD__, __AVX512VL__, __AVX512BW__, __AVX512DQ__
2884 #ifdef __AVX512VL__
2885 return _mm256_zextps128_ps256(a);
2886 #else
2887 __m256 zero_v = _mm256_setzero_ps();
2888 return _mm256_insertf128_ps(zero_v, a, 0);
2889 #endif
2890 }
2891
2892 // 4 target pixels, each from 16 source pixel/coeff pair
2893 // Called only when accessing 16 source pixels and coefficients at a time is safe
2894 AVS_FORCEINLINE static void process_pix4_coeff16_h_float_core_512(
2895 const float* src,
2896 int begin1, int begin2, int begin3, int begin4,
2897 const float* current_coeff,
2898 int filter_size,
2899 __m512& result1, __m512& result2, __m512& result3, __m512& result4)
2900 {
2901 // 16 source floats for each of the four beginning source offsets
2902 ✗ __m512 data_1 = _mm512_loadu_ps(src + begin1);
2903 ✗ __m512 data_2 = _mm512_loadu_ps(src + begin2);
2904 ✗ __m512 data_3 = _mm512_loadu_ps(src + begin3);
2905 ✗ __m512 data_4 = _mm512_loadu_ps(src + begin4);
2906
2907 // 16 coefficients for each of the four output pixels
2908 ✗ __m512 coeff_1 = _mm512_loadu_ps(current_coeff); // 16 coeffs for pixel 1
2909 ✗ __m512 coeff_2 = _mm512_loadu_ps(current_coeff + 1 * filter_size); // 16 coeffs for pixel 2
2910 ✗ __m512 coeff_3 = _mm512_loadu_ps(current_coeff + 2 * filter_size); // 16 coeffs for pixel 3
2911 ✗ __m512 coeff_4 = _mm512_loadu_ps(current_coeff + 3 * filter_size); // 16 coeffs for pixel 4
2912
2913 // multiply and accumulate
2914 ✗ result1 = _mm512_fmadd_ps(data_1, coeff_1, result1);
2915 ✗ result2 = _mm512_fmadd_ps(data_2, coeff_2, result2);
2916 ✗ result3 = _mm512_fmadd_ps(data_3, coeff_3, result3);
2917 ✗ result4 = _mm512_fmadd_ps(data_4, coeff_4, result4);
2918 ✗ }
2919
2920 // 4 target pixels, each from 8 source pixel/coeff pair
2921 // Called only when accessing 8 source pixels and coefficients at a time is safe
2922 AVS_FORCEINLINE static void process_pix4_coeff8_h_float_core(
2923 const float* src,
2924 int begin1, int begin2, int begin3, int begin4,
2925 const float* current_coeff,
2926 int filter_size,
2927 __m256& result1, __m256& result2, __m256& result3, __m256& result4)
2928 {
2929 // Load 8 source floats for each of the four beginning source offsets
2930 // Load 8 coefficients for each of the four output pixels
2931 ✗ __m256 data_1 = _mm256_loadu_ps(src + begin1);
2932 ✗ __m256 coeff_1 = _mm256_load_ps(current_coeff); // 8 coeffs for pixel 1
2933 ✗ result1 = _mm256_fmadd_ps(data_1, coeff_1, result1);
2934
2935 ✗ __m256 data_2 = _mm256_loadu_ps(src + begin2);
2936 ✗ __m256 coeff_2 = _mm256_load_ps(current_coeff + 1 * filter_size); // 8 coeffs for pixel 2
2937 ✗ result2 = _mm256_fmadd_ps(data_2, coeff_2, result2);
2938
2939 ✗ __m256 data_3 = _mm256_loadu_ps(src + begin3);
2940 ✗ __m256 coeff_3 = _mm256_load_ps(current_coeff + 2 * filter_size); // 8 coeffs for pixel 3
2941 ✗ result3 = _mm256_fmadd_ps(data_3, coeff_3, result3);
2942
2943 ✗ __m256 data_4 = _mm256_loadu_ps(src + begin4);
2944 ✗ __m256 coeff_4 = _mm256_load_ps(current_coeff + 3 * filter_size); // 8 coeffs for pixel 4
2945 ✗ result4 = _mm256_fmadd_ps(data_4, coeff_4, result4);
2946 ✗ }
2947
2948 // 4 target pixels, each from 4 source pixel/coeff pair.
2949 // Called only for first iteration when results are not initialized.
2950 // Otherwise same as process_pix4_coeff8_h_float_core.
2951 AVS_FORCEINLINE static void process_pix4_coeff4_h_float_core_first(
2952 const float* src,
2953 int begin1, int begin2, int begin3, int begin4,
2954 const float* current_coeff,
2955 int filter_size,
2956 __m256& result1, __m256& result2, __m256& result3, __m256& result4)
2957 {
2958 // Pixel 1: Load, Multiply, and Zero-Extend to __m256
2959 ✗ __m128 data_1 = _mm_loadu_ps(src + begin1);
2960 ✗ __m128 coeff_1 = _mm_load_ps(current_coeff);
2961 ✗ __m128 mul_result1 = _mm_mul_ps(data_1, coeff_1);
2962 ✗ result1 = _mm256_zextps128_ps256(mul_result1); // Sets upper 128 bits to zero
2963
2964 // Pixel 2: Load, Multiply, and Zero-Extend to __m256
2965 ✗ __m128 data_2 = _mm_loadu_ps(src + begin2);
2966 ✗ __m128 coeff_2 = _mm_load_ps(current_coeff + 1 * filter_size);
2967 ✗ __m128 mul_result2 = _mm_mul_ps(data_2, coeff_2);
2968 ✗ result2 = _mm256_zextps128_ps256(mul_result2); // Sets upper 128 bits to zero
2969
2970 // Pixel 3: Load, Multiply, and Zero-Extend to __m256
2971 ✗ __m128 data_3 = _mm_loadu_ps(src + begin3);
2972 ✗ __m128 coeff_3 = _mm_load_ps(current_coeff + 2 * filter_size);
2973 ✗ __m128 mul_result3 = _mm_mul_ps(data_3, coeff_3);
2974 ✗ result3 = _mm256_zextps128_ps256(mul_result3); // Sets upper 128 bits to zero
2975
2976 // Pixel 4: Load, Multiply, and Zero-Extend to __m256
2977 ✗ __m128 data_4 = _mm_loadu_ps(src + begin4);
2978 ✗ __m128 coeff_4 = _mm_load_ps(current_coeff + 3 * filter_size);
2979 ✗ __m128 mul_result4 = _mm_mul_ps(data_4, coeff_4);
2980 ✗ result4 = _mm256_zextps128_ps256(mul_result4); // Sets upper 128 bits to zero
2981 ✗ }
2982
2983 // filtersize_hint: special: 0..4 for 4,8,16,24,32. Generic: -1
2984 // filter_size is an aligned value and always multiple of 8 (prerequisite)
2985 // Processing rules:
2986 // if filtersize_hint==0: filter size <=4, do one coeff4 step only
2987 // if filtersize_hint>=2: do 1 or 2 coeff16 steps
2988 // if filtersize_hint==1 or 3: do 1 coeff8 step (0*16 or 1*16 step done already)
2989 // if filtersize_hint==-1: unknown filter size, do 16,8 steps as possible
2990 template<bool safe_aligned_mode, int filtersize_hint>
2991 AVS_FORCEINLINE static void process_four_pixels_h_float_pix4of16_ks_4_8_16(
2992 const float* src_ptr,
2993 int begin1, int begin2, int begin3, int begin4,
2994 float* current_coeff,
2995 int filter_size,
2996 __m256& result1, __m256& result2, __m256& result3, __m256& result4,
2997 int kernel_size)
2998 {
2999
3000 // very special case: filter size <= 4
3001 if constexpr (safe_aligned_mode) {
3002 if (filtersize_hint == 0) {
3003 // Process 4 target pixels and 4 source pixels/coefficients at a time
3004 // XMM-based loop internally, but returns __m256 with upper 128 cleared
3005 // Do not assume initialized zeros in result1..4, they will be set here.
3006 process_pix4_coeff4_h_float_core_first(
3007 src_ptr + 0, begin1, begin2, begin3, begin4,
3008 current_coeff + 0,
3009 filter_size,
3010 result1, result2, result3, result4);
3011 ✗ return;
3012 }
3013 }
3014
3015 ✗ int i = 0;
3016
3017 // do by 16 coeffs until possible
3018 if (filtersize_hint == -1 || filtersize_hint >= 2) {
3019 ✗ __m512 result1_512 = _mm512_setzero_ps();
3020 ✗ __m512 result2_512 = _mm512_setzero_ps();
3021 ✗ __m512 result3_512 = _mm512_setzero_ps();
3022 ✗ __m512 result4_512 = _mm512_setzero_ps();
3023 ✗ const int ksmod16 = safe_aligned_mode ? (filter_size / 16 * 16) : (kernel_size / 16 * 16);
3024 // Process 4 target pixels and 16 source pixels/coefficients at a time (ZMM-based loop)
3025 ✗ for (; i < ksmod16; i += 16) {
3026 ✗ process_pix4_coeff16_h_float_core_512(
3027 ✗ src_ptr + i, begin1, begin2, begin3, begin4,
3028 ✗ current_coeff + i,
3029 filter_size,
3030 result1_512, result2_512, result3_512, result4_512);
3031 }
3032 // Horizontal sum reduction from __m512 to __m256
3033 ✗ result1 = _mm256_add_ps(_mm512_castps512_ps256(result1_512), _mm512_extractf32x8_ps(result1_512, 1));
3034 ✗ result2 = _mm256_add_ps(_mm512_castps512_ps256(result2_512), _mm512_extractf32x8_ps(result2_512, 1));
3035 ✗ result3 = _mm256_add_ps(_mm512_castps512_ps256(result3_512), _mm512_extractf32x8_ps(result3_512, 1));
3036 ✗ result4 = _mm256_add_ps(_mm512_castps512_ps256(result4_512), _mm512_extractf32x8_ps(result4_512, 1));
3037 }
3038
3039 // filter sizes 16 or 32 can return here
3040 if constexpr (safe_aligned_mode && (filtersize_hint == 2 || filtersize_hint == 4)) {
3041 ✗ return;
3042 }
3043
3044 if constexpr (!safe_aligned_mode) {
3045 ✗ if (i == kernel_size) return; // kernel_size is not known compile time
3046 }
3047
3048 // When to do the coeff8 step:
3049 // not safe-aligned mode: always. E.g. kernel_size == 28 -> 16 done, now 10 rest, do 8 next
3050 // filtersize_hint == -1: not-compile-time known filtersize (kernel_size / 16 * 16 done, rest follows)
3051 // filtersize_hint == 1 or 3: 0*16 or 1*16 done, now do 1*8
3052 if (!safe_aligned_mode || filtersize_hint == -1 || filtersize_hint == 1 || filtersize_hint == 3) {
3053 // 32 bytes contain 8 floats. We will use 256-bit registers (YMM).
3054 ✗ const int ksmod8 = safe_aligned_mode ? (filter_size / 8 * 8) : (kernel_size / 8 * 8);
3055
3056 // Process 4 target pixels and 8 source pixels/coefficients at a time (YMM-based loop)
3057 ✗ for (; i < ksmod8; i += 8) {
3058 ✗ process_pix4_coeff8_h_float_core(
3059 ✗ src_ptr + i, begin1, begin2, begin3, begin4,
3060 ✗ current_coeff + i,
3061 filter_size,
3062 result1, result2, result3, result4);
3063 }
3064 }
3065
3066 if constexpr (!safe_aligned_mode) {
3067 // Right edge case.
3068 // Coeffs are zero padded, reading them is no problem.
3069 // But if we read past the end of source then we can get possible NaN contamination.
3070 // Handle the remainder: 1 to 7 source/coefficient elements.
3071 // unaligned_kernel_size is used here, it's guaranteed that reading unaligned_kernel_size elements
3072 // from any pixel_offset[] is safe and ends within the source buffer.
3073 // Optional 4-2-1 processing loop.
3074
3075 ✗ if (i == kernel_size) return;
3076
3077 // --- Define Base Pointers for Source and Coefficients ---
3078 ✗ const float* src_ptr1 = src_ptr + begin1;
3079 ✗ const float* src_ptr2 = src_ptr + begin2;
3080 ✗ const float* src_ptr3 = src_ptr + begin3;
3081 ✗ const float* src_ptr4 = src_ptr + begin4;
3082
3083 ✗ float* current_coeff2 = current_coeff + 1 * filter_size;
3084 ✗ float* current_coeff3 = current_coeff + 2 * filter_size;
3085 ✗ float* current_coeff4 = current_coeff + 3 * filter_size;
3086
3087 ✗ const int ksmod4 = kernel_size / 4 * 4;
3088
3089 // -------------------------------------------------------------------
3090 // Mod 4 Block (4 elements for four pixels using __m128)
3091 // -------------------------------------------------------------------
3092 ✗ if (i < ksmod4) {
3093 // Load 4 source floats and 4 coefficients for each of the four output pixels
3094 ✗ __m128 data_1 = _mm_loadu_ps(src_ptr1 + i);
3095 ✗ __m128 coeff_1 = _mm_loadu_ps(current_coeff + i);
3096 ✗ __m128 temp_result1 = _mm_mul_ps(data_1, coeff_1);
3097
3098 ✗ __m128 data_2 = _mm_loadu_ps(src_ptr2 + i);
3099 ✗ __m128 coeff_2 = _mm_loadu_ps(current_coeff2 + i);
3100 ✗ __m128 temp_result2 = _mm_mul_ps(data_2, coeff_2);
3101
3102 ✗ __m128 data_3 = _mm_loadu_ps(src_ptr3 + i);
3103 ✗ __m128 coeff_3 = _mm_loadu_ps(current_coeff3 + i);
3104 ✗ __m128 temp_result3 = _mm_mul_ps(data_3, coeff_3);
3105
3106 ✗ __m128 data_4 = _mm_loadu_ps(src_ptr4 + i);
3107 ✗ __m128 coeff_4 = _mm_loadu_ps(current_coeff4 + i);
3108 ✗ __m128 temp_result4 = _mm_mul_ps(data_4, coeff_4);
3109
3110 // --- Accumulate 128-bit results into 256-bit registers ---
3111 // Note: Since we are using __m256, we must zero the high 128-bits before insertion/addition.
3112
3113 ✗ result1 = _mm256_add_ps(result1, _mm256_zextps128_ps256(temp_result1));
3114 ✗ result2 = _mm256_add_ps(result2, _mm256_zextps128_ps256(temp_result2));
3115 ✗ result3 = _mm256_add_ps(result3, _mm256_zextps128_ps256(temp_result3));
3116 ✗ result4 = _mm256_add_ps(result4, _mm256_zextps128_ps256(temp_result4));
3117
3118 ✗ i += 4;
3119 ✗ if (i == kernel_size) return;
3120 }
3121
3122 ✗ const int ksmod2 = kernel_size / 2 * 2;
3123
3124 // -------------------------------------------------------------------
3125 // New Mod 2 Block (2 elements for four pixels using __m128)
3126 // -------------------------------------------------------------------
3127 ✗ if (i < ksmod2) {
3128 // We only need to load 2 elements (4 floats) for the __m128 load,
3129 // but the low 2 elements of the __m128 register are used.
3130 // Since we use the scalar accumulation method, we load 4, but only the
3131 // first 2 elements will hold non-zero data (or load 2, and rely on
3132 // the two __m128 registers to contain the result).
3133
3134 // Let's stick to using the low 2 elements of __m128 for 2 elements.
3135
3136 // Load 2 source floats and 2 coefficients for each of the four output pixels
3137 ✗ __m128 data_1 = _mm_castpd_ps(_mm_load_sd(reinterpret_cast<const double*>(src_ptr1 + i))); // Load 2 floats (double)
3138 ✗ __m128 coeff_1 = _mm_castpd_ps(_mm_load_sd(reinterpret_cast<const double*>(current_coeff + i)));
3139 ✗ __m128 temp_result1 = _mm_mul_ps(data_1, coeff_1);
3140
3141 ✗ __m128 data_2 = _mm_castpd_ps(_mm_load_sd(reinterpret_cast<const double*>(src_ptr2 + i)));
3142 ✗ __m128 coeff_2 = _mm_castpd_ps(_mm_load_sd(reinterpret_cast<const double*>(current_coeff2 + i)));
3143 ✗ __m128 temp_result2 = _mm_mul_ps(data_2, coeff_2);
3144
3145 ✗ __m128 data_3 = _mm_castpd_ps(_mm_load_sd(reinterpret_cast<const double*>(src_ptr3 + i)));
3146 ✗ __m128 coeff_3 = _mm_castpd_ps(_mm_load_sd(reinterpret_cast<const double*>(current_coeff3 + i)));
3147 ✗ __m128 temp_result3 = _mm_mul_ps(data_3, coeff_3);
3148
3149 ✗ __m128 data_4 = _mm_castpd_ps(_mm_load_sd(reinterpret_cast<const double*>(src_ptr4 + i)));
3150 ✗ __m128 coeff_4 = _mm_castpd_ps(_mm_load_sd(reinterpret_cast<const double*>(current_coeff4 + i)));
3151 ✗ __m128 temp_result4 = _mm_mul_ps(data_4, coeff_4);
3152
3153 ✗ result1 = _mm256_add_ps(result1, _mm256_zextps128_ps256(temp_result1));
3154 ✗ result2 = _mm256_add_ps(result2, _mm256_zextps128_ps256(temp_result2));
3155 ✗ result3 = _mm256_add_ps(result3, _mm256_zextps128_ps256(temp_result3));
3156 ✗ result4 = _mm256_add_ps(result4, _mm256_zextps128_ps256(temp_result4));
3157
3158 ✗ i += 2;
3159 ✗ if (i == kernel_size) return;
3160 }
3161
3162 // -------------------------------------------------------------------
3163 // Fallback Scalar Operation (1 element remaining)
3164 // -------------------------------------------------------------------
3165 ✗ if (i < kernel_size) {
3166
3167 // Optimized scalar loop for the single remaining element
3168 ✗ float final_scalar1 = src_ptr1[i] * current_coeff[i];
3169 ✗ float final_scalar2 = src_ptr2[i] * current_coeff2[i];
3170 ✗ float final_scalar3 = src_ptr3[i] * current_coeff3[i];
3171 ✗ float final_scalar4 = src_ptr4[i] * current_coeff4[i];
3172
3173 ✗ __m128 s1_128 = _mm_set_ss(final_scalar1);
3174 ✗ __m128 s2_128 = _mm_set_ss(final_scalar2);
3175 ✗ __m128 s3_128 = _mm_set_ss(final_scalar3);
3176 ✗ __m128 s4_128 = _mm_set_ss(final_scalar4);
3177
3178 ✗ result1 = _mm256_add_ps(result1, _mm256_zextps128_ps256(s1_128));
3179 ✗ result2 = _mm256_add_ps(result2, _mm256_zextps128_ps256(s2_128));
3180 ✗ result3 = _mm256_add_ps(result3, _mm256_zextps128_ps256(s3_128));
3181 ✗ result4 = _mm256_add_ps(result4, _mm256_zextps128_ps256(s4_128));
3182
3183 // i is now equal to kernel_size (i++)
3184 }
3185 }
3186 }
3187
3188
3189 template<bool is_safe, int filtersize_hint>
3190 AVS_FORCEINLINE static void process_sixteen_pixels_h_float_pix16_sub4_ks_4_8_16(
3191 const float* src, int x, float* current_coeff_base,
3192 int filter_size, // 8, 16, 24, 32 are quasi-constexpr here, others not compile-time known but still aligned to 8
3193 float* dst,
3194 ResamplingProgram* program)
3195 {
3196 ✗ assert(program->filter_size_alignment >= 8);
3197
3198 ✗ float* current_coeff = current_coeff_base + x * filter_size;
3199 ✗ const int unaligned_kernel_size = program->filter_size_real;
3200 ✗ const __m256 zero256 = _mm256_setzero_ps();
3201
3202 // --- Block 1: Pixels 0, 1, 2, 3 ---
3203 ✗ __m256 result0 = zero256;
3204 ✗ __m256 result1 = zero256;
3205 ✗ __m256 result2 = zero256;
3206 ✗ __m256 result3 = zero256;
3207
3208 ✗ int begin0 = program->pixel_offset[x + 0];
3209 ✗ int begin1 = program->pixel_offset[x + 1];
3210 ✗ int begin2 = program->pixel_offset[x + 2];
3211 ✗ int begin3 = program->pixel_offset[x + 3];
3212
3213 process_four_pixels_h_float_pix4of16_ks_4_8_16<is_safe, filtersize_hint>(
3214 src, begin0, begin1, begin2, begin3, current_coeff, filter_size,
3215 result0, result1, result2, result3, unaligned_kernel_size);
3216 ✗ current_coeff += 4 * filter_size;
3217
3218 // --- Block 2: Pixels 4, 5, 6, 7 ---
3219 ✗ __m256 result4 = zero256;
3220 ✗ __m256 result5 = zero256;
3221 ✗ __m256 result6 = zero256;
3222 ✗ __m256 result7 = zero256;
3223
3224 ✗ int begin4 = program->pixel_offset[x + 4];
3225 ✗ int begin5 = program->pixel_offset[x + 5];
3226 ✗ int begin6 = program->pixel_offset[x + 6];
3227 ✗ int begin7 = program->pixel_offset[x + 7];
3228
3229 process_four_pixels_h_float_pix4of16_ks_4_8_16<is_safe, filtersize_hint>(
3230 src, begin4, begin5, begin6, begin7, current_coeff, filter_size,
3231 result4, result5, result6, result7, unaligned_kernel_size);
3232 ✗ current_coeff += 4 * filter_size;
3233
3234 // ---------------------------------------------------------------------------
3235 // REDUCTION FOR PIXELS 0-7 (Result256_low)
3236 // ---------------------------------------------------------------------------
3237
3238 // Round 1: Reduce pairs (8 vectors -> 4 vectors)
3239 ✗ __m256 sum01 = _mm256_hadd_ps(result0, result1);
3240 ✗ __m256 sum23 = _mm256_hadd_ps(result2, result3);
3241 ✗ __m256 sum45 = _mm256_hadd_ps(result4, result5);
3242 ✗ __m256 sum67 = _mm256_hadd_ps(result6, result7);
3243
3244 // Round 2: Reduce quads (4 vectors -> 2 vectors)
3245 ✗ __m256 sum0123 = _mm256_hadd_ps(sum01, sum23);
3246 ✗ __m256 sum4567 = _mm256_hadd_ps(sum45, sum67);
3247
3248 // Round 3: Final Merge (Add Lower 128-bit to Upper 128-bit)
3249 ✗ __m128 lo_0123 = _mm256_castps256_ps128(sum0123);
3250 ✗ __m128 lo_4567 = _mm256_castps256_ps128(sum4567);
3251 ✗ __m256 result_lo = _mm256_insertf128_ps(_mm256_castps128_ps256(lo_0123), lo_4567, 1);
3252
3253 ✗ __m128 hi_0123 = _mm256_extractf128_ps(sum0123, 1);
3254 ✗ __m128 hi_4567 = _mm256_extractf128_ps(sum4567, 1);
3255 ✗ __m256 result_hi = _mm256_insertf128_ps(_mm256_castps128_ps256(hi_0123), hi_4567, 1);
3256
3257 // Assemble the Low 256-bit result (Pixels 0-7)
3258 ✗ __m256 result256_low = _mm256_add_ps(result_lo, result_hi);
3259
3260
3261 // --- Block 3: Pixels 8, 9, 10, 11 ---
3262 ✗ __m256 result8 = zero256;
3263 ✗ __m256 result9 = zero256;
3264 ✗ __m256 result10 = zero256;
3265 ✗ __m256 result11 = zero256;
3266
3267 ✗ int begin8 = program->pixel_offset[x + 8];
3268 ✗ int begin9 = program->pixel_offset[x + 9];
3269 ✗ int begin10 = program->pixel_offset[x + 10];
3270 ✗ int begin11 = program->pixel_offset[x + 11];
3271
3272 process_four_pixels_h_float_pix4of16_ks_4_8_16<is_safe, filtersize_hint>(
3273 src, begin8, begin9, begin10, begin11, current_coeff, filter_size,
3274 result8, result9, result10, result11, unaligned_kernel_size);
3275 ✗ current_coeff += 4 * filter_size;
3276
3277 // --- Block 4: Pixels 12, 13, 14, 15 ---
3278 ✗ __m256 result12 = zero256;
3279 ✗ __m256 result13 = zero256;
3280 ✗ __m256 result14 = zero256;
3281 ✗ __m256 result15 = zero256;
3282
3283 ✗ int begin12 = program->pixel_offset[x + 12];
3284 ✗ int begin13 = program->pixel_offset[x + 13];
3285 ✗ int begin14 = program->pixel_offset[x + 14];
3286 ✗ int begin15 = program->pixel_offset[x + 15];
3287
3288 process_four_pixels_h_float_pix4of16_ks_4_8_16<is_safe, filtersize_hint>(
3289 src, begin12, begin13, begin14, begin15, current_coeff, filter_size,
3290 result12, result13, result14, result15, unaligned_kernel_size);
3291
3292
3293 // ---------------------------------------------------------------------------
3294 // REDUCTION FOR PIXELS 8-15 (Result256_high)
3295 // ---------------------------------------------------------------------------
3296
3297 // Round 1: Reduce pairs (8 vectors -> 4 vectors)
3298 ✗ __m256 sum89 = _mm256_hadd_ps(result8, result9);
3299 ✗ __m256 sum1011 = _mm256_hadd_ps(result10, result11);
3300 ✗ __m256 sum1213 = _mm256_hadd_ps(result12, result13);
3301 ✗ __m256 sum1415 = _mm256_hadd_ps(result14, result15);
3302
3303 // Round 2: Reduce quads (4 vectors -> 2 vectors)
3304 ✗ __m256 sum8_11 = _mm256_hadd_ps(sum89, sum1011);
3305 ✗ __m256 sum12_15 = _mm256_hadd_ps(sum1213, sum1415);
3306
3307 // Round 3: Final Merge (Add Lower 128-bit to Upper 128-bit)
3308 ✗ __m128 lo_8_11 = _mm256_castps256_ps128(sum8_11);
3309 ✗ __m128 lo_12_15 = _mm256_castps256_ps128(sum12_15);
3310 ✗ __m256 result_lo_high = _mm256_insertf128_ps(_mm256_castps128_ps256(lo_8_11), lo_12_15, 1);
3311
3312 ✗ __m128 hi_8_11 = _mm256_extractf128_ps(sum8_11, 1);
3313 ✗ __m128 hi_12_15 = _mm256_extractf128_ps(sum12_15, 1);
3314 ✗ __m256 result_hi_high = _mm256_insertf128_ps(_mm256_castps128_ps256(hi_8_11), hi_12_15, 1);
3315
3316 // Assemble the High 256-bit result (Pixels 8-15)
3317 ✗ __m256 result256_high = _mm256_add_ps(result_lo_high, result_hi_high);
3318
3319 // ---------------------------------------------------------------------------
3320 // Stream the two 256-bit results
3321 // ---------------------------------------------------------------------------
3322 ✗ _mm256_stream_ps(reinterpret_cast<float*>(dst + x), result256_low);
3323 ✗ _mm256_stream_ps(reinterpret_cast<float*>(dst + x + 8), result256_high);
3324 ✗ }
3325
3326 // filtersizealigned8: special: 0, 1..4, Generic : -1
3327 template<int filtersize_hint>
3328 ✗ static void internal_resizer_h_avx512_generic_float_pix16_sub4_ks_4_8_16(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) {
3329 AVS_UNUSED(bits_per_pixel);
3330 // filter_size is aligned to 8 (prerequisite), contrary that we have a special case for filter size <=4
3331
3332 // We note that when template is used, filter_size is quasi-constexpr if filtersize_hint != -1.
3333 // When filtersize_hint == -1, then program->filter_size is aligned to 8 anyway, but not known at compile time.
3334 ✗ const int filter_size =
3335 filtersize_hint == 0 ? 8 : // though we'll optimize for 4 internally, coeff buffer is still allocated for 8
3336 (filtersize_hint >= 1) ? filtersize_hint * 8 : program->filter_size; // this latter is always aligned to 8 as well
3337
3338 ✗ const float* src = (float*)src8;
3339 ✗ float* dst = (float*)dst8;
3340 ✗ dst_pitch = dst_pitch / sizeof(float);
3341 ✗ src_pitch = src_pitch / sizeof(float);
3342
3343 ✗ constexpr int PIXELS_AT_A_TIME = 16;
3344 // Align safe zone to 16 pixels
3345 ✗ const int w_safe_mod16 = (program->safelimit_16_pixels.overread_possible ? program->safelimit_16_pixels.source_overread_beyond_targetx : width) / PIXELS_AT_A_TIME * PIXELS_AT_A_TIME;
3346
3347 ✗ for (int y = 0; y < height; y++) {
3348 ✗ float* current_coeff_base = program->pixel_coefficient_float;
3349
3350 // Process safe aligned pixels
3351 ✗ for (int x = 0; x < w_safe_mod16; x += PIXELS_AT_A_TIME) {
3352 process_sixteen_pixels_h_float_pix16_sub4_ks_4_8_16<true, filtersize_hint>(src, x, current_coeff_base, filter_size, dst, program);
3353 }
3354
3355 // Process up to the actual kernel size (unsafe zone)
3356 ✗ for (int x = w_safe_mod16; x < width; x += PIXELS_AT_A_TIME) {
3357 process_sixteen_pixels_h_float_pix16_sub4_ks_4_8_16<false, filtersize_hint>(src, x, current_coeff_base, filter_size, dst, program);
3358 }
3359
3360 ✗ dst += dst_pitch;
3361 ✗ src += src_pitch;
3362 }
3363 ✗ }
3364
3365 // Winner implementation: resizer_h_avx512_generic_float_pix16_sub4_ks_4_8_16;
3366 // Other variants kept for reference, speed tested.
3367 // Main test dimensions: pixels per cycle: 8,16,32 (pixX); sub-loops: 2,4,8 (subX); aligned filter sizes (ksX): 4, 8,16
3368 // resizer_h_avx512_generic_float_pix8_sub8_ks16;
3369 // resizer_h_avx512_generic_float_pix16_sub16_ks8;
3370 // resizer_h_avx512_generic_float_pix32_sub8_ks8;
3371 // resizer_h_avx2_generic_float_pix8_sub2_ks8; // like AVX2 version resizer_h_avx2_generic_float
3372 // resizer_h_avx512_generic_float_pix8_sub2_ks8; // like AVX2 version with minor differences
3373 // resizer_h_avx512_generic_float_pix8_sub4_ks8;
3374 // resizer_h_avx512_generic_float_pix16_sub4_ks4;
3375 // resizer_h_avx512_generic_float_pix16_sub4_ks8;
3376
3377 // Features of the chosen implementation:
3378 // - 16 pixels per cycle
3379 // - sub-loop 4 pixels per loop
3380 // - filter size is aligned to 8 (prerequisite)
3381 // - Special cases for aligned filter sizes 4,8,16,24,32
3382 // - Depending on the filter size, calculates in chunks of 16, then 8, then 4 source pixels and coeffs at a time.
3383 ✗ void resizer_h_avx512_generic_float_pix16_sub4_ks_4_8_16(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) {
3384 ✗ const int filter_size = program->filter_size;
3385 // Expected alignment
3386 ✗ assert(program->filter_size_alignment >= 8);
3387
3388 // Dispatcher template now supports filter_size aligned to 8 (8, 16, 24, 32) and a special case for <=4
3389 // Larger filter sizes will use the generic method (-1) which still benefit from 16-8-4 coeff processing blocks.
3390 ✗ if (filter_size == 1 * 8)
3391 ✗ if (program->filter_size_real <= 4)
3392 ✗ internal_resizer_h_avx512_generic_float_pix16_sub4_ks_4_8_16<0>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel); // Internally optimized for 4
3393 else
3394 ✗ internal_resizer_h_avx512_generic_float_pix16_sub4_ks_4_8_16<1>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel); // Internally optimized for 8
3395 ✗ else if (filter_size == 2 * 8) // Internally optimized for 16
3396 ✗ internal_resizer_h_avx512_generic_float_pix16_sub4_ks_4_8_16<2>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
3397 ✗ else if (filter_size == 3 * 8) // Internally optimized for 16+8
3398 ✗ internal_resizer_h_avx512_generic_float_pix16_sub4_ks_4_8_16<3>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
3399 ✗ else if (filter_size == 4 * 8) // Internally optimized for 2*16
3400 ✗ internal_resizer_h_avx512_generic_float_pix16_sub4_ks_4_8_16<4>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
3401 else // -1: basic method, use program->filter_size, internally optimized for calculating coeffs in N*16 + 8 + 4 + 2 + 1 blocks
3402 ✗ internal_resizer_h_avx512_generic_float_pix16_sub4_ks_4_8_16<-1>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
3403 ✗ }
3404
3405 // Horizontals uint8
3406
3407 // uint8_t h pretransposed_coeffs _base wrappers
3408
3409 ✗ void resize_h_planar_uint8_avx512_permutex_vstripe_mpz_ks4_pretransposed_coeffs_base(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) {
3410 // template parameter false: no VNNI, base AVX512 madd
3411 ✗ resize_h_planar_uint8_avx512_permutex_vstripe_mpz_ks4_pretransposed_coeffs_internal<false>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
3412 ✗ }
3413 ✗ void resize_h_planar_uint8_avx512_permutex_vstripe_mpz_ks8_pretransposed_coeffs_base(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) {
3414 ✗ resize_h_planar_uint8_avx512_permutex_vstripe_mpz_ks8_pretransposed_coeffs_internal<false>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
3415 ✗ }
3416 ✗ void resize_h_planar_uint8_avx512_permutex_vstripe_2s32_ks8_pretransposed_coeffs_base(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) {
3417 ✗ resize_h_planar_uint8_avx512_permutex_vstripe_2s32_ks8_pretransposed_coeffs_internal<false>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
3418 ✗ }
3419 ✗ void resize_h_planar_uint8_avx512_permutex_vstripe_mpz_ks16_pretransposed_coeffs_base(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) {
3420 ✗ resize_h_planar_uint8_avx512_permutex_vstripe_mpz_ks16_pretransposed_coeffs_internal<false>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
3421 ✗ }
3422 ✗ void resize_h_planar_uint8_avx512_permutex_vstripe_mpz_2s32_ks64_pretransposed_coeffs_base(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) {
3423 ✗ resize_h_planar_uint8_avx512_permutex_vstripe_mpz_2s32_ks64_pretransposed_coeffs_internal<false>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
3424 ✗ }
3425
3426 // uint16_t h pretransposed_coeffs _base wrappers
3427
3428 template<bool lessthan16bit>
3429 ✗ void resize_h_planar_uint16_avx512_permutex_vstripe_mp_2s32_ks4_pretransposed_coeffs_base(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) {
3430 ✗ resize_h_planar_uint16_avx512_permutex_vstripe_mp_2s32_ks4_pretransposed_coeffs_internal<lessthan16bit, false>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
3431 ✗ }
3432 template<bool lessthan16bit>
3433 ✗ void resize_h_planar_uint16_avx512_permutex_vstripe_mp_2s32_ks8_pretransposed_coeffs_base(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) {
3434 ✗ resize_h_planar_uint16_avx512_permutex_vstripe_mp_2s32_ks8_pretransposed_coeffs_internal<lessthan16bit, false>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
3435 ✗ }
3436 template<bool lessthan16bit>
3437 ✗ void resize_h_planar_uint16_avx512_permutex_vstripe_mp_4s16_ks8_pretransposed_coeffs_base(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) {
3438 ✗ resize_h_planar_uint16_avx512_permutex_vstripe_mp_4s16_ks8_pretransposed_coeffs_internal<lessthan16bit, false>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
3439 ✗ }
3440 template<bool lessthan16bit>
3441 ✗ void resize_h_planar_uint16_avx512_permutex_vstripe_mp_ks16_pretransposed_coeffs_base(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) {
3442 ✗ resize_h_planar_uint16_avx512_permutex_vstripe_mp_ks16_pretransposed_coeffs_internal<lessthan16bit, false>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
3443 ✗ }
3444 template<bool lessthan16bit>
3445 ✗ void resize_h_planar_uint16_avx512_permutex_vstripe_mp_4s16_ks48_pretransposed_coeffs_base(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) {
3446 ✗ resize_h_planar_uint16_avx512_permutex_vstripe_mp_4s16_ks48_pretransposed_coeffs_internal<lessthan16bit, false>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
3447 ✗ }
3448
3449 // Explicit template instantiations for pretransposed uint16 _base variants
3450 template void resize_h_planar_uint16_avx512_permutex_vstripe_mp_2s32_ks4_pretransposed_coeffs_base<false>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
3451 template void resize_h_planar_uint16_avx512_permutex_vstripe_mp_2s32_ks4_pretransposed_coeffs_base<true>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
3452 template void resize_h_planar_uint16_avx512_permutex_vstripe_mp_2s32_ks8_pretransposed_coeffs_base<false>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
3453 template void resize_h_planar_uint16_avx512_permutex_vstripe_mp_2s32_ks8_pretransposed_coeffs_base<true>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
3454 template void resize_h_planar_uint16_avx512_permutex_vstripe_mp_4s16_ks8_pretransposed_coeffs_base<false>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
3455 template void resize_h_planar_uint16_avx512_permutex_vstripe_mp_4s16_ks8_pretransposed_coeffs_base<true>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
3456 template void resize_h_planar_uint16_avx512_permutex_vstripe_mp_ks16_pretransposed_coeffs_base<false>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
3457 template void resize_h_planar_uint16_avx512_permutex_vstripe_mp_ks16_pretransposed_coeffs_base<true>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
3458 template void resize_h_planar_uint16_avx512_permutex_vstripe_mp_4s16_ks48_pretransposed_coeffs_base<false>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
3459 template void resize_h_planar_uint16_avx512_permutex_vstripe_mp_4s16_ks48_pretransposed_coeffs_base<true>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
3460
3461 // fixed_kernel_size == 0 marks: "not fixed".
3462 ✗ void resize_prepare_coeffs_AVX512_H(ResamplingProgram* p, IScriptEnvironment* env, int iSamplesInTheGroup, int iGroupsCount, int fixed_kernel_size) {
3463 // note: filter_size_real was the max(kernel_sizes[])
3464 ✗ int filter_size_aligned = AlignNumber(p->filter_size_real, p->filter_size_alignment);
3465 // FIXME: really this needs to be dynamic based on SIMD used in resizer
3466
3467 ✗ int target_size_aligned = AlignNumber(p->target_size, ALIGN_RESIZER_TARGET_SIZE);
3468
3469 // align target_size to X units to allow safe, up to X pixels/cycle in H resizers.
3470 // also, this is the coeff table Y-size.
3471 // e.g. ALIGN_RESIZER_TARGET_SIZE = 64 allows to access coefficient table elements at
3472 // current_coeff + filter_size * 63, if we step current_coeff by 64 * filter_size
3473 ✗ p->target_size_alignment = ALIGN_RESIZER_TARGET_SIZE;
3474
3475 // Common variables for both float and integer paths
3476 ✗ void* SIMD_coeff = nullptr;
3477 ✗ size_t element_size = 0;
3478
3479 // allocate for a larger target_size area and nullify the coeffs.
3480 // Even between target_size and target_size_aligned.
3481 ✗ if (p->bits_per_pixel == 32) {
3482 ✗ element_size = sizeof(float);
3483 ✗ SIMD_coeff = env->Allocate(element_size * target_size_aligned * filter_size_aligned, 64, AVS_NORMAL_ALLOC);
3484 ✗ if (!SIMD_coeff) {
3485 ✗ env->Free(SIMD_coeff);
3486 ✗ env->ThrowError("Could not reserve memory in a resampler.");
3487 }
3488 ✗ std::fill_n((float*)SIMD_coeff, target_size_aligned * filter_size_aligned, 0.0f);
3489 }
3490 else {
3491 ✗ element_size = sizeof(short);
3492 ✗ SIMD_coeff = env->Allocate(element_size * target_size_aligned * filter_size_aligned, 64, AVS_NORMAL_ALLOC);
3493 ✗ if (!SIMD_coeff) {
3494 ✗ env->Free(SIMD_coeff);
3495 ✗ env->ThrowError("Could not reserve memory in a resampler.");
3496 }
3497 ✗ memset(SIMD_coeff, 0, element_size * target_size_aligned * filter_size_aligned);
3498 }
3499
3500 ✗ int iSamplesAtATime = iSamplesInTheGroup * iGroupsCount;
3501
3502 // Reset current_coeff for the start of the stripe (points to start of row's coeffs)
3503 ✗ const short* AVS_RESTRICT current_coeff = p->pixel_coefficient;
3504
3505 ✗ const int filter_size_padded = p->filter_size; // aligned, practically the coeff table stride, always mod2 ?
3506 ✗ const int filter_size_real = p->filter_size_real;
3507 ✗ int filter_size_to_process = filter_size_real;
3508 ✗ if ((filter_size_real / 2 * 2) != filter_size_real) filter_size_to_process++; // add last zero coeffs to unpack hi/lo, they must present in zero-padded resampling program
3509
3510 ✗ short* dst = (short*)SIMD_coeff;
3511
3512 // Process coefficients - common code for both types
3513 ✗ for (int x = 0; x < p->target_size; x += iSamplesAtATime)
3514 {
3515 // clamp j so the last partial group doesn't overread pixel_coefficient
3516 ✗ const int avail = std::min(iSamplesAtATime, p->target_size - x);
3517 ✗ auto gc = [&](int j, int ki) -> short {
3518 ✗ return *(current_coeff + filter_size_padded * std::min(j, avail - 1) + ki);
3519 ✗ };
3520 // process by 2 rows because madd/dp can only make FMA from 2 unpacked uint16 pairs
3521 // Specific unrolled kernels (ks4/ks8/ks16) read a fixed number of taps per group of x.
3522 // They advance current_coeff_SIMD by an appropriate fixed amount, regardless of
3523 // filter_size_real. The coefficient table must be padded to that same fixed width
3524 // (fixed_kernel_size) or else discrapancy happens after the first x group.
3525 // Extra iterations access zero-padded taps and produce zero coefficient pairs — harmless.
3526 // For variable loop kernels like ks48/ks64 (signalled as fixed_kernel_size==0) advance
3527 // current_coeff_SIMD by filter_size_real*2 (pairs), so they must store exactly filter_size_to_process
3528 // pairs, contrary to the previous case, padding to a fixed width would cause coeff desynchronization
3529 // after the first x group.
3530 ✗ const int i_limit = (fixed_kernel_size > 0) ? fixed_kernel_size : filter_size_to_process;
3531 ✗ for (int i = 0; i < i_limit; i += 2)
3532 {
3533 ✗ if (iSamplesAtATime == 64) // 2 groups of 32 coeffs for columns 0..31 and 32..63
3534 {
3535 // use slow C-gathering, it is only once per filter init
3536 // first row
3537 ✗ const __m512i coef_rN_0_31 = _mm512_set_epi16(
3538 ✗ gc(31,i+0),gc(30,i+0),gc(29,i+0),gc(28,i+0),gc(27,i+0),gc(26,i+0),gc(25,i+0),gc(24,i+0),
3539 ✗ gc(23,i+0),gc(22,i+0),gc(21,i+0),gc(20,i+0),gc(19,i+0),gc(18,i+0),gc(17,i+0),gc(16,i+0),
3540 ✗ gc(15,i+0),gc(14,i+0),gc(13,i+0),gc(12,i+0),gc(11,i+0),gc(10,i+0),gc( 9,i+0),gc( 8,i+0),
3541 ✗ gc( 7,i+0),gc( 6,i+0),gc( 5,i+0),gc( 4,i+0),gc( 3,i+0),gc( 2,i+0),gc( 1,i+0),gc( 0,i+0)
3542 );
3543 ✗ const __m512i coef_rN_32_63 = _mm512_set_epi16(
3544 ✗ gc(63,i+0),gc(62,i+0),gc(61,i+0),gc(60,i+0),gc(59,i+0),gc(58,i+0),gc(57,i+0),gc(56,i+0),
3545 ✗ gc(55,i+0),gc(54,i+0),gc(53,i+0),gc(52,i+0),gc(51,i+0),gc(50,i+0),gc(49,i+0),gc(48,i+0),
3546 ✗ gc(47,i+0),gc(46,i+0),gc(45,i+0),gc(44,i+0),gc(43,i+0),gc(42,i+0),gc(41,i+0),gc(40,i+0),
3547 ✗ gc(39,i+0),gc(38,i+0),gc(37,i+0),gc(36,i+0),gc(35,i+0),gc(34,i+0),gc(33,i+0),gc(32,i+0)
3548 );
3549
3550 // second row
3551 ✗ const __m512i coef_rNp1_0_31 = _mm512_set_epi16(
3552 ✗ gc(31,i+1),gc(30,i+1),gc(29,i+1),gc(28,i+1),gc(27,i+1),gc(26,i+1),gc(25,i+1),gc(24,i+1),
3553 ✗ gc(23,i+1),gc(22,i+1),gc(21,i+1),gc(20,i+1),gc(19,i+1),gc(18,i+1),gc(17,i+1),gc(16,i+1),
3554 ✗ gc(15,i+1),gc(14,i+1),gc(13,i+1),gc(12,i+1),gc(11,i+1),gc(10,i+1),gc( 9,i+1),gc( 8,i+1),
3555 ✗ gc( 7,i+1),gc( 6,i+1),gc( 5,i+1),gc( 4,i+1),gc( 3,i+1),gc( 2,i+1),gc( 1,i+1),gc( 0,i+1)
3556 );
3557 ✗ const __m512i coef_rNp1_32_63 = _mm512_set_epi16(
3558 ✗ gc(63,i+1),gc(62,i+1),gc(61,i+1),gc(60,i+1),gc(59,i+1),gc(58,i+1),gc(57,i+1),gc(56,i+1),
3559 ✗ gc(55,i+1),gc(54,i+1),gc(53,i+1),gc(52,i+1),gc(51,i+1),gc(50,i+1),gc(49,i+1),gc(48,i+1),
3560 ✗ gc(47,i+1),gc(46,i+1),gc(45,i+1),gc(44,i+1),gc(43,i+1),gc(42,i+1),gc(41,i+1),gc(40,i+1),
3561 ✗ gc(39,i+1),gc(38,i+1),gc(37,i+1),gc(36,i+1),gc(35,i+1),gc(34,i+1),gc(33,i+1),gc(32,i+1)
3562 );
3563
3564 // unpack hi/lo
3565 ✗ const __m512i coef_rNrNp1_0_31lo = _mm512_unpacklo_epi16(coef_rN_0_31, coef_rNp1_0_31);
3566 ✗ const __m512i coef_rNrNp1_0_31hi = _mm512_unpackhi_epi16(coef_rN_0_31, coef_rNp1_0_31);
3567
3568 ✗ const __m512i coef_rNrNp1_32_63lo = _mm512_unpacklo_epi16(coef_rN_32_63, coef_rNp1_32_63);
3569 ✗ const __m512i coef_rNrNp1_32_63hi = _mm512_unpackhi_epi16(coef_rN_32_63, coef_rNp1_32_63);
3570
3571 // store (as pairs of rows for 0..31 and 32..63). May be better make array of 4-members structure ?
3572 _mm512_store_si512(reinterpret_cast<__m512i*>(dst), coef_rNrNp1_0_31lo);
3573 ✗ _mm512_store_si512(reinterpret_cast<__m512i*>(dst + 32), coef_rNrNp1_0_31hi); // in count of shorts
3574 ✗ _mm512_store_si512(reinterpret_cast<__m512i*>(dst + 64), coef_rNrNp1_32_63lo);
3575 ✗ _mm512_store_si512(reinterpret_cast<__m512i*>(dst + 96), coef_rNrNp1_32_63hi);
3576
3577 ✗ dst += 128;
3578 }
3579 ✗ else if (iSamplesAtATime == 32) // 1 group of 32 coeffs for columns 0..31
3580 {
3581 // use slow C-gathering, it is only once per filter init
3582 // first row
3583 ✗ const __m512i coef_rN_0_31 = _mm512_set_epi16(
3584 ✗ gc(31,i+0),gc(30,i+0),gc(29,i+0),gc(28,i+0),gc(27,i+0),gc(26,i+0),gc(25,i+0),gc(24,i+0),
3585 ✗ gc(23,i+0),gc(22,i+0),gc(21,i+0),gc(20,i+0),gc(19,i+0),gc(18,i+0),gc(17,i+0),gc(16,i+0),
3586 ✗ gc(15,i+0),gc(14,i+0),gc(13,i+0),gc(12,i+0),gc(11,i+0),gc(10,i+0),gc( 9,i+0),gc( 8,i+0),
3587 ✗ gc( 7,i+0),gc( 6,i+0),gc( 5,i+0),gc( 4,i+0),gc( 3,i+0),gc( 2,i+0),gc( 1,i+0),gc( 0,i+0)
3588 );
3589
3590 // second row
3591 ✗ const __m512i coef_rNp1_0_31 = _mm512_set_epi16(
3592 ✗ gc(31,i+1),gc(30,i+1),gc(29,i+1),gc(28,i+1),gc(27,i+1),gc(26,i+1),gc(25,i+1),gc(24,i+1),
3593 ✗ gc(23,i+1),gc(22,i+1),gc(21,i+1),gc(20,i+1),gc(19,i+1),gc(18,i+1),gc(17,i+1),gc(16,i+1),
3594 ✗ gc(15,i+1),gc(14,i+1),gc(13,i+1),gc(12,i+1),gc(11,i+1),gc(10,i+1),gc( 9,i+1),gc( 8,i+1),
3595 ✗ gc( 7,i+1),gc( 6,i+1),gc( 5,i+1),gc( 4,i+1),gc( 3,i+1),gc( 2,i+1),gc( 1,i+1),gc( 0,i+1)
3596 );
3597
3598 // unpack hi/lo
3599 ✗ const __m512i coef_rNrNp1_0_31lo = _mm512_unpacklo_epi16(coef_rN_0_31, coef_rNp1_0_31);
3600 ✗ const __m512i coef_rNrNp1_0_31hi = _mm512_unpackhi_epi16(coef_rN_0_31, coef_rNp1_0_31);
3601
3602 // store (as pairs of rows for 0..31 and 32..63). May be better make array of 4-members structure ?
3603 _mm512_store_si512(reinterpret_cast<__m512i*>(dst), coef_rNrNp1_0_31lo);
3604 ✗ _mm512_store_si512(reinterpret_cast<__m512i*>(dst + 32), coef_rNrNp1_0_31hi); // in count of shorts
3605
3606 ✗ dst += 64;
3607 }
3608 else
3609 ✗ env->ThrowError("prepare for AVX512 permute-based H-resize: unsupported");
3610
3611 }
3612 ✗ current_coeff += filter_size_padded * iSamplesAtATime;
3613 }
3614
3615 // assign to ResamplingProgram
3616 ✗ p->pixel_coefficient_AVX512_H = (short*)SIMD_coeff;
3617 ✗ }
3618
3619 ✗ void resize_prepare_coeffs_AVX512_float_H(ResamplingProgram* p, IScriptEnvironment* env) {
3620 // Transpose float coefficients from [px][tap] row-major layout into
3621 // [x_group][tap][px] for 16-pixel-at-a-time permutex ks16 H-resizers.
3622 // Enables 16x _mm512_load_ps instead of 16x _mm512_i32gather_ps per x-group.
3623 ✗ assert(p->bits_per_pixel == 32);
3624 ✗ constexpr int PIXELS_AT_A_TIME = 16;
3625 ✗ constexpr int TAPS = 16;
3626
3627 ✗ const int target_size_aligned = AlignNumber(p->target_size, ALIGN_RESIZER_TARGET_SIZE);
3628 ✗ const int filter_size_padded = p->filter_size;
3629
3630 ✗ const size_t buf_count = (size_t)target_size_aligned * TAPS;
3631 ✗ float* buf = (float*)env->Allocate(sizeof(float) * buf_count, 64, AVS_NORMAL_ALLOC);
3632 ✗ if (!buf)
3633 ✗ env->ThrowError("resize_prepare_coeffs_AVX512_float_H: Could not reserve memory.");
3634 ✗ std::fill_n(buf, buf_count, 0.0f);
3635
3636 ✗ const float* src = p->pixel_coefficient_float;
3637 ✗ float* dst = buf;
3638
3639 ✗ for (int x = 0; x < p->target_size; x += PIXELS_AT_A_TIME) {
3640 ✗ const int avail = std::min(PIXELS_AT_A_TIME, p->target_size - x);
3641 ✗ for (int k = 0; k < TAPS; k++) {
3642 ✗ for (int j = 0; j < PIXELS_AT_A_TIME; j++) {
3643 ✗ const int pj = std::min(j, avail - 1);
3644 ✗ dst[k * PIXELS_AT_A_TIME + j] = (k < filter_size_padded) ? src[filter_size_padded * pj + k] : 0.0f;
3645 }
3646 }
3647 ✗ dst += TAPS * PIXELS_AT_A_TIME;
3648 ✗ src += filter_size_padded * PIXELS_AT_A_TIME;
3649 }
3650
3651 ✗ p->pixel_coefficient_AVX512_float_H = buf;
3652 ✗ }
3653
3654
3655