GCC Code Coverage Report

Directory: avs_core/
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 43.9% 806 / 0 / 1837
Functions: 30.5% 25 / 0 / 82
Branches: 28.5% 345 / 0 / 1210
filters/intel/resample_avx2.cpp
Line Branch Exec Source
1 // Avisynth v2.5. Copyright 2002 Ben Rudiak-Gould et al.
2 // http://avisynth.nl
3
4 // This program is free software; you can redistribute it and/or modify
5 // it under the terms of the GNU General Public License as published by
6 // the Free Software Foundation; either version 2 of the License, or
7 // (at your option) any later version.
8 //
9 // This program is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 // GNU General Public License for more details.
13 //
14 // You should have received a copy of the GNU General Public License
15 // along with this program; if not, write to the Free Software
16 // Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA, or visit
17 // http://www.gnu.org/copyleft/gpl.html .
18 //
19 // Linking Avisynth statically or dynamically with other modules is making a
20 // combined work based on Avisynth. Thus, the terms and conditions of the GNU
21 // General Public License cover the whole combination.
22 //
23 // As a special exception, the copyright holders of Avisynth give you
24 // permission to link Avisynth with independent modules that communicate with
25 // Avisynth solely through the interfaces defined in avisynth.h, regardless of the license
26 // terms of these independent modules, and to copy and distribute the
27 // resulting combined work under terms of your choice, provided that
28 // every copy of the combined work is accompanied by a complete copy of
29 // the source code of Avisynth (the version of Avisynth used to produce the
30 // combined work), being distributed under the terms of the GNU General
31 // Public License plus this exception. An independent module is a module
32 // which is not derived from or based on Avisynth, such as 3rd-party filters,
33 // import and export plugins, or graphical user interfaces.
34
35 // Intrinsics base header + really required extension headers
36 #if defined(_MSC_VER)
37 #include <intrin.h> // MSVC
38 #else
39 #include <x86intrin.h> // GCC/MinGW/Clang/LLVM
40 #endif
41
42 #include <vector>
43 #include <avisynth.h>
44 #include <avs/config.h>
45 #include <avs/types.h>
46 #include <cstdint>
47 #include "../resample_functions.h"
48 #include <cassert>
49 #include <type_traits>
50 #include <algorithm>
51
52 #if !defined(__FMA__)
53 // Assume that all processors that have AVX2 also have FMA3
54 #if defined (__GNUC__) && ! defined (__INTEL_COMPILER) && ! defined (__clang__)
55 // Prevent error message in g++ when using FMA intrinsics with avx2:
56 #pragma message "It is recommended to specify also option -mfma when using -mavx2 or higher"
57 #else
58 #define __FMA__ 1
59 #endif
60 #endif
61 // FMA3 instruction set
62 #if defined (__FMA__) && (defined(__GNUC__) || defined(__clang__)) && ! defined (__INTEL_COMPILER)
63 #include <fmaintrin.h>
64 #endif // __FMA__
65
66
67 #include "resample_avx2.h"
68
69 #ifndef _mm256_set_m128i
70 #define _mm256_set_m128i(v0, v1) _mm256_insertf128_si256(_mm256_castsi128_si256(v1), (v0), 1)
71 #endif
72
73 #ifndef _mm256_set_m128
74 #define _mm256_set_m128(v0, v1) _mm256_insertf128_ps(_mm256_castps128_ps256(v1), (v0), 1)
75 #endif
76
77 // Useful dual-source permute helper functions for AVX2
78
79 // Dual source float-granlarity lookup, like _mm512_permutex2var_ps-
80 // _mm512_permutex2var_ps: two __m512 sources, index 0..31, zeroing bit handled
81 // avx2_permutex2var_ps: two __m256 sources, index 0..15, no zeroing bit simulated
82 static AVS_FORCEINLINE __m256 avx2_permutex2var_ps(__m256 a, __m256 b, __m256i idx) {
83 // 1. Permute from both sources independently
84 __m256 res_a = _mm256_permutevar8x32_ps(a, idx);
85 __m256 res_b = _mm256_permutevar8x32_ps(b, idx);
86
87 // 2. Use the 4th bit of the index (value 8) to select between A and B
88 // In AVX2, we use _mm256_blendv_ps
89 // blendv uses the sign bit (bit 31) of each float, so we shift bit 3 (value 8) to bit 31
90 __m256i select_mask = _mm256_slli_epi32(idx, 28); // Move bit 3 (value 8) to sign bit
91 return _mm256_blendv_ps(res_a, res_b, _mm256_castsi256_ps(select_mask));
92 }
93
94 // Dual source byte-granularity lookup, like _mm512_permutex2var_epi8.
95 // _mm512_permutex2var_epi8: two __m512i sources, index 0..127, zeroing bit handled
96 // avx2_permutex2var_epi8: two __m256i sources, index 0..63, no zeroing bit simulated
97 static AVS_FORCEINLINE __m256i avx2_permutex2var_epi8(__m256i a, __m256i b, __m256i idx) {
98 // idx(0-63) Source Lane(0/1) Byte offset
99 // 0–15 a 0 0–15
100 // 16–31 a 1 0–15
101 // 32–47 b 0 0–15
102 // 48–63 b 1 0–15
103
104 // 1. Prepare swapped versions of A and B (Swap 128-bit lanes)
105 __m256i a_swapped = _mm256_permute2x128_si256(a, a, 0x01);
106 __m256i b_swapped = _mm256_permute2x128_si256(b, b, 0x01);
107
108 // 2. Lookup from A (Lanes 0 and 1)
109 // idx & 16: if 0, use original lane; if 1, use swapped lane
110 __m256i shuf_a_norm = _mm256_shuffle_epi8(a, idx);
111 __m256i shuf_a_swap = _mm256_shuffle_epi8(a_swapped, idx);
112 __m256i res_a = _mm256_blendv_epi8(shuf_a_norm, shuf_a_swap, _mm256_slli_epi32(idx, 3));
113 // Note: blendv uses the top bit, so we shift bit 4 (16) to bit 7 (128)
114
115 // 3. Lookup from B (Lanes 2 and 3)
116 __m256i idx_b = _mm256_sub_epi8(idx, _mm256_set1_epi8(32));
117 __m256i shuf_b_norm = _mm256_shuffle_epi8(b, idx_b);
118 __m256i shuf_b_swap = _mm256_shuffle_epi8(b_swapped, idx_b);
119 __m256i res_b = _mm256_blendv_epi8(shuf_b_norm, shuf_b_swap, _mm256_slli_epi32(idx_b, 3));
120
121 // 4. Final Blend: Decide between Result A or Result B based on Bit 5 (value 32)
122 return _mm256_blendv_epi8(res_a, res_b, _mm256_slli_epi32(idx, 2));
123 }
124
125 //-------- AVX2 Horizontals
126
127 // Dual line processing (speed gain): 2x16 pixels of two consecutive offset entries.
128 // Use aligned filtersize template until alignment and end conditions allow.
129 // Aligned case uses full 16 pix/coeffs in one cycle.
130 // Unsafe part starts with 16 pix/coeffs until safe, then 8, 4, 1.
131 // Basically the only difference between 8 and 10-16 bit is the load and store.
132 // Processing 8 bit pixels even has overhead:
133 // - need upconverting to 16 bit short on load
134 // - extra step when narrowing end results further down to 8 bits.
135 // When processing uint16_t, the exact 16 bit size needs an unsigned -> signed 16 bit conversion
136 // because multiple and add (madd) works in the signed 16 bit domain.
137
138 template<typename pixel_t, bool lessthan16bit, int filtersizealigned16>
139 AVS_FORCEINLINE static void process_two_16pixels_h_uint8_16_core(const pixel_t* AVS_RESTRICT src, int begin1, int begin2, int i, const short* AVS_RESTRICT current_coeff, int filter_size, __m256i& result1, __m256i& result2,
140 __m256i& shifttosigned) {
141 5404 filter_size = (filtersizealigned16 >= 1) ? filtersizealigned16 * 16 : filter_size;
142 // knowing a quasi-constexpr filter_size from template for commonly used sizes
143 // aligned_filter_size 16, 32, 48, 64, hugely helps compiler optimization
144
145 __m256i data_1, data_2;
146
147 if constexpr (sizeof(pixel_t) == 1) {
148 // pixel_t is uint8_t
149 1812 data_1 = _mm256_cvtepu8_epi16(_mm_loadu_si128(reinterpret_cast<const __m128i*>(src + begin1 + i)));
150 5436 data_2 = _mm256_cvtepu8_epi16(_mm_loadu_si128(reinterpret_cast<const __m128i*>(src + begin2 + i)));
151 }
152 else {
153 // pixel_t is uint16_t, at exact 16 bit size an unsigned -> signed 16 bit conversion needed
154 7184 data_1 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src + begin1 + i));
155 if constexpr (!lessthan16bit)
156 1812 data_1 = _mm256_add_epi16(data_1, shifttosigned); // unsigned -> signed
157 5372 data_2 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src + begin2 + i));
158 if constexpr (!lessthan16bit)
159 3624 data_2 = _mm256_add_epi16(data_2, shifttosigned); // unsigned -> signed
160 }
161 5404 __m256i coeff_1 = _mm256_load_si256(reinterpret_cast<const __m256i*>(current_coeff)); // 16 coeffs
162 10808 __m256i coeff_2 = _mm256_load_si256(reinterpret_cast<const __m256i*>(current_coeff + 1 * filter_size)); // 16x second pixel's coefficients
163 10808 result1 = _mm256_add_epi32(result1, _mm256_madd_epi16(data_1, coeff_1));
164 5404 result2 = _mm256_add_epi32(result2, _mm256_madd_epi16(data_2, coeff_2));
165 5404 }
166
167 // filtersizealigned16: special: 1..4. Generic: -1
168 template<bool safe_aligned_mode, typename pixel_t, bool lessthan16bit, int filtersizealigned16>
169 AVS_FORCEINLINE static void process_two_pixels_h_uint8_16(const pixel_t* AVS_RESTRICT src_ptr, int begin1, int begin2, const short* AVS_RESTRICT current_coeff, int filter_size, __m256i& result1, __m256i& result2, int kernel_size,
170 __m256i& shifttosigned) {
171
172 5224 filter_size = (filtersizealigned16 >= 1) ? filtersizealigned16 * 16 : filter_size;
173 // knowing a quasi-constexpr filter_size from template for commonly used sizes
174 // aligned_filter_size 16, 32, 48, 64, hugely helps compiler optimization
175
176 int ksmod16;
177 if constexpr (safe_aligned_mode)
178 3964 ksmod16 = filter_size / 16 * 16;
179 else
180 1260 ksmod16 = kernel_size / 16 * 16; // danger zone, scanline overread possible. Use exact unaligned kernel_size
181 5224 const pixel_t* src_ptr1 = src_ptr + begin1;
182 5224 const pixel_t* src_ptr2 = src_ptr + begin2;
183 5224 int i = 0;
184
185 // Process 16 elements at a time
186
38/38
✓ Branch 42 → 24 taken 425 times.
✓ Branch 42 → 43 taken 325 times.
✓ Branch 46 → 24 taken 866 times.
✓ Branch 46 → 47 taken 666 times.
✓ Branch 67 → 49 taken 425 times.
✓ Branch 67 → 68 taken 325 times.
✓ Branch 75 → 53 taken 866 times.
✓ Branch 75 → 76 taken 666 times.
✓ Branch 94 → 76 taken 425 times.
✓ Branch 94 → 95 taken 325 times.
✓ Branch 106 → 84 taken 866 times.
✓ Branch 106 → 107 taken 666 times.
✓ Branch 119 → 101 taken 425 times.
✓ Branch 119 → 120 taken 325 times.
✓ Branch 135 → 113 taken 866 times.
✓ Branch 135 → 136 taken 666 times.
✓ Branch 167 → 149 taken 20 times.
✓ Branch 167 → 168 taken 55 times.
✓ Branch 189 → 167 taken 40 times.
✓ Branch 189 → 190 taken 260 times.
✓ Branch 272 → 254 taken 20 times.
✓ Branch 272 → 273 taken 55 times.
✓ Branch 310 → 288 taken 20 times.
✓ Branch 310 → 311 taken 186 times.
✓ Branch 314 → 292 taken 20 times.
✓ Branch 314 → 315 taken 74 times.
✓ Branch 379 → 361 taken 20 times.
✓ Branch 379 → 380 taken 55 times.
✓ Branch 433 → 411 taken 20 times.
✓ Branch 433 → 434 taken 186 times.
✓ Branch 441 → 419 taken 20 times.
✓ Branch 441 → 442 taken 74 times.
✓ Branch 484 → 466 taken 20 times.
✓ Branch 484 → 485 taken 55 times.
✓ Branch 554 → 532 taken 20 times.
✓ Branch 554 → 555 taken 186 times.
✓ Branch 566 → 544 taken 20 times.
✓ Branch 566 → 567 taken 74 times.
 10628 for (; i < ksmod16; i += 16) {
187 5404 process_two_16pixels_h_uint8_16_core<pixel_t, lessthan16bit, filtersizealigned16>(src_ptr, begin1, begin2, i, current_coeff + i, filter_size, result1, result2, shifttosigned);
188 }
189
190 if constexpr (!safe_aligned_mode) {
191 // working with the original, unaligned kernel_size
192
11/22
✗ Branch 168 → 169 not taken.
✓ Branch 168 → 170 taken 55 times.
✗ Branch 190 → 191 not taken.
✓ Branch 190 → 192 taken 260 times.
✗ Branch 273 → 274 not taken.
✓ Branch 273 → 275 taken 55 times.
✗ Branch 311 → 312 not taken.
✓ Branch 311 → 313 taken 186 times.
✗ Branch 315 → 316 not taken.
✓ Branch 315 → 317 taken 74 times.
✗ Branch 380 → 381 not taken.
✓ Branch 380 → 382 taken 55 times.
✗ Branch 434 → 435 not taken.
✓ Branch 434 → 436 taken 186 times.
✗ Branch 442 → 443 not taken.
✓ Branch 442 → 444 taken 74 times.
✗ Branch 485 → 486 not taken.
✓ Branch 485 → 487 taken 55 times.
✗ Branch 555 → 556 not taken.
✓ Branch 555 → 557 taken 186 times.
✗ Branch 567 → 568 not taken.
✓ Branch 567 → 569 taken 74 times.
 1260 if (i == kernel_size) return;
193
194 1260 const short* current_coeff2 = current_coeff + filter_size; // Points to second pixel's coefficients
195 1260 const int ksmod8 = kernel_size / 8 * 8;
196 1260 const int ksmod4 = kernel_size / 4 * 4;
197
198 // Process 8 elements if needed
199
23/23
✓ Branch 170 → 171 taken 10 times.
✓ Branch 170 → 193 taken 45 times.
✓ Branch 192 → 193 taken 74 times.
✓ Branch 192 → 223 taken 64 times.
✓ Branch 192 → 227 taken 122 times.
✓ Branch 275 → 276 taken 10 times.
✓ Branch 275 → 298 taken 45 times.
✓ Branch 313 → 314 taken 64 times.
✓ Branch 313 → 348 taken 122 times.
✓ Branch 317 → 318 taken 10 times.
✓ Branch 317 → 348 taken 64 times.
✓ Branch 382 → 383 taken 10 times.
✓ Branch 382 → 405 taken 45 times.
✓ Branch 436 → 437 taken 64 times.
✓ Branch 436 → 471 taken 122 times.
✓ Branch 444 → 445 taken 10 times.
✓ Branch 444 → 475 taken 64 times.
✓ Branch 487 → 488 taken 10 times.
✓ Branch 487 → 510 taken 45 times.
✓ Branch 557 → 558 taken 64 times.
✓ Branch 557 → 592 taken 122 times.
✓ Branch 569 → 570 taken 10 times.
✓ Branch 569 → 600 taken 64 times.
 1260 if (i < ksmod8) {
200 // Process 8 elements for first pixel
201 __m128i data_1;
202 if constexpr (sizeof(pixel_t) == 1)
203 512 data_1 = _mm_cvtepu8_epi16(_mm_loadl_epi64(reinterpret_cast<const __m128i*>(src_ptr1 + i)));
204 else {
205 // uint16_t
206 80 data_1 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src_ptr1 + i));
207 if constexpr (!lessthan16bit)
208 80 data_1 = _mm_add_epi16(data_1, _mm256_castsi256_si128(shifttosigned)); // unsigned -> signed
209 }
210
211 672 __m128i coeff_1 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(current_coeff + i));
212 336 __m128i temp_result1 = _mm_madd_epi16(data_1, coeff_1);
213
214 // Process 8 elements for second pixel
215 __m128i data_2;
216 if constexpr (sizeof(pixel_t) == 1)
217 512 data_2 = _mm_cvtepu8_epi16(_mm_loadl_epi64(reinterpret_cast<const __m128i*>(src_ptr2 + i)));
218 else {
219 80 data_2 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src_ptr2 + i));
220 if constexpr (!lessthan16bit)
221 80 data_2 = _mm_add_epi16(data_2, _mm256_castsi256_si128(shifttosigned)); // unsigned -> signed
222 }
223 672 __m128i coeff_2 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(current_coeff2 + i));
224 336 __m128i temp_result2 = _mm_madd_epi16(data_2, coeff_2);
225
226 // update result vectors
227 336 __m256i temp1 = _mm256_setzero_si256();
228 336 __m256i temp2 = _mm256_setzero_si256();
229 336 temp1 = _mm256_insertf128_si256(temp1, temp_result1, 0);
230 336 temp2 = _mm256_insertf128_si256(temp2, temp_result2, 0);
231 336 result1 = _mm256_add_epi32(result1, temp1);
232 336 result2 = _mm256_add_epi32(result2, temp2);
233
234 336 i += 8;
235
15/22
✗ Branch 191 → 192 not taken.
✓ Branch 191 → 193 taken 10 times.
✗ Branch 221 → 222 not taken.
✓ Branch 221 → 223 taken 10 times.
✓ Branch 225 → 226 taken 27 times.
✓ Branch 225 → 227 taken 37 times.
✗ Branch 296 → 297 not taken.
✓ Branch 296 → 298 taken 10 times.
✓ Branch 346 → 347 taken 27 times.
✓ Branch 346 → 348 taken 47 times.
✗ Branch 403 → 404 not taken.
✓ Branch 403 → 405 taken 10 times.
✓ Branch 469 → 470 taken 27 times.
✓ Branch 469 → 471 taken 37 times.
✗ Branch 473 → 474 not taken.
✓ Branch 473 → 475 taken 10 times.
✗ Branch 508 → 509 not taken.
✓ Branch 508 → 510 taken 10 times.
✓ Branch 590 → 591 taken 27 times.
✓ Branch 590 → 592 taken 37 times.
✗ Branch 598 → 599 not taken.
✓ Branch 598 → 600 taken 10 times.
 336 if (i == kernel_size) return;
236 }
237
238 // Process 4 elements if needed
239
23/23
✓ Branch 193 → 194 taken 45 times.
✓ Branch 193 → 232 taken 10 times.
✓ Branch 223 → 224 taken 64 times.
✓ Branch 223 → 270 taken 10 times.
✓ Branch 227 → 228 taken 94 times.
✓ Branch 227 → 266 taken 65 times.
✓ Branch 298 → 299 taken 45 times.
✓ Branch 298 → 337 taken 10 times.
✓ Branch 348 → 349 taken 158 times.
✓ Branch 348 → 387 taken 65 times.
✓ Branch 348 → 395 taken 10 times.
✓ Branch 405 → 406 taken 45 times.
✓ Branch 405 → 444 taken 10 times.
✓ Branch 471 → 472 taken 94 times.
✓ Branch 471 → 510 taken 65 times.
✓ Branch 475 → 476 taken 64 times.
✓ Branch 475 → 522 taken 10 times.
✓ Branch 510 → 511 taken 45 times.
✓ Branch 510 → 549 taken 10 times.
✓ Branch 592 → 593 taken 94 times.
✓ Branch 592 → 631 taken 65 times.
✓ Branch 600 → 601 taken 64 times.
✓ Branch 600 → 647 taken 10 times.
 1152 if (i < ksmod4) {
240 // Process 4 elements for first pixel
241 __m128i data_1;
242 if constexpr (sizeof(pixel_t) == 1)
243 752 data_1= _mm_cvtepu8_epi16(_mm_cvtsi32_si128(*reinterpret_cast<const int*>(src_ptr1 + i)));
244 else {
245 436 data_1 = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(src_ptr1 + i));
246 if constexpr (!lessthan16bit)
247 512 data_1 = _mm_add_epi16(data_1, _mm256_castsi256_si128(shifttosigned)); // unsigned -> signed
248 }
249 1624 __m128i coeff_1 = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(current_coeff + i));
250 812 __m128i temp_result1 = _mm_madd_epi16(data_1, coeff_1);
251
252 // Process 4 elements for second pixel
253 __m128i data_2;
254 if constexpr (sizeof(pixel_t) == 1)
255 752 data_2 = _mm_cvtepu8_epi16(_mm_cvtsi32_si128(*reinterpret_cast<const int*>(src_ptr2 + i)));
256 else {
257 436 data_2 = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(src_ptr2 + i));
258 if constexpr (!lessthan16bit)
259 512 data_2 = _mm_add_epi16(data_2, _mm256_castsi256_si128(shifttosigned)); // unsigned -> signed
260 }
261 1624 __m128i coeff_2 = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(current_coeff2 + i));
262 812 __m128i temp_result2 = _mm_madd_epi16(data_2, coeff_2);
263
264 // update result vectors
265 812 __m256i temp1 = _mm256_setzero_si256();
266 812 __m256i temp2 = _mm256_setzero_si256();
267 812 temp1 = _mm256_insertf128_si256(temp1, temp_result1, 0);
268 812 temp2 = _mm256_insertf128_si256(temp2, temp_result2, 0);
269 812 result1 = _mm256_add_epi32(result1, temp1);
270 812 result2 = _mm256_add_epi32(result2, temp2);
271
272 812 i += 4;
273
24/24
✓ Branch 230 → 231 taken 30 times.
✓ Branch 230 → 232 taken 15 times.
✓ Branch 264 → 265 taken 79 times.
✓ Branch 264 → 266 taken 15 times.
✓ Branch 268 → 269 taken 49 times.
✓ Branch 268 → 270 taken 15 times.
✓ Branch 335 → 336 taken 30 times.
✓ Branch 335 → 337 taken 15 times.
✓ Branch 385 → 386 taken 79 times.
✓ Branch 385 → 387 taken 15 times.
✓ Branch 393 → 394 taken 49 times.
✓ Branch 393 → 395 taken 15 times.
✓ Branch 442 → 443 taken 30 times.
✓ Branch 442 → 444 taken 15 times.
✓ Branch 508 → 509 taken 79 times.
✓ Branch 508 → 510 taken 15 times.
✓ Branch 520 → 521 taken 49 times.
✓ Branch 520 → 522 taken 15 times.
✓ Branch 547 → 548 taken 30 times.
✓ Branch 547 → 549 taken 15 times.
✓ Branch 629 → 630 taken 79 times.
✓ Branch 629 → 631 taken 15 times.
✓ Branch 645 → 646 taken 49 times.
✓ Branch 645 → 647 taken 15 times.
 812 if (i == kernel_size) return;
274 }
275
276 // Process remaining elements with scalar operations
277
12/24
✓ Branch 232 → 233 taken 25 times.
✗ Branch 232 → 249 not taken.
✓ Branch 266 → 267 taken 80 times.
✗ Branch 266 → 283 not taken.
✓ Branch 270 → 271 taken 25 times.
✗ Branch 270 → 287 not taken.
✓ Branch 337 → 338 taken 25 times.
✗ Branch 337 → 354 not taken.
✓ Branch 387 → 388 taken 80 times.
✗ Branch 387 → 404 not taken.
✓ Branch 395 → 396 taken 25 times.
✗ Branch 395 → 412 not taken.
✓ Branch 444 → 445 taken 25 times.
✗ Branch 444 → 461 not taken.
✓ Branch 510 → 511 taken 80 times.
✗ Branch 510 → 527 not taken.
✓ Branch 522 → 523 taken 25 times.
✗ Branch 522 → 539 not taken.
✓ Branch 549 → 550 taken 25 times.
✗ Branch 549 → 566 not taken.
✓ Branch 631 → 632 taken 80 times.
✗ Branch 631 → 648 not taken.
✓ Branch 647 → 648 taken 25 times.
✗ Branch 647 → 664 not taken.
 520 if (i < kernel_size) {
278 520 int scalar_sum1[4] = { 0, 0, 0, 0 }; // like an __m128i
279 520 int scalar_sum2[4] = { 0, 0, 0, 0 };
280
281
282
24/24
✓ Branch 235 → 234 taken 60 times.
✓ Branch 235 → 236 taken 25 times.
✓ Branch 269 → 268 taken 147 times.
✓ Branch 269 → 270 taken 80 times.
✓ Branch 273 → 272 taken 60 times.
✓ Branch 273 → 274 taken 25 times.
✓ Branch 340 → 339 taken 60 times.
✓ Branch 340 → 341 taken 25 times.
✓ Branch 390 → 389 taken 147 times.
✓ Branch 390 → 391 taken 80 times.
✓ Branch 398 → 397 taken 60 times.
✓ Branch 398 → 399 taken 25 times.
✓ Branch 447 → 446 taken 60 times.
✓ Branch 447 → 448 taken 25 times.
✓ Branch 513 → 512 taken 147 times.
✓ Branch 513 → 514 taken 80 times.
✓ Branch 525 → 524 taken 60 times.
✓ Branch 525 → 526 taken 25 times.
✓ Branch 552 → 551 taken 60 times.
✓ Branch 552 → 553 taken 25 times.
✓ Branch 634 → 633 taken 147 times.
✓ Branch 634 → 635 taken 80 times.
✓ Branch 650 → 649 taken 60 times.
✓ Branch 650 → 651 taken 25 times.
 1588 for (; i < kernel_size; i++) {
283 if constexpr (sizeof(pixel_t) == 1) {
284 588 scalar_sum1[i % 4] += src_ptr1[i] * current_coeff[i];
285 588 scalar_sum2[i % 4] += src_ptr2[i] * current_coeff2[i];
286 }
287 else {
288 480 uint16_t pix1 = src_ptr1[i];
289 480 uint16_t pix2 = src_ptr2[i];
290
291 if constexpr (!lessthan16bit) {
292 240 pix1 -= 32768;
293 240 pix2 -= 32768;
294 }
295
296 480 scalar_sum1[i % 4] += (short)pix1 * current_coeff[i];
297 480 scalar_sum2[i % 4] += (short)pix2 * current_coeff2[i];
298 }
299 }
300
301 // Convert scalar results to SIMD and add to result vectors
302 520 __m128i temp_result1 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(scalar_sum1));
303 520 __m128i temp_result2 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(scalar_sum2));
304
305 520 __m256i temp1 = _mm256_setzero_si256();
306 520 __m256i temp2 = _mm256_setzero_si256();
307 520 temp1 = _mm256_insertf128_si256(temp1, temp_result1, 0);
308 520 temp2 = _mm256_insertf128_si256(temp2, temp_result2, 0);
309 520 result1 = _mm256_add_epi32(result1, temp1);
310 1040 result2 = _mm256_add_epi32(result2, temp2);
311 }
312 }
313 3964 }
314
315 // filtersizealigned16: special: 1..4. Generic: -1
316 template<bool is_safe, typename pixel_t, bool lessthan16bit, int filtersizealigned16>
317 AVS_FORCEINLINE static void process_eight_pixels_h_uint8_16(const pixel_t* src, int x, const short* current_coeff_base, int filter_size,
318 __m256i& rounder256, __m256i& shifttosigned, __m128i& clamp_limit,
319 pixel_t* dst,
320 ResamplingProgram* program)
321 {
322
3/6
✗ Branch 19 → 20 not taken.
✓ Branch 19 → 21 taken 991 times.
✗ Branch 144 → 145 not taken.
✓ Branch 144 → 146 taken 55 times.
✗ Branch 162 → 163 not taken.
✓ Branch 162 → 164 taken 260 times.
 1306 assert(program->filter_size_alignment >= 16); // code assumes this
323
324 1306 filter_size = (filtersizealigned16 >= 1) ? filtersizealigned16 * 16 : filter_size;
325 // knowing a quasi-constexpr filter_size from template for commonly used sizes
326 // aligned_filter_size 16, 32, 48, 64, hugely helps compiler optimization
327
328 1306 const short* AVS_RESTRICT current_coeff = current_coeff_base + x * filter_size;
329 1306 const int unaligned_kernel_size = program->filter_size_real;
330
331 // Unrolled processing of all 8 pixels
332
333 // 0 & 1
334 1306 __m256i result0 = rounder256;
335 1306 __m256i result1 = rounder256;
336 1306 int begin0 = program->pixel_offset[x + 0];
337 1306 int begin1 = program->pixel_offset[x + 1];
338 process_two_pixels_h_uint8_16<is_safe, pixel_t, lessthan16bit, filtersizealigned16>(src, begin0, begin1, current_coeff, filter_size, result0, result1, unaligned_kernel_size, shifttosigned);
339 1306 current_coeff += 2 * filter_size;
340 1306 __m256i sumQuad12 = _mm256_hadd_epi32(result0, result1);
341
342 // 2 & 3
343 1306 result0 = rounder256;
344 1306 result1 = rounder256;
345 1306 begin0 = program->pixel_offset[x + 2];
346 1306 begin1 = program->pixel_offset[x + 3];
347 process_two_pixels_h_uint8_16<is_safe, pixel_t, lessthan16bit, filtersizealigned16>(src, begin0, begin1, current_coeff, filter_size, result0, result1, unaligned_kernel_size, shifttosigned);
348 1306 current_coeff += 2 * filter_size;
349 2612 __m256i sumQuad1234 = _mm256_hadd_epi32(sumQuad12, _mm256_hadd_epi32(result0, result1));
350
351 // 4 & 5
352 1306 result0 = rounder256;
353 1306 result1 = rounder256;
354 1306 begin0 = program->pixel_offset[x + 4];
355 1306 begin1 = program->pixel_offset[x + 5];
356 process_two_pixels_h_uint8_16<is_safe, pixel_t, lessthan16bit, filtersizealigned16>(src, begin0, begin1, current_coeff, filter_size, result0, result1, unaligned_kernel_size, shifttosigned);
357 1306 current_coeff += 2 * filter_size;
358 1306 __m256i sumQuad56 = _mm256_hadd_epi32(result0, result1);
359
360 // 6 & 7
361 1306 result0 = rounder256;
362 1306 result1 = rounder256;
363 1306 begin0 = program->pixel_offset[x + 6];
364 1306 begin1 = program->pixel_offset[x + 7];
365 process_two_pixels_h_uint8_16<is_safe, pixel_t, lessthan16bit, filtersizealigned16>(src, begin0, begin1, current_coeff, filter_size, result0, result1, unaligned_kernel_size, shifttosigned);
366 //current_coeff += 2 * filter_size;
367 2612 __m256i sumQuad5678 = _mm256_hadd_epi32(sumQuad56, _mm256_hadd_epi32(result0, result1));
368
369 1306 __m128i pix1234 = _mm_add_epi32(_mm256_extractf128_si256(sumQuad1234, 0), _mm256_extractf128_si256(sumQuad1234, 1));
370 1306 __m128i pix5678 = _mm_add_epi32(_mm256_extractf128_si256(sumQuad5678, 0), _mm256_extractf128_si256(sumQuad5678, 1));
371 1713 __m256i result_8x_uint32 = _mm256_set_m128i(pix5678, pix1234);
372
373 // correct if signed, scale back, store
374 if constexpr (sizeof(pixel_t) == 2 && !lessthan16bit) {
375 407 const __m256i shiftfromsigned = _mm256_set1_epi32(+32768 << FPScale16bits); // yes, 32 bit data. for 16 bits only
376 407 result_8x_uint32 = _mm256_add_epi32(result_8x_uint32, shiftfromsigned);
377 }
378
379 1306 const int current_fp_scale_bits = (sizeof(pixel_t) == 1) ? FPScale8bits : FPScale16bits;
380
381 // scale back, shuffle, store
382 1306 __m256i result = _mm256_srai_epi32(result_8x_uint32, current_fp_scale_bits);
383 1306 __m256i result_2x4x_uint16 = _mm256_packus_epi32(result, result /* n/a */);
384 1825 __m128i result_2x4x_uint16_128 = _mm256_castsi256_si128(_mm256_permute4x64_epi64(result_2x4x_uint16, (0 << 0) | (2 << 2) | (0 << 4) | (0 << 6)));
385 if constexpr (sizeof(pixel_t) == 2 && lessthan16bit)
386 380 result_2x4x_uint16_128 = _mm_min_epu16(result_2x4x_uint16_128, clamp_limit); // extra clamp for 10-14 bits
387
388 if constexpr (sizeof(pixel_t) == 1) {
389 519 __m128i result_2x4x_uint8 = _mm_packus_epi16(result_2x4x_uint16_128, _mm_setzero_si128());
390 519 _mm_storel_epi64(reinterpret_cast<__m128i*>(dst + x), result_2x4x_uint8);
391 }
392 else {
393 787 _mm_stream_si128(reinterpret_cast<__m128i*>(dst + x), result_2x4x_uint16_128);
394 }
395 1306 }
396
397 // filtersizealigned16: special: 1..4. Generic: -1
398 template<typename pixel_t, bool lessthan16bit, int filtersizealigned16>
399 40 static void internal_resizer_h_avx2_generic_uint8_16_t(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) {
400 40 const int filter_size = (filtersizealigned16 >= 1) ? filtersizealigned16 * 16 : program->filter_size;
401 // knowing a quasi-constexpr filter_size from template for commonly used sizes
402 // aligned_filter_size 16, 32, 48, 64, hugely helps compiler optimization
403
404 40 __m256i shifttosigned = _mm256_set1_epi16(-32768); // for 16 bits only
405
406 40 const int current_fp_scale_bits = (sizeof(pixel_t) == 1) ? FPScale8bits : FPScale16bits;
407 40 __m256i rounder256 = _mm256_setr_epi32(1 << (current_fp_scale_bits - 1), 0, 0, 0, 0, 0, 0, 0);
408
409 40 __m128i clamp_limit = _mm_set1_epi16((short)((1 << bits_per_pixel) - 1)); // clamp limit for 8< <16 bits
410
411 40 const pixel_t* AVS_RESTRICT src = reinterpret_cast<const pixel_t*>(src8);
412 40 pixel_t* AVS_RESTRICT dst = reinterpret_cast<pixel_t*>(dst8);
413 40 dst_pitch /= sizeof(pixel_t);
414 40 src_pitch /= sizeof(pixel_t);
415
416
6/30
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned char, true, 1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 14 → 15 taken 27 times.
✗ Branch 14 → 16 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned char, true, 2>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 14 → 15 taken 1 time.
✗ Branch 14 → 16 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned char, true, 3>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 14 → 15 not taken.
✗ Branch 14 → 16 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned char, true, 4>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 14 → 15 not taken.
✗ Branch 14 → 16 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned char, true, -1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 14 → 15 not taken.
✗ Branch 14 → 16 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, false, 1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 14 → 15 taken 7 times.
✗ Branch 14 → 16 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, false, 2>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 14 → 15 taken 1 time.
✗ Branch 14 → 16 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, false, 3>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 14 → 15 not taken.
✗ Branch 14 → 16 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, false, 4>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 14 → 15 not taken.
✗ Branch 14 → 16 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, false, -1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 14 → 15 not taken.
✗ Branch 14 → 16 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, true, 1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 14 → 15 taken 3 times.
✗ Branch 14 → 16 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, true, 2>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 14 → 15 taken 1 time.
✗ Branch 14 → 16 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, true, 3>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 14 → 15 not taken.
✗ Branch 14 → 16 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, true, 4>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 14 → 15 not taken.
✗ Branch 14 → 16 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, true, -1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 14 → 15 not taken.
✗ Branch 14 → 16 not taken.
 40 const int w_safe_mod8 = (program->safelimit_filter_size_aligned.overread_possible ? program->safelimit_filter_size_aligned.source_overread_beyond_targetx : width) / 8 * 8;
417
418
12/30
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned char, true, 1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 673 → 18 taken 146 times.
✓ Branch 673 → 674 taken 27 times.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned char, true, 2>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 673 → 18 taken 5 times.
✓ Branch 673 → 674 taken 1 time.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned char, true, 3>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 673 → 18 not taken.
✗ Branch 673 → 674 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned char, true, 4>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 673 → 18 not taken.
✗ Branch 673 → 674 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned char, true, -1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 673 → 18 not taken.
✗ Branch 673 → 674 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, false, 1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 689 → 18 taken 34 times.
✓ Branch 689 → 690 taken 7 times.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, false, 2>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 689 → 18 taken 5 times.
✓ Branch 689 → 690 taken 1 time.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, false, 3>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 689 → 18 not taken.
✗ Branch 689 → 690 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, false, 4>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 689 → 18 not taken.
✗ Branch 689 → 690 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, false, -1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 689 → 18 not taken.
✗ Branch 689 → 690 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, true, 1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 589 → 18 taken 15 times.
✓ Branch 589 → 590 taken 3 times.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, true, 2>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 589 → 18 taken 5 times.
✓ Branch 589 → 590 taken 1 time.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, true, 3>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 589 → 18 not taken.
✗ Branch 589 → 590 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, true, 4>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 589 → 18 not taken.
✗ Branch 589 → 590 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, true, -1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 589 → 18 not taken.
✗ Branch 589 → 590 not taken.
 250 for (int y = 0; y < height; y++) {
419 210 const short* AVS_RESTRICT current_coeff_base = program->pixel_coefficient;
420
421 // Process safe aligned pixels
422
12/30
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned char, true, 1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 160 → 19 taken 233 times.
✓ Branch 160 → 161 taken 146 times.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned char, true, 2>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 160 → 19 taken 100 times.
✓ Branch 160 → 161 taken 5 times.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned char, true, 3>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 160 → 19 not taken.
✗ Branch 160 → 161 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned char, true, 4>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 160 → 19 not taken.
✗ Branch 160 → 161 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned char, true, -1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 160 → 19 not taken.
✗ Branch 160 → 161 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, false, 1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 160 → 19 taken 233 times.
✓ Branch 160 → 161 taken 34 times.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, false, 2>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 160 → 19 taken 100 times.
✓ Branch 160 → 161 taken 5 times.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, false, 3>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 160 → 19 not taken.
✗ Branch 160 → 161 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, false, 4>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 160 → 19 not taken.
✗ Branch 160 → 161 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, false, -1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 160 → 19 not taken.
✗ Branch 160 → 161 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, true, 1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 142 → 19 taken 225 times.
✓ Branch 142 → 143 taken 15 times.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, true, 2>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 142 → 19 taken 100 times.
✓ Branch 142 → 143 taken 5 times.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, true, 3>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 142 → 19 not taken.
✗ Branch 142 → 143 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, true, 4>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 142 → 19 not taken.
✗ Branch 142 → 143 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, true, -1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 142 → 19 not taken.
✗ Branch 142 → 143 not taken.
 1201 for (int x = 0; x < w_safe_mod8; x += 8) {
423 process_eight_pixels_h_uint8_16<true, pixel_t, lessthan16bit, filtersizealigned16>(src, x, current_coeff_base, filter_size, rounder256, shifttosigned, clamp_limit, dst, program);
424 }
425
426 // Process up to the actual kernel size instead of the aligned filter_size to prevent overreading beyond the last source pixel.
427 // We assume extra offset entries were added to the p->pixel_offset array (aligned to 8 during initialization).
428 // This may store 1-7 false pixels, but they are ignored since Avisynth will not read beyond the width.
429
12/30
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned char, true, 1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 671 → 162 taken 166 times.
✓ Branch 671 → 672 taken 146 times.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned char, true, 2>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 671 → 162 taken 20 times.
✓ Branch 671 → 672 taken 5 times.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned char, true, 3>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 671 → 162 not taken.
✗ Branch 671 → 672 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned char, true, 4>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 671 → 162 not taken.
✗ Branch 671 → 672 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned char, true, -1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 671 → 162 not taken.
✗ Branch 671 → 672 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, false, 1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 687 → 162 taken 54 times.
✓ Branch 687 → 688 taken 34 times.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, false, 2>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 687 → 162 taken 20 times.
✓ Branch 687 → 688 taken 5 times.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, false, 3>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 687 → 162 not taken.
✗ Branch 687 → 688 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, false, 4>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 687 → 162 not taken.
✗ Branch 687 → 688 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, false, -1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 687 → 162 not taken.
✗ Branch 687 → 688 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, true, 1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 587 → 144 taken 35 times.
✓ Branch 587 → 588 taken 15 times.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, true, 2>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 587 → 144 taken 20 times.
✓ Branch 587 → 588 taken 5 times.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, true, 3>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 587 → 144 not taken.
✗ Branch 587 → 588 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, true, 4>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 587 → 144 not taken.
✗ Branch 587 → 588 not taken.
void internal_resizer_h_avx2_generic_uint8_16_t<unsigned short, true, -1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 587 → 144 not taken.
✗ Branch 587 → 588 not taken.
 525 for (int x = w_safe_mod8; x < width; x += 8) {
430 process_eight_pixels_h_uint8_16<false, pixel_t, lessthan16bit, filtersizealigned16>(src, x, current_coeff_base, filter_size, rounder256, shifttosigned, clamp_limit, dst, program);
431 }
432
433 210 dst += dst_pitch;
434 210 src += src_pitch;
435 }
436 40 }
437
438 // coeffs are safely padded/aligned to 16
439
440 // 8 bit Horizontal
441
442 28 void resizer_h_avx2_generic_uint8_t(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) {
443 AVS_UNUSED(bits_per_pixel);
444 28 const int filter_size = program->filter_size;
445
1/2
✗ Branch 2 → 3 not taken.
✓ Branch 2 → 4 taken 28 times.
 28 assert(program->filter_size_alignment == 16);
446
447
2/2
✓ Branch 4 → 5 taken 27 times.
✓ Branch 4 → 6 taken 1 time.
 28 if (filter_size == 16)
448 27 internal_resizer_h_avx2_generic_uint8_16_t<uint8_t, true, 1>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
449
1/2
✓ Branch 6 → 7 taken 1 time.
✗ Branch 6 → 8 not taken.
 1 else if (filter_size == 2*16)
450 1 internal_resizer_h_avx2_generic_uint8_16_t<uint8_t, true, 2>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
451 ✗ else if (filter_size == 3*16)
452 ✗ internal_resizer_h_avx2_generic_uint8_16_t<uint8_t, true, 3>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
453 ✗ else if (filter_size == 4*16)
454 ✗ internal_resizer_h_avx2_generic_uint8_16_t<uint8_t, true, 4>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
455 else // -1: basic method, use program->filter_size
456 ✗ internal_resizer_h_avx2_generic_uint8_16_t<uint8_t, true, -1>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
457 28 }
458
459 // 16 bit Horizontal
460
461 template<bool lessthan16bit>
462 12 void resizer_h_avx2_generic_uint16_t(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) {
463 12 const int filter_size = program->filter_size;
464
2/4
void resizer_h_avx2_generic_uint16_t<false>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 2 → 3 not taken.
✓ Branch 2 → 4 taken 8 times.
void resizer_h_avx2_generic_uint16_t<true>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 2 → 3 not taken.
✓ Branch 2 → 4 taken 4 times.
 12 assert(program->filter_size_alignment == 16);
465
466
4/4
void resizer_h_avx2_generic_uint16_t<false>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 4 → 5 taken 7 times.
✓ Branch 4 → 6 taken 1 time.
void resizer_h_avx2_generic_uint16_t<true>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 4 → 5 taken 3 times.
✓ Branch 4 → 6 taken 1 time.
 12 if (filter_size== 16)
467 10 internal_resizer_h_avx2_generic_uint8_16_t<uint16_t, lessthan16bit, 1>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
468
2/4
void resizer_h_avx2_generic_uint16_t<false>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 6 → 7 taken 1 time.
✗ Branch 6 → 8 not taken.
void resizer_h_avx2_generic_uint16_t<true>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 6 → 7 taken 1 time.
✗ Branch 6 → 8 not taken.
 2 else if (filter_size == 2 * 16)
469 2 internal_resizer_h_avx2_generic_uint8_16_t<uint16_t, lessthan16bit, 2>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
470 ✗ else if (filter_size == 3 * 16)
471 ✗ internal_resizer_h_avx2_generic_uint8_16_t<uint16_t, lessthan16bit, 3>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
472 ✗ else if (filter_size == 4 * 16)
473 ✗ internal_resizer_h_avx2_generic_uint8_16_t<uint16_t, lessthan16bit, 4>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
474 else // -1: basic method, use program->filter_size
475 ✗ internal_resizer_h_avx2_generic_uint8_16_t<uint16_t, lessthan16bit, -1>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
476 12 }
477
478 // AVX2 Horizontal float
479
480 // 2x8 pixels of two consecutive offset entries.
481 AVS_FORCEINLINE static void process_pix2_coeff8_h_float_core(const float* src, int begin1, int begin2, int i, float* current_coeff, int filter_size, __m256& result1, __m256& result2) {
482 ✗ __m256 data_1 = _mm256_loadu_ps(src + begin1 + i);
483 ✗ __m256 data_2 = _mm256_loadu_ps(src + begin2 + i);
484 ✗ __m256 coeff_1 = _mm256_load_ps(current_coeff); // 8 coeffs
485 ✗ __m256 coeff_2 = _mm256_load_ps(current_coeff + 1 * filter_size); // 8x second pixel's coefficients
486 ✗ result1 = _mm256_fmadd_ps(data_1, coeff_1, result1); // a*b + c
487 ✗ result2 = _mm256_fmadd_ps(data_2, coeff_2, result2);
488 ✗ }
489
490 template<bool safe_aligned_mode>
491 AVS_FORCEINLINE static void process_two_pixels_h_float(const float* src_ptr, int begin1, int begin2, float* current_coeff, int filter_size, __m256& result1, __m256& result2, int kernel_size) {
492 int ksmod8;
493 // 32 bytes contain 8 floats
494 if constexpr (safe_aligned_mode)
495 ✗ ksmod8 = filter_size / 8 * 8;
496 else
497 ✗ ksmod8 = kernel_size / 8 * 8; // danger zone, scanline overread possible. Use exact unaligned kernel_size
498 ✗ const float* src_ptr1 = src_ptr + begin1;
499 ✗ const float* src_ptr2 = src_ptr + begin2;
500 ✗ int i = 0;
501
502 // Process 8 elements at a time
503 ✗ for (; i < ksmod8; i += 8) {
504 ✗ process_pix2_coeff8_h_float_core(src_ptr, begin1, begin2, i, current_coeff + i, filter_size, result1, result2);
505 }
506
507 if constexpr (!safe_aligned_mode) {
508 // working with the original, unaligned kernel_size
509 ✗ if (i == kernel_size) return;
510
511 ✗ float* current_coeff2 = current_coeff + filter_size; // Points to second pixel's coefficients
512 ✗ const int ksmod4 = kernel_size / 4 * 4;
513
514 // Process 4 elements if needed
515 ✗ if (i < ksmod4) {
516 // Process 4 elements for first pixel
517 ✗ __m128 data_1 = _mm_loadu_ps(src_ptr1 + i);
518 ✗ __m128 coeff_1 = _mm_load_ps(current_coeff + i);
519 ✗ __m128 temp_result1 = _mm_mul_ps(data_1, coeff_1);
520
521 // Process 4 elements for second pixel
522 ✗ __m128 data_2 = _mm_loadu_ps(src_ptr2 + i);
523 ✗ __m128 coeff_2 = _mm_load_ps(current_coeff2 + i);
524 ✗ __m128 temp_result2 = _mm_mul_ps(data_2, coeff_2);
525
526 // update result vectors
527 ✗ __m256 temp1 = _mm256_setzero_ps();
528 ✗ __m256 temp2 = _mm256_setzero_ps();
529 ✗ temp1 = _mm256_insertf128_ps(temp1, temp_result1, 0);
530 ✗ temp2 = _mm256_insertf128_ps(temp2, temp_result2, 0);
531 ✗ result1 = _mm256_add_ps(result1, temp1);
532 ✗ result2 = _mm256_add_ps(result2, temp2);
533
534 ✗ i += 4;
535 ✗ if (i == kernel_size) return;
536 }
537
538 // Process remaining elements with scalar operations
539 ✗ if (i < kernel_size) {
540 ✗ float scalar_sum1[4] = { 0, 0, 0, 0 }; // like an __m128
541 ✗ float scalar_sum2[4] = { 0, 0, 0, 0 };
542
543 ✗ for (; i < kernel_size; i++) {
544 ✗ scalar_sum1[i % 4] += src_ptr1[i] * current_coeff[i];
545 ✗ scalar_sum2[i % 4] += src_ptr2[i] * current_coeff2[i];
546 }
547
548 // Convert scalar results to SIMD and add to result vectors
549 ✗ __m128 temp_result1 = _mm_loadu_ps(scalar_sum1);
550 ✗ __m128 temp_result2 = _mm_loadu_ps(scalar_sum2);
551
552 ✗ __m256 temp1 = _mm256_setzero_ps();
553 ✗ __m256 temp2 = _mm256_setzero_ps();
554 ✗ temp1 = _mm256_insertf128_ps(temp1, temp_result1, 0);
555 ✗ temp2 = _mm256_insertf128_ps(temp2, temp_result2, 0);
556 ✗ result1 = _mm256_add_ps(result1, temp1);
557 ✗ result2 = _mm256_add_ps(result2, temp2);
558 }
559 }
560 ✗ }
561
562 template<bool is_safe>
563 AVS_FORCEINLINE static void process_eight_pixels_h_float(const float* src, int x, float* current_coeff_base, int filter_size,
564 __m128& zero128, __m256& zero256,
565 float* dst,
566 ResamplingProgram* program)
567 {
568 ✗ assert(program->filter_size_alignment >= 8); // code assumes this
569
570 ✗ float* current_coeff = current_coeff_base + x * filter_size;
571 ✗ const int unaligned_kernel_size = program->filter_size_real;
572
573 // Unrolled processing of all 8 pixels
574
575 // 0 & 1
576 ✗ __m256 result0 = zero256;
577 ✗ __m256 result1 = zero256;
578 ✗ int begin0 = program->pixel_offset[x + 0];
579 ✗ int begin1 = program->pixel_offset[x + 1];
580 process_two_pixels_h_float<is_safe>(src, begin0, begin1, current_coeff, filter_size, result0, result1, unaligned_kernel_size);
581 ✗ current_coeff += 2 * filter_size;
582 ✗ __m256 sumQuad12 = _mm256_hadd_ps(result0, result1); // L1L1L1L1L1L1L1L1 + L2L2L2L2L2L2L2L2L2 = L1L1 L2L2 L1L1 L2L2
583
584 // 2 & 3
585 ✗ result0 = zero256;
586 ✗ result1 = zero256;
587 ✗ begin0 = program->pixel_offset[x + 2];
588 ✗ begin1 = program->pixel_offset[x + 3];
589 process_two_pixels_h_float<is_safe>(src, begin0, begin1, current_coeff, filter_size, result0, result1, unaligned_kernel_size);
590 ✗ current_coeff += 2 * filter_size;
591 ✗ __m256 sumQuad1234 = _mm256_hadd_ps(sumQuad12, _mm256_hadd_ps(result0, result1));
592
593 ✗ __m128 result_lo = _mm_add_ps(_mm256_castps256_ps128(sumQuad1234), _mm256_extractf128_ps(sumQuad1234, 1)); // L1 L2 L3 L4
594
595 // 4 & 5
596 ✗ result0 = zero256;
597 ✗ result1 = zero256;
598 ✗ begin0 = program->pixel_offset[x + 4];
599 ✗ begin1 = program->pixel_offset[x + 5];
600 process_two_pixels_h_float<is_safe>(src, begin0, begin1, current_coeff, filter_size, result0, result1, unaligned_kernel_size);
601 ✗ current_coeff += 2 * filter_size;
602 ✗ __m256 sumQuad56 = _mm256_hadd_ps(result0, result1); // L1L1L1L1L1L1L1L1 + L2L2L2L2L2L2L2L2L2 = L1L1 L2L2 L1L1 L2L2
603
604 // 6 & 7
605 ✗ result0 = zero256;
606 ✗ result1 = zero256;
607 ✗ begin0 = program->pixel_offset[x + 6];
608 ✗ begin1 = program->pixel_offset[x + 7];
609 process_two_pixels_h_float<is_safe>(src, begin0, begin1, current_coeff, filter_size, result0, result1, unaligned_kernel_size);
610 //current_coeff += 2 * filter_size;
611 ✗ __m256 sumQuad5678 = _mm256_hadd_ps(sumQuad56, _mm256_hadd_ps(result0, result1));
612
613 ✗ __m128 result_hi = _mm_add_ps(_mm256_castps256_ps128(sumQuad5678), _mm256_extractf128_ps(sumQuad5678, 1)); // L1 L2 L3 L4
614
615 ✗ __m256 result256 = _mm256_insertf128_ps(_mm256_castps128_ps256(result_lo), result_hi, 1); // merge result, result_hi
616
617 ✗ _mm256_stream_ps(reinterpret_cast<float*>(dst + x), result256); // 8 results at a time
618
619 ✗ }
620
621 // filtersizealigned8: special: 1..4. Generic: -1
622 template<int filtersizealigned8>
623 ✗ static void internal_resizer_h_avx2_generic_float(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) {
624 AVS_UNUSED(bits_per_pixel);
625 ✗ const int filter_size = (filtersizealigned8 >= 1) ? filtersizealigned8 * 8 : program->filter_size;
626 // knowing a quasi-constexpr filter_size from template for commonly used sizes
627 // aligned_filter_size 8, 16, 24, 32 hugely helps compiler optimization
628
629 ✗ __m128 zero128 = _mm_setzero_ps();
630 ✗ __m256 zero256 = _mm256_setzero_ps();
631
632 ✗ const float* src = (float*)src8;
633 ✗ float* dst = (float*)dst8;
634 ✗ dst_pitch = dst_pitch / sizeof(float);
635 ✗ src_pitch = src_pitch / sizeof(float);
636
637 ✗ const int w_safe_mod8 = (program->safelimit_8_pixels.overread_possible ? program->safelimit_8_pixels.source_overread_beyond_targetx : width) / 8 * 8;
638
639 ✗ for (int y = 0; y < height; y++) {
640 ✗ float* current_coeff_base = program->pixel_coefficient_float;
641
642 // Process safe aligned pixels
643 ✗ for (int x = 0; x < w_safe_mod8; x += 8) {
644 process_eight_pixels_h_float<true>(src, x, current_coeff_base, filter_size, zero128, zero256, dst, program);
645 }
646
647 // Process up to the actual kernel size instead of the aligned filter_size to prevent overreading beyond the last source pixel.
648 // We assume extra offset entries were added to the p->pixel_offset array (aligned to 8 during initialization).
649 // This may store 1-7 false pixels, but they are ignored since Avisynth will not read beyond the width.
650 ✗ for (int x = w_safe_mod8; x < width; x += 8) {
651 process_eight_pixels_h_float<false>(src, x, current_coeff_base, filter_size, zero128, zero256, dst, program);
652 }
653
654 ✗ dst += dst_pitch;
655 ✗ src += src_pitch;
656 }
657 ✗ }
658
659 // here we cover filter sizes up to 32 (4x8) efficiently through template specialization
660 ✗ void resizer_h_avx2_generic_float(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) {
661 ✗ const int filter_size = program->filter_size;
662 ✗ assert(program->filter_size_alignment >= 8); // coeffs are float, 8x4 float can be aligned_loaded with AVX2
663
664 ✗ if (filter_size == 8)
665 ✗ internal_resizer_h_avx2_generic_float<1>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
666 ✗ else if (filter_size == 2 * 8)
667 ✗ internal_resizer_h_avx2_generic_float<2>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
668 ✗ else if (filter_size == 3 * 8)
669 ✗ internal_resizer_h_avx2_generic_float<3>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
670 ✗ else if (filter_size == 4 * 8)
671 ✗ internal_resizer_h_avx2_generic_float<4>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
672 else // -1: basic method, use program->filter_size
673 ✗ internal_resizer_h_avx2_generic_float< -1>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
674 ✗ }
675
676 // end of H float
677
678 //-------- 256 bit Verticals
679 // On x86-32 keep the 1×16 (or 2-lane/16-pixel) kernel
680 // On x86-64 use the 2×16 (4-lane/32-pixel) kernel.
681 // On 32-bit fewer YMM registers are available, 2x16 kernel causes register pressure issues.
682 // 10% performance loss on x86-32 with 2x16 kernel.
683
684 ✗ static void resize_v_avx2_planar_uint8_pix16(BYTE* AVS_RESTRICT dst, const BYTE* src, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int target_height, int bits_per_pixel)
685 {
686 AVS_UNUSED(bits_per_pixel);
687 ✗ int filter_size = program->filter_size;
688 ✗ const short* AVS_RESTRICT current_coeff = program->pixel_coefficient;
689 ✗ __m256i rounder = _mm256_set1_epi32(1 << (FPScale8bits - 1));
690 ✗ __m256i zero = _mm256_setzero_si256();
691
692 ✗ const int kernel_size = program->filter_size_real; // not the aligned
693 ✗ const int kernel_size_mod2 = (kernel_size / 2) * 2;
694
695 ✗ for (int y = 0; y < target_height; y++) {
696 ✗ int offset = program->pixel_offset[y];
697 ✗ const BYTE* AVS_RESTRICT src_ptr = src + offset * src_pitch;
698
699 // 16 byte 16 pixel
700 // no need wmod16, alignment is safe at least 32
701 ✗ for (int x = 0; x < width; x += 16) {
702
703 ✗ __m256i result_single_lo = rounder;
704 ✗ __m256i result_single_hi = rounder;
705
706 ✗ const uint8_t* AVS_RESTRICT src2_ptr = src_ptr + x;
707
708 // Process pairs of rows for better efficiency (2 coeffs/cycle)
709 ✗ int i = 0;
710 ✗ for (; i < kernel_size_mod2; i += 2) {
711
712 // Load two coefficients as a single packed value and broadcast
713 ✗ __m256i coeff = _mm256_set1_epi32(*reinterpret_cast<const int*>(current_coeff + i)); // CO|co|CO|co|CO|co|CO|co CO|co|CO|co|CO|co|CO|co
714
715 ✗ __m256i src_even = _mm256_cvtepu8_epi16(_mm_loadu_si128(reinterpret_cast<const __m128i*>(src2_ptr))); // 16x 8->16bit pixels
716 ✗ __m256i src_odd = _mm256_cvtepu8_epi16(_mm_loadu_si128(reinterpret_cast<const __m128i*>(src2_ptr + src_pitch))); // 16x 8->16bit pixels
717 ✗ __m256i src_lo = _mm256_unpacklo_epi16(src_even, src_odd);
718 ✗ __m256i src_hi = _mm256_unpackhi_epi16(src_even, src_odd);
719
720 ✗ result_single_lo = _mm256_add_epi32(result_single_lo, _mm256_madd_epi16(src_lo, coeff)); // a*b + c
721 ✗ result_single_hi = _mm256_add_epi32(result_single_hi, _mm256_madd_epi16(src_hi, coeff)); // a*b + c
722 ✗ src2_ptr += 2 * src_pitch;
723 }
724
725 // Process the last odd row if needed
726 ✗ for (; i < kernel_size; i++) {
727 // Broadcast a single coefficients
728 ✗ __m256i coeff = _mm256_set1_epi16(*reinterpret_cast<const short*>(current_coeff + i)); // 0|co|0|co|0|co|0|co 0|co|0|co|0|co|0|co
729
730 ✗ __m256i src_even = _mm256_cvtepu8_epi16(_mm_loadu_si128(reinterpret_cast<const __m128i*>(src2_ptr))); // 16x 8->16bit pixels
731 ✗ __m256i src_lo = _mm256_unpacklo_epi16(src_even, zero);
732 ✗ __m256i src_hi = _mm256_unpackhi_epi16(src_even, zero);
733 ✗ result_single_lo = _mm256_add_epi32(result_single_lo, _mm256_madd_epi16(src_lo, coeff)); // a*b + c
734 ✗ result_single_hi = _mm256_add_epi32(result_single_hi, _mm256_madd_epi16(src_hi, coeff)); // a*b + c
735 ✗ src2_ptr += src_pitch;
736
737 }
738
739 // scale back, store
740 ✗ __m256i result_lo = result_single_lo;
741 ✗ __m256i result_hi = result_single_hi;
742 // shift back integer arithmetic 14 bits precision
743 ✗ result_lo = _mm256_srai_epi32(result_lo, FPScale8bits);
744 ✗ result_hi = _mm256_srai_epi32(result_hi, FPScale8bits);
745
746 ✗ __m256i result_2x8x_uint16 = _mm256_packus_epi32(result_lo, result_hi);
747
748 ✗ __m128i result128_lo = _mm256_castsi256_si128(result_2x8x_uint16);
749 ✗ __m128i result128_hi = _mm256_extractf128_si256(result_2x8x_uint16, 1);
750 ✗ __m128i result128 = _mm_packus_epi16(result128_lo, result128_hi);
751 ✗ _mm_stream_si128(reinterpret_cast<__m128i*>(dst + x), result128);
752
753 }
754 ✗ dst += dst_pitch;
755 ✗ current_coeff += filter_size;
756 }
757 ✗ }
758
759 30 static void resize_v_avx2_planar_uint8_pix32(BYTE* AVS_RESTRICT dst, const BYTE* src, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int target_height, int bits_per_pixel)
760 {
761 AVS_UNUSED(bits_per_pixel);
762 30 int filter_size = program->filter_size;
763 30 const short* AVS_RESTRICT current_coeff = program->pixel_coefficient;
764 30 __m256i rounder = _mm256_set1_epi32(1 << (FPScale8bits - 1));
765 30 __m256i zero = _mm256_setzero_si256();
766
767 30 const int kernel_size = program->filter_size_real; // not the aligned
768 30 const int kernel_size_mod2 = (kernel_size / 2) * 2;
769
770
2/2
✓ Branch 118 → 7 taken 115 times.
✓ Branch 118 → 119 taken 30 times.
 145 for (int y = 0; y < target_height; y++) {
771 115 int offset = program->pixel_offset[y];
772 115 const BYTE* AVS_RESTRICT src_ptr = src + offset * src_pitch;
773
774 // 32 byte 32 pixel
775 // alignment is safe till 64
776
2/2
✓ Branch 116 → 9 taken 139 times.
✓ Branch 116 → 117 taken 115 times.
 254 for (int x = 0; x < width; x += 32) {
777
778 139 __m256i result_single_lo = rounder;
779 139 __m256i result_single_hi = rounder;
780
781 139 __m256i result_single_lo2 = rounder;
782 139 __m256i result_single_hi2 = rounder;
783
784 139 const uint8_t* AVS_RESTRICT src2_ptr = src_ptr + x;
785
786 // Process pairs of rows for better efficiency (2 coeffs/cycle)
787 139 int i = 0;
788
2/2
✓ Branch 53 → 10 taken 349 times.
✓ Branch 53 → 54 taken 139 times.
 488 for (; i < kernel_size_mod2; i += 2) {
789
790 // Load two coefficients as a single packed value and broadcast
791 698 __m256i coeff = _mm256_set1_epi32(*reinterpret_cast<const int*>(current_coeff + i)); // CO|co|CO|co|CO|co|CO|co CO|co|CO|co|CO|co|CO|co
792
793 349 __m256i src_even = _mm256_cvtepu8_epi16(_mm_loadu_si128(reinterpret_cast<const __m128i*>(src2_ptr))); // 16x 8->16bit pixels
794 698 __m256i src_odd = _mm256_cvtepu8_epi16(_mm_loadu_si128(reinterpret_cast<const __m128i*>(src2_ptr + src_pitch))); // 16x 8->16bit pixels
795
796 698 __m256i src_even2 = _mm256_cvtepu8_epi16(_mm_loadu_si128(reinterpret_cast<const __m128i*>(src2_ptr + 16))); // 16x 8->16bit pixels
797 1047 __m256i src_odd2 = _mm256_cvtepu8_epi16(_mm_loadu_si128(reinterpret_cast<const __m128i*>(src2_ptr + src_pitch + 16))); // 16x 8->16bit pixels
798
799
800 349 __m256i src_lo = _mm256_unpacklo_epi16(src_even, src_odd);
801 349 __m256i src_hi = _mm256_unpackhi_epi16(src_even, src_odd);
802
803 349 __m256i src_lo2 = _mm256_unpacklo_epi16(src_even2, src_odd2);
804 349 __m256i src_hi2 = _mm256_unpackhi_epi16(src_even2, src_odd2);
805
806
807 698 result_single_lo = _mm256_add_epi32(result_single_lo, _mm256_madd_epi16(src_lo, coeff)); // a*b + c
808 698 result_single_hi = _mm256_add_epi32(result_single_hi, _mm256_madd_epi16(src_hi, coeff)); // a*b + c
809
810 698 result_single_lo2 = _mm256_add_epi32(result_single_lo2, _mm256_madd_epi16(src_lo2, coeff)); // a*b + c
811 349 result_single_hi2 = _mm256_add_epi32(result_single_hi2, _mm256_madd_epi16(src_hi2, coeff)); // a*b + c
812
813 349 src2_ptr += 2 * src_pitch;
814 }
815
816 // Process the last odd row if needed
817
2/2
✓ Branch 92 → 55 taken 80 times.
✓ Branch 92 → 93 taken 139 times.
 219 for (; i < kernel_size; i++) {
818 // Broadcast a single coefficients
819 160 __m256i coeff = _mm256_set1_epi16(*reinterpret_cast<const short*>(current_coeff + i)); // 0|co|0|co|0|co|0|co 0|co|0|co|0|co|0|co
820
821 80 __m256i src_even = _mm256_cvtepu8_epi16(_mm_loadu_si128(reinterpret_cast<const __m128i*>(src2_ptr))); // 16x 8->16bit pixels
822
823 240 __m256i src_even2 = _mm256_cvtepu8_epi16(_mm_loadu_si128(reinterpret_cast<const __m128i*>(src2_ptr + 16))); // 16x 8->16bit pixels
824
825 80 __m256i src_lo = _mm256_unpacklo_epi16(src_even, zero);
826 80 __m256i src_hi = _mm256_unpackhi_epi16(src_even, zero);
827
828 80 __m256i src_lo2 = _mm256_unpacklo_epi16(src_even2, zero);
829 80 __m256i src_hi2 = _mm256_unpackhi_epi16(src_even2, zero);
830
831 160 result_single_lo = _mm256_add_epi32(result_single_lo, _mm256_madd_epi16(src_lo, coeff)); // a*b + c
832 160 result_single_hi = _mm256_add_epi32(result_single_hi, _mm256_madd_epi16(src_hi, coeff)); // a*b + c
833
834 160 result_single_lo2 = _mm256_add_epi32(result_single_lo2, _mm256_madd_epi16(src_lo2, coeff)); // a*b + c
835 80 result_single_hi2 = _mm256_add_epi32(result_single_hi2, _mm256_madd_epi16(src_hi2, coeff)); // a*b + c
836
837 80 src2_ptr += src_pitch;
838
839 }
840
841 // scale back, store
842 139 __m256i result_lo = result_single_lo;
843 139 __m256i result_hi = result_single_hi;
844
845 139 __m256i result_lo2 = result_single_lo2;
846 139 __m256i result_hi2 = result_single_hi2;
847
848
849 // shift back integer arithmetic 14 bits precision
850 139 result_lo = _mm256_srai_epi32(result_lo, FPScale8bits);
851 139 result_hi = _mm256_srai_epi32(result_hi, FPScale8bits);
852
853 139 result_lo2 = _mm256_srai_epi32(result_lo2, FPScale8bits);
854 139 result_hi2 = _mm256_srai_epi32(result_hi2, FPScale8bits);
855
856 139 __m256i result_2x8x_uint16 = _mm256_packus_epi32(result_lo, result_hi);
857
858 139 __m256i result_2x8x_uint16_2 = _mm256_packus_epi32(result_lo2, result_hi2);
859
860 139 __m128i result128_lo = _mm256_castsi256_si128(result_2x8x_uint16);
861 139 __m128i result128_hi = _mm256_extractf128_si256(result_2x8x_uint16, 1);
862 139 __m128i result128 = _mm_packus_epi16(result128_lo, result128_hi);
863
864 139 __m128i result128_lo2 = _mm256_castsi256_si128(result_2x8x_uint16_2);
865 139 __m128i result128_hi2 = _mm256_extractf128_si256(result_2x8x_uint16_2, 1);
866 139 __m128i result128_2 = _mm_packus_epi16(result128_lo2, result128_hi2);
867
868
869 139 _mm_stream_si128(reinterpret_cast<__m128i*>(dst + x), result128);
870 139 _mm_stream_si128(reinterpret_cast<__m128i*>(dst + x + 16), result128_2);
871
872 }
873 115 dst += dst_pitch;
874 115 current_coeff += filter_size;
875 }
876 30 }
877
878 #if defined(X86_64)
879 30 static void resize_v_avx2_planar_impl(BYTE* dst8, const BYTE* src, int dst_pitch, int src_pitch,
880 ResamplingProgram* program, int width, int target_height, int bits_per_pixel)
881 {
882 30 resize_v_avx2_planar_uint8_pix32(dst8, src, dst_pitch, src_pitch, program, width, target_height, bits_per_pixel);
883 30 }
884 #elif defined(X86_32)
885 static void resize_v_avx2_planar_impl(BYTE* dst8, const BYTE* src, int dst_pitch, int src_pitch,
886 ResamplingProgram* program, int width, int target_height, int bits_per_pixel)
887 {
888 resize_v_avx2_planar_uint8_pix16(dst8, src, dst_pitch, src_pitch, program, width, target_height, bits_per_pixel);
889 }
890 #else
891 #error Unsupported target for resize_v_avx2_planar_uint8_t
892 #endif
893
894 30 void resize_v_avx2_planar_uint8_t(BYTE* dst8, const BYTE* src, int dst_pitch, int src_pitch,
895 ResamplingProgram* program, int width, int target_height, int bits_per_pixel)
896 {
897 30 resize_v_avx2_planar_impl(dst8, src, dst_pitch, src_pitch, program, width, target_height, bits_per_pixel);
898 30 }
899
900
901 template<bool lessthan16bit>
902 13 void resize_v_avx2_planar_uint16_t(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int target_height, int bits_per_pixel)
903 {
904 13 int filter_size = program->filter_size;
905 13 const short* AVS_RESTRICT current_coeff = program->pixel_coefficient;
906
907 13 const __m256i zero = _mm256_setzero_si256();
908
909 // for 16 bits only
910 13 const __m256i shifttosigned = _mm256_set1_epi16(-32768);
911 13 const __m256i shiftfromsigned = _mm256_set1_epi32(32768 << FPScale16bits);
912
913 13 const __m256i rounder = _mm256_set1_epi32(1 << (FPScale16bits - 1));
914
915 13 const uint16_t* src = (uint16_t*)src8;
916 13 uint16_t* AVS_RESTRICT dst = (uint16_t* AVS_RESTRICT)dst8;
917 13 dst_pitch = dst_pitch / sizeof(uint16_t);
918 13 src_pitch = src_pitch / sizeof(uint16_t);
919
920 13 const int kernel_size = program->filter_size_real; // not the aligned
921 13 const int kernel_size_mod2 = (kernel_size / 2) * 2;
922
923 13 const int limit = (1 << bits_per_pixel) - 1;
924 13 __m256i clamp_limit = _mm256_set1_epi16((short)limit); // clamp limit for <16 bits
925
926
4/4
void resize_v_avx2_planar_uint16_t<false>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 81 → 17 taken 39 times.
✓ Branch 81 → 82 taken 9 times.
void resize_v_avx2_planar_uint16_t<true>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 73 → 17 taken 23 times.
✓ Branch 73 → 74 taken 4 times.
 75 for (int y = 0; y < target_height; y++) {
927 62 int offset = program->pixel_offset[y];
928 62 const uint16_t* src_ptr = src + offset * src_pitch;
929
930 // 32 byte 16 word
931 // no need wmod16, alignment is safe at least 32
932
933
4/4
void resize_v_avx2_planar_uint16_t<false>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 79 → 19 taken 83 times.
✓ Branch 79 → 80 taken 39 times.
void resize_v_avx2_planar_uint16_t<true>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 71 → 19 taken 64 times.
✓ Branch 71 → 72 taken 23 times.
 209 for (int x = 0; x < width; x += 16) {
934
935 147 __m256i result_single_lo = rounder;
936 147 __m256i result_single_hi = rounder;
937
938 147 const uint16_t* AVS_RESTRICT src2_ptr = src_ptr + x;
939
940 // Process pairs of rows for better efficiency (2 coeffs/cycle)
941 147 int i = 0;
942
4/4
void resize_v_avx2_planar_uint16_t<false>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 43 → 20 taken 260 times.
✓ Branch 43 → 44 taken 83 times.
void resize_v_avx2_planar_uint16_t<true>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 39 → 20 taken 172 times.
✓ Branch 39 → 40 taken 64 times.
 579 for (; i < kernel_size_mod2; i += 2) {
943 // Load two coefficients as a single packed value and broadcast
944 864 __m256i coeff = _mm256_set1_epi32(*reinterpret_cast<const int*>(current_coeff + i)); // CO|co|CO|co|CO|co|CO|co CO|co|CO|co|CO|co|CO|co
945
946 432 __m256i src_even = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src2_ptr)); // 16x 16bit pixels
947 864 __m256i src_odd = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src2_ptr + src_pitch)); // 16x 16bit pixels
948 if constexpr (!lessthan16bit) {
949 260 src_even = _mm256_add_epi16(src_even, shifttosigned);
950 260 src_odd = _mm256_add_epi16(src_odd, shifttosigned);
951 }
952 432 __m256i src_lo = _mm256_unpacklo_epi16(src_even, src_odd);
953 432 __m256i src_hi = _mm256_unpackhi_epi16(src_even, src_odd);
954
955 864 result_single_lo = _mm256_add_epi32(result_single_lo, _mm256_madd_epi16(src_lo, coeff)); // a*b + c
956 432 result_single_hi = _mm256_add_epi32(result_single_hi, _mm256_madd_epi16(src_hi, coeff)); // a*b + c
957
958 432 src2_ptr += 2 * src_pitch;
959 }
960
961 // Process the last odd row if needed
962
4/4
void resize_v_avx2_planar_uint16_t<false>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 66 → 45 taken 50 times.
✓ Branch 66 → 67 taken 83 times.
void resize_v_avx2_planar_uint16_t<true>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 60 → 41 taken 64 times.
✓ Branch 60 → 61 taken 64 times.
 261 for (; i < kernel_size; i++) {
963 // Broadcast a single coefficients
964 228 __m256i coeff = _mm256_set1_epi16(current_coeff[i]); // 0|co|0|co|0|co|0|co 0|co|0|co|0|co|0|co
965
966 114 __m256i src_even = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src2_ptr)); // 16x 16bit pixels
967 if constexpr (!lessthan16bit) {
968 50 src_even = _mm256_add_epi16(src_even, shifttosigned);
969 }
970 114 __m256i src_lo = _mm256_unpacklo_epi16(src_even, zero);
971 114 __m256i src_hi = _mm256_unpackhi_epi16(src_even, zero);
972 228 result_single_lo = _mm256_add_epi32(result_single_lo, _mm256_madd_epi16(src_lo, coeff)); // a*b + c
973 114 result_single_hi = _mm256_add_epi32(result_single_hi, _mm256_madd_epi16(src_hi, coeff)); // a*b + c
974
975 114 src2_ptr += src_pitch;
976 }
977
978 // correct if signed, scale back, store
979 147 __m256i result_lo = result_single_lo;
980 147 __m256i result_hi = result_single_hi;
981 if constexpr (!lessthan16bit) {
982 83 result_lo = _mm256_add_epi32(result_lo, shiftfromsigned);
983 83 result_hi = _mm256_add_epi32(result_hi, shiftfromsigned);
984 }
985 // shift back integer arithmetic 13 bits precision
986 147 result_lo = _mm256_srai_epi32(result_lo, FPScale16bits);
987 147 result_hi = _mm256_srai_epi32(result_hi, FPScale16bits);
988
989 147 __m256i result_2x8x_uint16 = _mm256_packus_epi32(result_lo, result_hi);
990 if constexpr (lessthan16bit) {
991 64 result_2x8x_uint16 = _mm256_min_epu16(result_2x8x_uint16, clamp_limit); // extra clamp for 10-14 bit
992 }
993 147 _mm256_stream_si256(reinterpret_cast<__m256i*>(dst + x), result_2x8x_uint16);
994
995 }
996
997 62 dst += dst_pitch;
998 62 current_coeff += filter_size;
999 }
1000 13 }
1001
1002 //-------- 256 bit float Verticals
1003
1004 1 void resize_v_avx2_planar_float(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int target_height, int bits_per_pixel)
1005 {
1006 AVS_UNUSED(bits_per_pixel);
1007
1008 1 const int filter_size = program->filter_size;
1009 1 const float* AVS_RESTRICT current_coeff = program->pixel_coefficient_float;
1010
1011 1 const float* src = (const float*)src8;
1012 1 float* AVS_RESTRICT dst = (float*)dst8;
1013 1 dst_pitch = dst_pitch / sizeof(float);
1014 1 src_pitch = src_pitch / sizeof(float);
1015
1016 1 const int kernel_size = program->filter_size_real; // not the aligned
1017 1 const int kernel_size_mod2 = (kernel_size / 2) * 2; // Process pairs of rows for better efficiency
1018 1 const bool notMod2 = kernel_size_mod2 < kernel_size;
1019
1020
2/2
✓ Branch 38 → 3 taken 5 times.
✓ Branch 38 → 39 taken 1 time.
 6 for (int y = 0; y < target_height; y++) {
1021 5 int offset = program->pixel_offset[y];
1022 5 const float* src_ptr = src + offset * src_pitch;
1023
1024 // 32 byte 8 floats (AVX2 register holds 8 floats)
1025 // no need for wmod8, alignment is safe 32 bytes at least
1026
2/2
✓ Branch 36 → 5 taken 25 times.
✓ Branch 36 → 37 taken 5 times.
 30 for (int x = 0; x < width; x += 8) {
1027 25 __m256 result_single = _mm256_setzero_ps();
1028 25 __m256 result_single_2 = _mm256_setzero_ps();
1029
1030 25 const float* AVS_RESTRICT src2_ptr = src_ptr + x; // __restrict here
1031
1032 // Process pairs of rows for better efficiency (2 coeffs/cycle)
1033 // two result variables for potential parallel operation
1034 25 int i = 0;
1035
2/2
✓ Branch 23 → 10 taken 25 times.
✓ Branch 23 → 24 taken 25 times.
 50 for (; i < kernel_size_mod2; i += 2) {
1036 25 __m256 coeff_even = _mm256_set1_ps(current_coeff[i]);
1037 50 __m256 coeff_odd = _mm256_set1_ps(current_coeff[i + 1]);
1038
1039 25 __m256 src_even = _mm256_loadu_ps(src2_ptr);
1040 50 __m256 src_odd = _mm256_loadu_ps(src2_ptr + src_pitch);
1041
1042 25 result_single = _mm256_fmadd_ps(src_even, coeff_even, result_single);
1043 25 result_single_2 = _mm256_fmadd_ps(src_odd, coeff_odd, result_single_2);
1044
1045 25 src2_ptr += 2 * src_pitch;
1046 }
1047
1048 25 result_single = _mm256_add_ps(result_single, result_single_2);
1049
1050 // Process the last odd row if needed
1051
1/2
✓ Branch 26 → 27 taken 25 times.
✗ Branch 26 → 34 not taken.
 25 if (notMod2) {
1052 50 __m256 coeff = _mm256_set1_ps(current_coeff[i]);
1053 25 __m256 src_val = _mm256_loadu_ps(src2_ptr);
1054 25 result_single = _mm256_fmadd_ps(src_val, coeff, result_single);
1055 }
1056
1057 25 _mm256_stream_ps(dst + x, result_single);
1058 }
1059
1060 5 dst += dst_pitch;
1061 5 current_coeff += filter_size;
1062 }
1063 1 }
1064
1065 // Memory-transfer optimized version of resize_v_avx2_planar_float
1066 2 void resize_v_avx2_planar_float_w_sr(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int target_height, int bits_per_pixel)
1067 {
1068 AVS_UNUSED(bits_per_pixel);
1069
1070 2 const int filter_size = program->filter_size;
1071 2 const float* AVS_RESTRICT current_coeff = program->pixel_coefficient_float;
1072
1073 2 const float* src = (const float*)src8;
1074 2 float* AVS_RESTRICT dst = (float*)dst8;
1075 2 dst_pitch = dst_pitch / sizeof(float);
1076 2 src_pitch = src_pitch / sizeof(float);
1077
1078 2 const int kernel_size = program->filter_size_real; // not the aligned
1079 2 const int kernel_size_mod2 = (kernel_size / 2) * 2; // Process pairs of rows for better efficiency
1080 2 const bool notMod2 = kernel_size_mod2 < kernel_size;
1081
1082 2 const int width_mod32 = (width / 32) * 32;
1083
1084
2/2
✓ Branch 74 → 3 taken 11 times.
✓ Branch 74 → 75 taken 2 times.
 13 for (int y = 0; y < target_height; y++) {
1085 11 int offset = program->pixel_offset[y];
1086 11 const float* src_ptr = src + offset * src_pitch;
1087 // Part #1: process 32 floats at a time
1088 // Optimize for memory throughput: process 32 floats (4x256bit) in parallel
1089 // Process by 4x 256bit (8 x 8 floats) to make memory read/write linear streams
1090 // longer, 16x256 bit registers in 64bit mode should be enough
1091
2/2
✓ Branch 39 → 5 taken 11 times.
✓ Branch 39 → 40 taken 11 times.
 22 for (int x = 0; x < width_mod32; x += 32) {
1092 11 __m256 result_1 = _mm256_setzero_ps();
1093 11 __m256 result_2 = _mm256_setzero_ps();
1094 11 __m256 result_3 = _mm256_setzero_ps();
1095 11 __m256 result_4 = _mm256_setzero_ps();
1096
1097 11 const float* AVS_RESTRICT src2_ptr = src_ptr + x; // __restrict here
1098 // single coeffs/cycle, but 32 floats processed in parallel
1099
2/2
✓ Branch 33 → 14 taken 33 times.
✓ Branch 33 → 34 taken 11 times.
 44 for (int i = 0; i < kernel_size; i++) {
1100 // coefs are equal for all H-samples
1101 66 __m256 coeff = _mm256_set1_ps(current_coeff[i]);
1102
1103 // source always aligned in V-resizers
1104 33 __m256 src_1 = _mm256_load_ps(src2_ptr);
1105 33 __m256 src_2 = _mm256_load_ps(src2_ptr + 8);
1106 33 __m256 src_3 = _mm256_load_ps(src2_ptr + 16);
1107 66 __m256 src_4 = _mm256_load_ps(src2_ptr + 24);
1108
1109 33 result_1 = _mm256_fmadd_ps(src_1, coeff, result_1);
1110 33 result_2 = _mm256_fmadd_ps(src_2, coeff, result_2);
1111 33 result_3 = _mm256_fmadd_ps(src_3, coeff, result_3);
1112 33 result_4 = _mm256_fmadd_ps(src_4, coeff, result_4);
1113
1114 33 src2_ptr += src_pitch;
1115 }
1116 // here we use stream instead of store; in multithreading stream is better;
1117 // consider two templated versions if needed depending on actual MT usage
1118 11 _mm256_stream_ps(dst + x, result_1);
1119 11 _mm256_stream_ps(dst + x + 8, result_2);
1120 11 _mm256_stream_ps(dst + x + 16, result_3);
1121 11 _mm256_stream_ps(dst + x + 24, result_4);
1122 } // width_mod32
1123
1124 // Part #2: process remaining. 32 byte 8 floats (AVX2 register holds 8 floats)
1125 // From now on the old resize_v_avx2_planar_float, starting at width_mod32.
1126
1127 // No need for wmod8, scanline alignment is safe 32 bytes at least (really 64)
1128
2/2
✓ Branch 72 → 41 taken 11 times.
✓ Branch 72 → 73 taken 11 times.
 22 for (int x = width_mod32; x < width; x += 8) {
1129 11 __m256 result_single = _mm256_setzero_ps();
1130 11 __m256 result_single_2 = _mm256_setzero_ps();
1131
1132 11 const float* AVS_RESTRICT src2_ptr = src_ptr + x; // __restrict here
1133
1134 // Process pairs of rows for better efficiency (2 coeffs/cycle)
1135 // two result variables for potential parallel operation
1136 11 int i = 0;
1137
2/2
✓ Branch 59 → 46 taken 11 times.
✓ Branch 59 → 60 taken 11 times.
 22 for (; i < kernel_size_mod2; i += 2) {
1138 11 __m256 coeff_even = _mm256_set1_ps(current_coeff[i]);
1139 22 __m256 coeff_odd = _mm256_set1_ps(current_coeff[i + 1]);
1140
1141 11 __m256 src_even = _mm256_load_ps(src2_ptr);
1142 22 __m256 src_odd = _mm256_load_ps(src2_ptr + src_pitch);
1143
1144 11 result_single = _mm256_fmadd_ps(src_even, coeff_even, result_single);
1145 11 result_single_2 = _mm256_fmadd_ps(src_odd, coeff_odd, result_single_2);
1146
1147 11 src2_ptr += 2 * src_pitch;
1148 }
1149
1150 11 result_single = _mm256_add_ps(result_single, result_single_2);
1151
1152 // Process the last odd row if needed
1153
1/2
✓ Branch 62 → 63 taken 11 times.
✗ Branch 62 → 70 not taken.
 11 if (notMod2) {
1154 22 __m256 coeff = _mm256_set1_ps(current_coeff[i]);
1155 11 __m256 src_val = _mm256_load_ps(src2_ptr);
1156 11 result_single = _mm256_fmadd_ps(src_val, coeff, result_single);
1157 }
1158
1159 11 _mm256_stream_ps(dst + x, result_single);
1160 }
1161
1162 11 dst += dst_pitch;
1163 11 current_coeff += filter_size;
1164 }
1165 2 }
1166
1167 // avx2 16bit
1168 template void resizer_h_avx2_generic_uint16_t<false>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
1169 // avx2 10-14bit
1170 template void resizer_h_avx2_generic_uint16_t<true>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
1171
1172
1173
1174 // avx2 16
1175 template void resize_v_avx2_planar_uint16_t<false>(BYTE* dst0, const BYTE* src0, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int target_height, int bits_per_pixel);
1176 // avx2 10-14bit
1177 template void resize_v_avx2_planar_uint16_t<true>(BYTE* dst0, const BYTE* src0, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int target_height, int bits_per_pixel);
1178
1179
1180
1181 // Helper for horizontal resampling 32 bit float
1182 // Safe dual lane partial load with AVX
1183 // Read exactly N pixels, avoiding
1184 // - reading beyond the end of the source buffer.
1185 // - avoid NaN contamination, since event with zero coefficients NaN * 0 = NaN
1186 template <int Nmod4>
1187 AVS_FORCEINLINE static __m256 _mm256_load_partial_safe_2_m128(const float* src_ptr_offsetted1, const float* src_ptr_offsetted2) {
1188 __m128 s1;
1189 __m128 s2;
1190 switch (Nmod4) {
1191 case 1:
1192 ✗ s1 = _mm_set_ps(0.0f, 0.0f, 0.0f, src_ptr_offsetted1[0]);
1193 ✗ s2 = _mm_set_ps(0.0f, 0.0f, 0.0f, src_ptr_offsetted2[0]);
1194 // ideally: movss
1195 ✗ break;
1196 case 2:
1197 80 s1 = _mm_set_ps(0.0f, 0.0f, src_ptr_offsetted1[1], src_ptr_offsetted1[0]);
1198 40 s2 = _mm_set_ps(0.0f, 0.0f, src_ptr_offsetted2[1], src_ptr_offsetted2[0]);
1199 // ideally: movsd
1200 40 break;
1201 case 3:
1202 ✗ s1 = _mm_set_ps(0.0f, src_ptr_offsetted1[2], src_ptr_offsetted1[1], src_ptr_offsetted1[0]);
1203 ✗ s2 = _mm_set_ps(0.0f, src_ptr_offsetted2[2], src_ptr_offsetted2[1], src_ptr_offsetted2[0]);
1204 // ideally: movss + movsd + shuffle or movsd + insert
1205 ✗ break;
1206 case 0:
1207 ✗ s1 = _mm_set_ps(src_ptr_offsetted1[3], src_ptr_offsetted1[2], src_ptr_offsetted1[1], src_ptr_offsetted1[0]);
1208 ✗ s2 = _mm_set_ps(src_ptr_offsetted2[3], src_ptr_offsetted2[2], src_ptr_offsetted2[1], src_ptr_offsetted2[0]);
1209 // ideally: movups
1210 ✗ break;
1211 default:
1212 s1 = _mm_setzero_ps(); // n/a cannot happen
1213 s2 = _mm_setzero_ps();
1214 }
1215 40 return _mm256_set_m128(s2, s1);
1216 }
1217
1218
1219 // Processes a horizontal resampling kernel of up to four coefficients for float pixel types.
1220 // Supports BilinearResize, BicubicResize, or sinc with up to 2 taps (filter size <= 4).
1221 // Loads and processes four float coefficients and eight pixels simultaneously.
1222 // The 'filtersizemod4' template parameter (0-3) helps optimize for different filter sizes modulo 4.
1223
1224 // this is a generic varsion for small kernels up to 4 taps, regardless of up or down scaling
1225 // Note: there is a further optimized version of ks4 resampler, which combines gather or permutex.
1226 template<int filtersizemod4>
1227 ✗ void resize_h_planar_float_avx_transpose_vstripe_ks4(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) {
1228 assert(filtersizemod4 >= 0 && filtersizemod4 <= 3);
1229
1230 ✗ const int filter_size = program->filter_size; // aligned, practically the coeff table stride
1231
1232 ✗ src_pitch /= sizeof(float);
1233 ✗ dst_pitch /= sizeof(float);
1234
1235 ✗ float* src = (float*)src8;
1236 ✗ float* dst = (float*)dst8;
1237
1238 ✗ const float* AVS_RESTRICT current_coeff = (const float* AVS_RESTRICT)program->pixel_coefficient_float;
1239
1240 ✗ constexpr int PIXELS_AT_A_TIME = 8; // Process eight pixels in parallel using AVX2 (2x4 using m128 lanes)
1241
1242 // 'source_overread_beyond_targetx' indicates if the filter kernel can read beyond the target width.
1243 // Even if the filter alignment allows larger reads, our safety boundary for unaligned loads starts at 4 pixels back
1244 // from the target width, as we load 4 floats at once with '_mm_loadu_ps'.
1245 ✗ const int width_safe_mod = (program->safelimit_4_pixels.overread_possible ? program->safelimit_4_pixels.source_overread_beyond_targetx : width) / PIXELS_AT_A_TIME * PIXELS_AT_A_TIME;
1246
1247 // Preconditions:
1248 ✗ assert(program->filter_size_real <= 4); // We preload all relevant coefficients (up to 4) before the height loop.
1249
1250 // 'target_size_alignment' ensures we can safely access coefficients using offsets like
1251 // 'filter_size * 7' when processing 8 H pixels at a time or
1252 // 'filter_size * 15' when processing 16 H pixels at a time
1253 ✗ assert(program->target_size_alignment >= 8);
1254
1255 // Ensure that coefficient loading beyond the valid target size is safe for 4x4 float loads.
1256 ✗ assert(program->filter_size_alignment >= 4);
1257
1258 ✗ int x = 0;
1259
1260 // This 'auto' lambda construct replaces the need of templates
1261 ✗ auto do_h_float_core = [&](auto partial_load) {
1262 // Load up to 2x4 coefficients at once before the height loop.
1263 // Pre-loading and transposing coefficients keeps register usage efficient.
1264 // Assumes 'filter_size_aligned' is at least 4.
1265
1266 // Coefficients for the source pixel offset (for src_ptr + begin1 [0..3] and for src_ptr + begin5 [0..3] )
1267 ✗ __m256 coef_1_coef_5 = _mm256_load_2_m128(current_coeff + filter_size * 0, current_coeff + filter_size * 4);
1268 ✗ __m256 coef_2_coef_6 = _mm256_load_2_m128(current_coeff + filter_size * 1, current_coeff + filter_size * 5);
1269 ✗ __m256 coef_3_coef_7 = _mm256_load_2_m128(current_coeff + filter_size * 2, current_coeff + filter_size * 6);
1270 ✗ __m256 coef_4_coef_8 = _mm256_load_2_m128(current_coeff + filter_size * 3, current_coeff + filter_size * 7);
1271
1272 ✗ _MM_TRANSPOSE8_LANE4_PS(coef_1_coef_5, coef_2_coef_6, coef_3_coef_7, coef_4_coef_8);
1273
1274 ✗ float* AVS_RESTRICT dst_ptr = dst + x;
1275 ✗ const float* src_ptr = src;
1276
1277 // Pixel offsets for the current target x-positions.
1278 // Even for x >= width, these offsets are guaranteed to be within the allocated 'target_size_alignment'.
1279 ✗ const int begin1 = program->pixel_offset[x + 0];
1280 ✗ const int begin2 = program->pixel_offset[x + 1];
1281 ✗ const int begin3 = program->pixel_offset[x + 2];
1282 ✗ const int begin4 = program->pixel_offset[x + 3];
1283 ✗ const int begin5 = program->pixel_offset[x + 4];
1284 ✗ const int begin6 = program->pixel_offset[x + 5];
1285 ✗ const int begin7 = program->pixel_offset[x + 6];
1286 ✗ const int begin8 = program->pixel_offset[x + 7];
1287
1288 ✗ for (int y = 0; y < height; y++)
1289 {
1290 __m256 data_1_data_5;
1291 __m256 data_2_data_6;
1292 __m256 data_3_data_7;
1293 __m256 data_4_data_8;
1294
1295 if constexpr (partial_load) {
1296 // In the potentially unsafe zone (near the right edge of the image), we use a safe loading function
1297 // to prevent reading beyond the allocated source scanline. This handles cases where loading 4 floats
1298 // starting from 'src_ptr + beginX' might exceed the source buffer.
1299
1300 // Example of the unsafe scenario: If target width is 320, a naive load at src_ptr + 317
1301 // would attempt to read floats at indices 317, 318, 319, and 320, potentially going out of bounds.
1302
1303 // Two main issues in the unsafe zone:
1304 // 1.) Out-of-bounds memory access: Reading beyond the allocated memory for the source scanline can
1305 // lead to access violations and crashes. '_mm_loadu_ps' attempts to load 16 bytes, so even if
1306 // the starting address is within bounds, subsequent reads might not be.
1307 // 2.) Garbage or NaN values: Even if a read doesn't cause a crash, accessing uninitialized or
1308 // out-of-bounds memory (especially for float types) can result in garbage data, including NaN.
1309 // Multiplying by a valid coefficient and accumulating this NaN can contaminate the final result.
1310
1311 // '_mm256_load_partial_safe_2_m128' safely loads up to 'filter_size_real' pixels and pads with zeros if needed,
1312 // preventing out-of-bounds reads and ensuring predictable results even near the image edges.
1313
1314 ✗ data_1_data_5 = _mm256_load_partial_safe_2_m128<filtersizemod4>(src_ptr + begin1, src_ptr + begin5);
1315 ✗ data_2_data_6 = _mm256_load_partial_safe_2_m128<filtersizemod4>(src_ptr + begin2, src_ptr + begin6);
1316 ✗ data_3_data_7 = _mm256_load_partial_safe_2_m128<filtersizemod4>(src_ptr + begin3, src_ptr + begin7);
1317 ✗ data_4_data_8 = _mm256_load_partial_safe_2_m128<filtersizemod4>(src_ptr + begin4, src_ptr + begin8);
1318 }
1319 else {
1320 // In the safe zone, we can directly load 4 pixels at a time using unaligned loads.
1321 ✗ data_1_data_5 = _mm256_loadu_2_m128(src_ptr + begin1, src_ptr + begin5);
1322 ✗ data_2_data_6 = _mm256_loadu_2_m128(src_ptr + begin2, src_ptr + begin6);
1323 ✗ data_3_data_7 = _mm256_loadu_2_m128(src_ptr + begin3, src_ptr + begin7);
1324 ✗ data_4_data_8 = _mm256_loadu_2_m128(src_ptr + begin4, src_ptr + begin8);
1325 }
1326
1327 ✗ _MM_TRANSPOSE8_LANE4_PS(data_1_data_5, data_2_data_6, data_3_data_7, data_4_data_8);
1328
1329 // two sets, hint for the compiler to allow parallel fma's
1330 ✗ __m256 result_0 = _mm256_mul_ps(data_1_data_5, coef_1_coef_5);
1331 ✗ __m256 result_1 = _mm256_mul_ps(data_2_data_6, coef_2_coef_6);
1332 ✗ result_0 = _mm256_fmadd_ps(data_3_data_7, coef_3_coef_7, result_0);
1333 ✗ result_1 = _mm256_fmadd_ps(data_4_data_8, coef_4_coef_8, result_1);
1334
1335 ✗ _mm256_stream_ps(dst_ptr, _mm256_add_ps(result_0, result_1));
1336
1337 ✗ dst_ptr += dst_pitch;
1338 ✗ src_ptr += src_pitch;
1339 } // y
1340 ✗ current_coeff += filter_size * 8; // Move to the next set of coefficients for the next 8 output pixels
1341 }; // end of lambda
1342
1343 // Process the 'safe zone' where direct full unaligned loads are acceptable.
1344 ✗ for (; x < width_safe_mod; x += PIXELS_AT_A_TIME)
1345 {
1346 ✗ do_h_float_core(std::false_type{}); // partial_load == false, use direct _mm_loadu_ps
1347 }
1348
1349 // Process the potentially 'unsafe zone' near the image edge, using safe loading.
1350 ✗ for (; x < width; x += PIXELS_AT_A_TIME)
1351 {
1352 ✗ do_h_float_core(std::true_type{}); // partial_load == true, use the safer '_mm256_load_partial_safe_2_m128'
1353 }
1354 ✗ }
1355
1356 // Instantiate them
1357 template void resize_h_planar_float_avx_transpose_vstripe_ks4<0>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
1358 template void resize_h_planar_float_avx_transpose_vstripe_ks4<1>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
1359 template void resize_h_planar_float_avx_transpose_vstripe_ks4<2>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
1360 template void resize_h_planar_float_avx_transpose_vstripe_ks4<3>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
1361
1362
1363 /**
1364 * resize_h_planar_float_avx2_gather_permutex_vstripe_ks4 with per-frame check or
1365 * distinct checker, gather, permutex methods
1366 *
1367 * AVX2-optimized horizontal resampler for float planar images with small kernel sizes (filter_size_real <= 4).
1368 * Supports both upsampling and downsampling scenarios, automatically selecting the most efficient SIMD strategy.
1369 * (For larger kernels, use resizer_h_avx2_generic_float or other specialized functions.)
1370 *
1371 * Algorithm:
1372 * - Analyzes the resampling program's pixel offset pattern to choose between two SIMD strategies.
1373 * The upsampling scenario is divided into sub-cases, and the decision is made by analyzing the pixel offset pattern in the resampling program.
1374 * The code checks, for each group of 8 output pixels, how far apart the corresponding source pixel offsets are.
1375 *
1376 * If the span of source pixels (end_off - start_off) plus the kernel size (max filter_size_real - 1, that is 3)
1377 * exceeds 8, it goes to gather based method: the required source pixels are not all within a single 8-float block.
1378 *
1379 * If the span is <= 8, the function can use a single 8-float block load and permute (which is faster).
1380 *
1381 * For "high upsampling ratio" (output much larger than input, so output pixels are close together in input),
1382 * the offsets are usually contiguous, and the permute-transpose path is used.
1383 *
1384 * 1. Gather-based: For downsampling (or no-resize convolution) or non-contiguous pixel offsets, uses AVX2
1385 * gather instructions to fetch each required source pixel.
1386 * 2. Permutex-based: For upsampling or contiguous pixel offsets, loads a block of 8 source floats and uses
1387 * AVX2 permute instructions for fast access.
1388 *
1389 * - Handles edge cases and buffer boundaries safely, using partial loads to avoid out-of-bounds memory access.
1390 * - Processes 8 output pixels in parallel for high throughput.
1391 *
1392 * Assumes that resampling program provides sufficient alignment and padding for safe SIMD loads.
1393 *
1394 * Typical dispatcher usage:
1395 * switch (program->filter_size_real) {
1396 * case 1: return resize_h_planar_float_avx2_gather_permutex_vstripe_ks4<1>;
1397 * case 2: return resize_h_planar_float_avx2_gather_permutex_vstripe_ks4<2>;
1398 * case 3: return resize_h_planar_float_avx2_gather_permutex_vstripe_ks4<3>;
1399 * case 4: return resize_h_planar_float_avx2_gather_permutex_vstripe_ks4<0>;
1400 * default: return resizer_h_avx2_generic_float;
1401 * }
1402 *
1403 * See also:
1404 * - resize_h_planar_float_avx2_transpose_vstripe_ks4
1405 * - resize_h_planar_float_avx2_permutex_vstripe_ks4
1406 * - resize_h_planar_float_avx_transpose_vstripe_ks4
1407 * - resizer_h_avx2_generic_float
1408
1409 */
1410
1411 // Test script for the ks<=4 gather/permutex horizontal, float resampler cases
1412 /*
1413 SetMaxCPU("AVX2")
1414 BlankClip(width=640, height=480, pixel_type="YUV444PS")
1415 #BlankClip(width=640-1, height=480, pixel_type="YUV444PS") # -1 to -7 to test partial loads
1416 Expr("sx 2 % 1.0 * ", "0", "0") # vertical stripes
1417 BicubicResize(width*2,height) # permute, H kernel size 4
1418 or
1419 LanczosResize(width*2, height, taps=1) # permute, H kernel size 2
1420 or
1421 LanczosResize(int(width*0.5), height, taps=1) # gather, H kernel size 4
1422 or
1423 BilinearResize(int(width*0.97), height) # gather, H kernel size 3
1424 */
1425
1426 /*
1427 * Analyse input resampling program to select method of processing.
1428 *
1429 * This check determines whether the AVX2 permutex optimization is valid for a block of 8 output pixels.
1430 * In the permutex path, we load 8 consecutive source floats starting at program->pixel_offset[x + 0] ('begin1').
1431 * Each output pixel's convolution window is indexed using perm_0..perm_3, which are offsets relative to begin1.
1432 * These permutation indices span from begin1 (program->pixel_offset[x + 0]) up to begin8 + 3 (program->pixel_offset[x + 7] + 3).
1433 * For the permute to be safe, ALL indices accessed (from begin1 to begin8 + 3) must fit within the loaded 8-float block.
1434 * This is guaranteed if (program->pixel_offset[x + 7] + 3 - program->pixel_offset[x + 0]) < 8.
1435 * In order the check work for the right edge, pixel_offset entries padded till target_size_aligned must repeat the last
1436 * valid offset, and not 0 (see in resize_prepare_coeffs).
1437 *
1438 * If the span is not in the 0-7 range, some required source pixels for the convolution will fall outside
1439 * the loaded block, and the permutex method cannot be used; we must fall back to gather.
1440 * This logic relies on the assumption that pixel_offset[] is strictly increasing (or non-decreasing).
1441 * We check the maximum index accessed by the permutation logic, and since we use a fixed 4 coefficients
1442 * per output pixel, not just the filter_size_real, we add 3 to the last offset.
1443 *
1444 * It is ensured during the resampling program setup (resize_prepare_coeffs) that pixel_offsets will
1445 * not only contain valid source offsets, but so that (pixel_offsets[x] + filter_size_real - 1) still
1446 * indexes valid source pixels.
1447 * On the right side of the image, this means that the end-of-line coefficients are shifted leftwards
1448 * during the pre-calculation so that the filter kernel will never read beyond the coefficient array
1449 * nor past the source buffer.
1450 * Out of bounds target pixels coefficients are padded with zeros up to program->filter_size_alignment.
1451 */
1452
1453 // resize_h_planar_float_avx2_xxx_vstripe_ks4 method #1: gather-based
1454 template<int filtersizemod4>
1455 2 void resize_h_planar_float_avx2_transpose_vstripe_ks4(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel)
1456 {
1457 assert(filtersizemod4 >= 0 && filtersizemod4 <= 3);
1458
1459 2 const int filter_size = program->filter_size; // aligned, practically the coeff table stride
1460
1461 2 src_pitch /= sizeof(float);
1462 2 dst_pitch /= sizeof(float);
1463
1464 2 float* src = (float*)src8;
1465 2 float* dst = (float*)dst8;
1466
1467 2 constexpr int PIXELS_AT_A_TIME = 8; // Process eight pixels in parallel using AVX2 (2x4 using m128 lanes)
1468
1469 // 'source_overread_beyond_targetx' indicates if the filter kernel can read beyond the target width.
1470 // Even if the filter alignment allows larger reads, our safety boundary for unaligned loads starts at 4 pixels back
1471 // from the target width, as we load 4 floats at once with '_mm_loadu_ps'.
1472 // So contrary to the 8-pixel-at-a-time fact, we only require safety for 4 pixels at a time here.
1473
1/8
void resize_h_planar_float_avx2_transpose_vstripe_ks4<0>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 2 → 3 not taken.
✗ Branch 2 → 4 not taken.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 2 → 3 not taken.
✗ Branch 2 → 4 not taken.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<2>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 2 → 3 taken 2 times.
✗ Branch 2 → 4 not taken.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<3>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 2 → 3 not taken.
✗ Branch 2 → 4 not taken.
 2 const int width_safe_mod = (program->safelimit_4_pixels.overread_possible ? program->safelimit_4_pixels.source_overread_beyond_targetx : width) / PIXELS_AT_A_TIME * PIXELS_AT_A_TIME;
1474
1475 // Preconditions:
1476
1/8
void resize_h_planar_float_avx2_transpose_vstripe_ks4<0>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 5 → 6 not taken.
✗ Branch 5 → 7 not taken.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 5 → 6 not taken.
✗ Branch 5 → 7 not taken.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<2>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 5 → 6 not taken.
✓ Branch 5 → 7 taken 2 times.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<3>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 5 → 6 not taken.
✗ Branch 5 → 7 not taken.
 2 assert(program->filter_size_real <= 4); // We preload all relevant coefficients (up to 4) before the height loop.
1477 // this goes to filtersizemod4 template parameter from 0 to 3
1478
1479 // 'target_size_alignment' ensures we can safely access coefficients using offsets like
1480 // 'filter_size * 7', and pixel_offsets[x to x + 7] when processing 8 H pixels at a time
1481
1/8
void resize_h_planar_float_avx2_transpose_vstripe_ks4<0>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 7 → 8 not taken.
✗ Branch 7 → 9 not taken.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 7 → 8 not taken.
✗ Branch 7 → 9 not taken.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<2>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 7 → 8 not taken.
✓ Branch 7 → 9 taken 2 times.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<3>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 7 → 8 not taken.
✗ Branch 7 → 9 not taken.
 2 assert(program->target_size_alignment >= 8);
1482
1483 // Ensure that coefficient loading beyond the valid target size is safe for 4x4 float loads.
1484
1/8
void resize_h_planar_float_avx2_transpose_vstripe_ks4<0>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 9 → 10 not taken.
✗ Branch 9 → 11 not taken.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 9 → 10 not taken.
✗ Branch 9 → 11 not taken.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<2>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 9 → 10 not taken.
✓ Branch 9 → 11 taken 2 times.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<3>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 9 → 10 not taken.
✗ Branch 9 → 11 not taken.
 2 assert(program->filter_size_alignment >= 4);
1485
1486 // Split to H-stripes to make better source data locality in L2 cache (if L2 present per core ?)
1487 2 constexpr int STRIPE_ALIGN = 8; // this must be multiple of PIXELS_AT_A_TIME
1488
1489 //max_scanlines = program->target_size; // test
1490 2 int max_scanlines = program->max_scanlines / STRIPE_ALIGN * STRIPE_ALIGN;
1491
1/8
void resize_h_planar_float_avx2_transpose_vstripe_ks4<0>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 11 → 12 not taken.
✗ Branch 11 → 13 not taken.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 11 → 12 not taken.
✗ Branch 11 → 13 not taken.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<2>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 11 → 12 not taken.
✓ Branch 11 → 13 taken 2 times.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<3>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 11 → 12 not taken.
✗ Branch 11 → 13 not taken.
 2 if (max_scanlines < STRIPE_ALIGN) max_scanlines = STRIPE_ALIGN;
1492
1493
2/8
void resize_h_planar_float_avx2_transpose_vstripe_ks4<0>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 24 → 14 not taken.
✗ Branch 24 → 25 not taken.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 24 → 14 not taken.
✗ Branch 24 → 25 not taken.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<2>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 24 → 14 taken 2 times.
✓ Branch 24 → 25 taken 2 times.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<3>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 24 → 14 not taken.
✗ Branch 24 → 25 not taken.
 4 for (auto y_from = 0; y_from < height; y_from += max_scanlines) {
1494 2 size_t y_to = std::min(y_from + max_scanlines, height);
1495
1496 // Reset current_coeff for the start of the stripe
1497 2 const float* AVS_RESTRICT current_coeff = program->pixel_coefficient_float; // +iYstart * filter_size;
1498
1499 2 int x = 0;
1500
1501 // This 'auto' lambda construct replaces the need of templates
1502 34 auto do_h_float_core = [&](auto partial_load) {
1503 // Load up to 2x4 coefficients at once before the height loop.
1504 // Pre-loading and transposing coefficients keeps register usage efficient.
1505 // Assumes 'filter_size_aligned' is at least 4.
1506
1507 // Coefficients for the source pixel offset (for src_ptr + begin1 [0..3] and for src_ptr + begin5 [0..3] )
1508 48 __m256 coef_1_coef_5 = _mm256_load_2_m128(current_coeff + filter_size * 0, current_coeff + filter_size * 4);
1509 48 __m256 coef_2_coef_6 = _mm256_load_2_m128(current_coeff + filter_size * 1, current_coeff + filter_size * 5);
1510 48 __m256 coef_3_coef_7 = _mm256_load_2_m128(current_coeff + filter_size * 2, current_coeff + filter_size * 6);
1511 64 __m256 coef_4_coef_8 = _mm256_load_2_m128(current_coeff + filter_size * 3, current_coeff + filter_size * 7);
1512
1513 48 _MM_TRANSPOSE8_LANE4_PS(coef_1_coef_5, coef_2_coef_6, coef_3_coef_7, coef_4_coef_8);
1514
1515 // Pixel offsets for the current target x-positions.
1516 // Even for x >= width, these offsets are guaranteed to be within the allocated 'target_size_alignment'.
1517 16 const int begin1 = program->pixel_offset[x + 0];
1518 16 const int begin2 = program->pixel_offset[x + 1];
1519 16 const int begin3 = program->pixel_offset[x + 2];
1520 16 const int begin4 = program->pixel_offset[x + 3];
1521 16 const int begin5 = program->pixel_offset[x + 4];
1522 16 const int begin6 = program->pixel_offset[x + 5];
1523 16 const int begin7 = program->pixel_offset[x + 6];
1524 16 const int begin8 = program->pixel_offset[x + 7];
1525
1526 16 size_t y = y_from;
1527
1528 16 float* AVS_RESTRICT dst_ptr = dst + y * dst_pitch + x;
1529 16 const float* src_ptr = src + y * src_pitch;
1530
4/4
✓ Branch 85 → 43 taken 70 times.
✓ Branch 85 → 86 taken 14 times.
✓ Branch 97 → 43 taken 10 times.
✓ Branch 97 → 98 taken 2 times.
 96 for (; y < y_to; ++y) {
1531 //float* AVS_RESTRICT dst_ptr = dst + y * dst_pitch + x;
1532 //const float* src_ptr = src + y * src_pitch;
1533
1534 __m256 data_1_data_5;
1535 __m256 data_2_data_6;
1536 __m256 data_3_data_7;
1537 __m256 data_4_data_8;
1538
1539 if constexpr (partial_load) {
1540 // In the potentially unsafe zone (near the right edge of the image), we use a safe loading function
1541 // to prevent reading beyond the allocated source scanline. This handles cases where loading 4 floats
1542 // starting from 'src_ptr + beginX' might exceed the source buffer.
1543
1544 // Example of the unsafe scenario: If target width is 320, a load at src_ptr + 317
1545 // would attempt to read floats at indices 317, 318, 319, and 320, potentially going out of bounds.
1546
1547 // Two main issues in the unsafe zone:
1548 // 1.) Out-of-bounds memory access: Reading beyond the allocated memory for the source scanline can
1549 // lead to access violations and crashes. '_mm_loadu_ps' attempts to load 16 bytes, so even if
1550 // the starting address is within bounds, subsequent reads might not be.
1551 // 2.) Garbage or NaN values: Even if a read doesn't cause a crash, accessing uninitialized or
1552 // out-of-bounds memory (especially for float types) can result in garbage data, including NaN.
1553 // Multiplying by a valid coefficient and accumulating this NaN can contaminate the final result.
1554
1555 // '_mm256_load_partial_safe_2_m128' safely loads up to 'filter_size_real' pixels and pads with zeros if needed,
1556 // preventing out-of-bounds reads and ensuring predictable results even near the image edges.
1557
1558 10 data_1_data_5 = _mm256_load_partial_safe_2_m128<filtersizemod4>(src_ptr + begin1, src_ptr + begin5);
1559 10 data_2_data_6 = _mm256_load_partial_safe_2_m128<filtersizemod4>(src_ptr + begin2, src_ptr + begin6);
1560 10 data_3_data_7 = _mm256_load_partial_safe_2_m128<filtersizemod4>(src_ptr + begin3, src_ptr + begin7);
1561 20 data_4_data_8 = _mm256_load_partial_safe_2_m128<filtersizemod4>(src_ptr + begin4, src_ptr + begin8);
1562 }
1563 else {
1564 // In the safe zone, we can directly load 4 pixels at a time using unaligned loads.
1565 210 data_1_data_5 = _mm256_loadu_2_m128(src_ptr + begin1, src_ptr + begin5);
1566 210 data_2_data_6 = _mm256_loadu_2_m128(src_ptr + begin2, src_ptr + begin6);
1567 210 data_3_data_7 = _mm256_loadu_2_m128(src_ptr + begin3, src_ptr + begin7);
1568 280 data_4_data_8 = _mm256_loadu_2_m128(src_ptr + begin4, src_ptr + begin8);
1569 }
1570
1571 320 _MM_TRANSPOSE8_LANE4_PS(data_1_data_5, data_2_data_6, data_3_data_7, data_4_data_8);
1572
1573 80 __m256 result = _mm256_mul_ps(data_1_data_5, coef_1_coef_5);
1574 80 result = _mm256_fmadd_ps(data_2_data_6, coef_2_coef_6, result);
1575 80 result = _mm256_fmadd_ps(data_3_data_7, coef_3_coef_7, result);
1576 80 result = _mm256_fmadd_ps(data_4_data_8, coef_4_coef_8, result);
1577
1578 _mm256_stream_ps(dst_ptr, result);
1579 80 dst_ptr += dst_pitch;
1580 80 src_ptr += src_pitch;
1581 } // y
1582 16 current_coeff += filter_size * 8; // Move to the next set of coefficients for the next 8 output pixels
1583 }; // end of lambda
1584
1585 // Process the 'safe zone' where direct full unaligned loads are acceptable.
1586
2/8
void resize_h_planar_float_avx2_transpose_vstripe_ks4<0>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 18 → 16 not taken.
✗ Branch 18 → 19 not taken.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 18 → 16 not taken.
✗ Branch 18 → 19 not taken.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<2>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 18 → 16 taken 14 times.
✓ Branch 18 → 19 taken 2 times.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<3>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 18 → 16 not taken.
✗ Branch 18 → 19 not taken.
 16 for (; x < width_safe_mod; x += PIXELS_AT_A_TIME)
1587 {
1588 14 do_h_float_core(std::false_type{}); // partial_load == false, use direct _mm256_loadu_ps
1589 }
1590
1591 // Process the potentially 'unsafe zone' near the image edge, using safe loading.
1592
2/8
void resize_h_planar_float_avx2_transpose_vstripe_ks4<0>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 22 → 20 not taken.
✗ Branch 22 → 23 not taken.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 22 → 20 not taken.
✗ Branch 22 → 23 not taken.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<2>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 22 → 20 taken 2 times.
✓ Branch 22 → 23 taken 2 times.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<3>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 22 → 20 not taken.
✗ Branch 22 → 23 not taken.
 4 for (; x < width; x += PIXELS_AT_A_TIME)
1593 {
1594
1/8
void resize_h_planar_float_avx2_transpose_vstripe_ks4<0>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 20 → 21 not taken.
✗ Branch 20 → 26 not taken.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 20 → 21 not taken.
✗ Branch 20 → 26 not taken.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<2>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 20 → 21 taken 2 times.
✗ Branch 20 → 26 not taken.
void resize_h_planar_float_avx2_transpose_vstripe_ks4<3>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 20 → 21 not taken.
✗ Branch 20 → 26 not taken.
 2 do_h_float_core(std::true_type{}); // partial_load == true, use the safer _mm256_load_partial_safe_2_m128
1595 }
1596 }
1597 2 }
1598
1599 // Helper for permutex style horizontal resampling 32 bit float
1600 // Safe partial load for 1-7 floats, padding with zeros to avoid NaN contamination
1601 ✗ static inline __m256 _mm256_load_partial_safe(const float* src_ptr, int floats_to_load) {
1602 ✗ if (floats_to_load == 1)
1603 ✗ return _mm256_setr_ps(src_ptr[0], 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f);
1604 ✗ if (floats_to_load == 2)
1605 ✗ return _mm256_setr_ps(src_ptr[0], src_ptr[1], 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f);
1606 ✗ if (floats_to_load == 3)
1607 ✗ return _mm256_setr_ps(src_ptr[0], src_ptr[1], src_ptr[2], 0.0f, 0.0f, 0.0f, 0.0f, 0.0f);
1608 ✗ if (floats_to_load == 4)
1609 ✗ return _mm256_setr_ps(src_ptr[0], src_ptr[1], src_ptr[2], src_ptr[3], 0.0f, 0.0f, 0.0f, 0.0f);
1610 ✗ if (floats_to_load == 5)
1611 ✗ return _mm256_setr_ps(src_ptr[0], src_ptr[1], src_ptr[2], src_ptr[3], src_ptr[4], 0.0f, 0.0f, 0.0f);
1612 ✗ if (floats_to_load == 6)
1613 ✗ return _mm256_setr_ps(src_ptr[0], src_ptr[1], src_ptr[2], src_ptr[3], src_ptr[4], src_ptr[5], 0.0f, 0.0f);
1614 ✗ if (floats_to_load == 7)
1615 ✗ return _mm256_setr_ps(src_ptr[0], src_ptr[1], src_ptr[2], src_ptr[3], src_ptr[4], src_ptr[5], src_ptr[6], 0.0f);
1616 ✗ if (floats_to_load == 8)
1617 ✗ return _mm256_loadu_ps(src_ptr); // n/a cannot happen
1618 else
1619 ✗ return _mm256_setzero_ps(); // n/a cannot happen
1620 }
1621
1622
1623 // resize_h_planar_float_avx2_xxx_vstripe_ks4 method #2: permutex-based
1624 1 void resize_h_planar_float_avx2_permutex_vstripe_ks4(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel)
1625 {
1626 1 const int filter_size = program->filter_size; // aligned, practically the coeff table stride
1627
1628 1 src_pitch /= sizeof(float);
1629 1 dst_pitch /= sizeof(float);
1630
1631 1 float* src = (float*)src8;
1632 1 float* dst = (float*)dst8;
1633
1634 1 constexpr int PIXELS_AT_A_TIME = 8; // Process eight pixels in parallel in AVX2
1635
1636 // Pre-checked for permutex-based upsampling: the source pixels will surely fit within single 8 float loads
1637 // The right edge handling will be done via safe partial loads when needed, loading 8 pixels at once
1638 // may not be safe there.
1639
1640 // 'source_overread_beyond_targetx' marks the x position in the target (output) scanline where,
1641 // if we process N pixels at a time (e.g., 8 for AVX2), the filter kernel may overread the source
1642 // buffer near the right edge due to kernel size and pixel offsets. Beyond this value, it is no
1643 // longer safe to read N source pixels at once from pixel_offset[].
1644
1645 // For x positions < source_overread_beyond_targetx, it is safe to load N source pixels at once.
1646 // For x positions >= source_overread_beyond_targetx, we must use a safer loading method (e.g.,
1647 // partial loads with padding) to avoid out-of-bounds memory access.
1648
1649 // permutex is even more special: the safety analysis is performed only for the beginning of each
1650 // block of 8 pixels processed at a time, so only the source loads for the offset position of
1651 // every 8th target pixel are considered. This is 'safelimit_8_pixels_each8th_target'.
1652 // The program's safe limits are pre-calculated during program setup.
1653
1654
1/2
✗ Branch 2 → 3 not taken.
✓ Branch 2 → 4 taken 1 time.
 1 const int width_safe_mod = (program->safelimit_8_pixels_each8th_target.overread_possible ? program->safelimit_8_pixels_each8th_target.source_overread_beyond_targetx : width) / PIXELS_AT_A_TIME * PIXELS_AT_A_TIME;
1655
1656 // Preconditions:
1657
1/2
✗ Branch 5 → 6 not taken.
✓ Branch 5 → 7 taken 1 time.
 1 assert(program->filter_size_real <= 4); // We preload all relevant coefficients (up to 4) before the height loop.
1658
1659 // 'target_size_alignment' ensures we can safely access coefficients using offsets like
1660 // coeff + filter_size*0 to filter_size*7 when processing 8 H pixels at a time
1661
1/2
✗ Branch 7 → 8 not taken.
✓ Branch 7 → 9 taken 1 time.
 1 assert(program->target_size_alignment >= 8);
1662
1663 // Ensure that coefficient loading is safe for 4 float loads,
1664 // if less than 4, padded with zeros till filter_size_alignment.
1665
1/2
✗ Branch 9 → 10 not taken.
✓ Branch 9 → 11 taken 1 time.
 1 assert(program->filter_size_alignment >= 4);
1666
1667 1 const int max_scanlines = program->max_scanlines;
1668
1669
2/2
✓ Branch 22 → 12 taken 1 time.
✓ Branch 22 → 23 taken 1 time.
 2 for (int y_from = 0; y_from < height; y_from += max_scanlines) {
1670 1 int y_to = std::min(y_from + max_scanlines, height);
1671 // Reset current_coeff for the start of the stripe
1672 1 const float* AVS_RESTRICT current_coeff = program->pixel_coefficient_float; // +iYstart * filter_size;
1673
1674 1 int x = 0;
1675
1676 // This 'auto' lambda construct replaces the need of templates
1677 8 auto do_h_float_core = [&](auto partial_load) {
1678 // Assumes 'filter_size_alignment' <= 4, 'target_size_alignment' >= 8
1679 // Prepare 4 coefs per pixel for 8 pixels in transposed V-form at once before the height loop.
1680 24 __m256 coef_0 = _mm256_load_2_m128(current_coeff + filter_size * 0, current_coeff + filter_size * 4);
1681 24 __m256 coef_1 = _mm256_load_2_m128(current_coeff + filter_size * 1, current_coeff + filter_size * 5);
1682 24 __m256 coef_2 = _mm256_load_2_m128(current_coeff + filter_size * 2, current_coeff + filter_size * 6);
1683 32 __m256 coef_3 = _mm256_load_2_m128(current_coeff + filter_size * 3, current_coeff + filter_size * 7);
1684
1685 24 _MM_TRANSPOSE8_LANE4_PS(coef_0, coef_1, coef_2, coef_3);
1686
1687 // convert resampling program in H-form into permuting indexes for src transposition in V-form
1688 8 __m256i perm_0 = _mm256_loadu_si256((__m256i*)(&program->pixel_offset[x]));
1689 8 int iStart = program->pixel_offset[x];
1690 16 perm_0 = _mm256_sub_epi32(perm_0, _mm256_set1_epi32(iStart));
1691 /* like this:
1692 __m256i perm_0 = _mm512_set_epi32(
1693 program->pixel_offset[x + 7] - iStart,
1694 ...
1695 program->pixel_offset[x + 0] - iStart);
1696 */
1697
1698 8 __m256i one_epi32 = _mm256_set1_epi32(1);
1699 8 __m256i perm_1 = _mm256_add_epi32(perm_0, one_epi32); // begin8_rel+1, begin7_rel+1, ... begin2_rel+1, begin1_rel+1
1700 8 __m256i perm_2 = _mm256_add_epi32(perm_1, one_epi32); // begin8_rel+2, begin7_rel+2, ... begin2_rel+2, begin1_rel+2
1701 8 __m256i perm_3 = _mm256_add_epi32(perm_2, one_epi32); // begin8_rel+3, begin7_rel+3, ... begin2_rel+3, begin1_rel+3
1702 // These indexes are guaranteed to be 0..7 due to the earlier analysis,
1703 // and can be used for the indexing parameter in _mm256_permutevar8x32_ps
1704 8 float* AVS_RESTRICT dst_ptr = dst + x + y_from * dst_pitch;
1705 8 const float* src_ptr = src + iStart + y_from * src_pitch;
1706
1707 // for partial_load only
1708 8 const int remaining = program->source_size - iStart;
1709 8 const int floats_to_load = remaining >= 8 ? 8 : remaining;
1710
1711
2/4
auto resize_h_planar_float_avx2_permutex_vstripe_ks4(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int)::{lambda(auto:1)#1}::operator()<std::integral_constant<bool, false> >(std::integral_constant<bool, false>) const:
✓ Branch 73 → 51 taken 40 times.
✓ Branch 73 → 74 taken 8 times.
auto resize_h_planar_float_avx2_permutex_vstripe_ks4(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int)::{lambda(auto:1)#1}::operator()<std::integral_constant<bool, true> >(std::integral_constant<bool, true>) const:
✗ Branch 72 → 51 not taken.
✗ Branch 72 → 73 not taken.
 48 for (int y = y_from; y < y_to; ++y) {
1712
1713 // process scanline y
1714 __m256 data_src;
1715 // We'll need exactly 8 floats starting from src+iStart
1716 if constexpr (partial_load) {
1717 // In the potentially unsafe zone (near the right edge of the image), we use a safe loading function
1718 // to prevent reading beyond the allocated source scanline. This handles cases where loading 8 floats
1719 // starting from 'src_ptr + beginX' might exceed the source buffer.
1720 ✗ data_src = _mm256_load_partial_safe(src_ptr, floats_to_load);
1721 }
1722 else {
1723 40 data_src = _mm256_loadu_ps(src_ptr); // load 8 source pixels, can contain garbage beyond the right edge in the last loop
1724 }
1725
1726 // After we load 8 source pixels starting from begin1, we can be sure, that pixel_offset[x+0] .. pixel_offset[x+7] + 3 is
1727 // within valid source range. Pre-check chooses permutex method only if all needed pixels fit within these 8 loaded pixels.
1728
1729 // perm_0 .. perm_3 contain the indexes to permute data_src into the correct order
1730 // for each of the 8 output pixels so they index into 0..7 (guaranteed) range of the source data loaded above
1731 40 __m256 data_0 = _mm256_permutevar8x32_ps(data_src, perm_0);
1732 40 __m256 data_1 = _mm256_permutevar8x32_ps(data_src, perm_1);
1733 40 __m256 data_2 = _mm256_permutevar8x32_ps(data_src, perm_2);
1734 40 __m256 data_3 = _mm256_permutevar8x32_ps(data_src, perm_3);
1735
1736 40 __m256 result0 = _mm256_mul_ps(data_0, coef_0);
1737 40 __m256 result1 = _mm256_mul_ps(data_2, coef_2);
1738
1739 40 result0 = _mm256_fmadd_ps(data_1, coef_1, result0);
1740 40 result1 = _mm256_fmadd_ps(data_3, coef_3, result1);
1741
1742 // this must be stream until partial tile interface done
1743 40 _mm256_stream_ps(dst_ptr, _mm256_add_ps(result0, result1));
1744
1745 40 dst_ptr += dst_pitch;
1746 40 src_ptr += src_pitch;
1747 }
1748 8 current_coeff += filter_size * 8;
1749 9 }; // end of lambda
1750
1751 // Process the 'safe zone' where direct full unaligned loads are acceptable.
1752
2/2
✓ Branch 16 → 14 taken 8 times.
✓ Branch 16 → 17 taken 1 time.
 9 for (; x < width_safe_mod; x += PIXELS_AT_A_TIME)
1753 {
1754 8 do_h_float_core(std::false_type{}); // partial_load == false, use direct _mm_loadu_ps
1755 }
1756
1757 // Process the potentially 'unsafe zone' near the image edge, using safe loading.
1758
1/2
✗ Branch 20 → 18 not taken.
✓ Branch 20 → 21 taken 1 time.
 1 for (; x < width; x += PIXELS_AT_A_TIME)
1759 {
1760 ✗ do_h_float_core(std::true_type{}); // partial_load == true, use the safer '_mm256_load_partial_safe'
1761 }
1762 }
1763 1 }
1764
1765 #if 0
1766 // resize_h_planar_float_avx2_permutex_vstripe_ks8:
1767 // Like ks4 but handles kernel sizes 5-8 using avx2_permutex2var_ps (two YMMs = 16-float range).
1768 // Dispatch condition: check(8, 16, 8) = false — 8 output pixels' source span <= 16 floats.
1769 // Covers upscale through ~0.875x downscale with LanczosResize(taps=4..8) / Spline36 etc.
1770 void resize_h_planar_float_avx2_permutex_vstripe_ks8(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel)
1771 {
1772 const int filter_size = program->filter_size;
1773
1774 src_pitch /= sizeof(float);
1775 dst_pitch /= sizeof(float);
1776
1777 float* src = (float*)src8;
1778 float* dst = (float*)dst8;
1779
1780 constexpr int PIXELS_AT_A_TIME = 8;
1781
1782 // Safe limit: check only at every 8th target pixel start, loading 16 floats from src+pixel_offset[x].
1783 // Beyond source_overread_beyond_targetx, load 16 floats at once is unsafe — switch to partial loads.
1784 const int width_safe_mod = (program->safelimit_16_pixels_each8th_target.overread_possible
1785 ? program->safelimit_16_pixels_each8th_target.source_overread_beyond_targetx
1786 : width) / PIXELS_AT_A_TIME * PIXELS_AT_A_TIME;
1787
1788 assert(program->filter_size_real <= 8);
1789 assert(program->target_size_alignment >= 8);
1790 assert(program->filter_size_alignment >= 8); // need 8 floats/pixel so +4 offset into tap-row is valid
1791
1792 const int max_scanlines = program->max_scanlines;
1793
1794 for (int y_from = 0; y_from < height; y_from += max_scanlines) {
1795 int y_to = std::min(y_from + max_scanlines, height);
1796 const float* AVS_RESTRICT current_coeff = program->pixel_coefficient_float;
1797
1798 int x = 0;
1799
1800 auto do_h_float_core = [&](auto partial_load) {
1801 // Load 8 taps per pixel for 8 output pixels.
1802 // Two groups of 4 taps each, transposed so that each coef_k vector holds
1803 // tap k for all 8 output pixels simultaneously.
1804 //
1805 // Taps 0-3: load from current_coeff + filter_size*j + 0..3 for j=0,1,...,7
1806 __m256 coef_0 = _mm256_load_2_m128(current_coeff + filter_size * 0, current_coeff + filter_size * 4);
1807 __m256 coef_1 = _mm256_load_2_m128(current_coeff + filter_size * 1, current_coeff + filter_size * 5);
1808 __m256 coef_2 = _mm256_load_2_m128(current_coeff + filter_size * 2, current_coeff + filter_size * 6);
1809 __m256 coef_3 = _mm256_load_2_m128(current_coeff + filter_size * 3, current_coeff + filter_size * 7);
1810 _MM_TRANSPOSE8_LANE4_PS(coef_0, coef_1, coef_2, coef_3);
1811 // Taps 4-7: load from current_coeff + filter_size*j + 4 for j=0,1,...,7
1812 // filter_size is multiple of 8, so +4 floats (+16 bytes) keeps 16-byte alignment for _mm_load_ps.
1813 __m256 coef_4 = _mm256_load_2_m128(current_coeff + filter_size * 0 + 4, current_coeff + filter_size * 4 + 4);
1814 __m256 coef_5 = _mm256_load_2_m128(current_coeff + filter_size * 1 + 4, current_coeff + filter_size * 5 + 4);
1815 __m256 coef_6 = _mm256_load_2_m128(current_coeff + filter_size * 2 + 4, current_coeff + filter_size * 6 + 4);
1816 __m256 coef_7 = _mm256_load_2_m128(current_coeff + filter_size * 3 + 4, current_coeff + filter_size * 7 + 4);
1817 _MM_TRANSPOSE8_LANE4_PS(coef_4, coef_5, coef_6, coef_7);
1818 // After transpose: coef_k[lane i] = tap k coefficient for output pixel i.
1819
1820 // Permute indices: perm_k[i] = pixel_offset[x+i] - iStart + k.
1821 // Guaranteed 0..15 by check(8,16,8)=false, suitable for avx2_permutex2var_ps (16-float range).
1822 __m256i perm_0 = _mm256_loadu_si256((__m256i*)(&program->pixel_offset[x]));
1823 const int iStart = program->pixel_offset[x];
1824 perm_0 = _mm256_sub_epi32(perm_0, _mm256_set1_epi32(iStart));
1825 const __m256i one_epi32 = _mm256_set1_epi32(1);
1826 __m256i perm_1 = _mm256_add_epi32(perm_0, one_epi32);
1827 __m256i perm_2 = _mm256_add_epi32(perm_1, one_epi32);
1828 __m256i perm_3 = _mm256_add_epi32(perm_2, one_epi32);
1829 __m256i perm_4 = _mm256_add_epi32(perm_3, one_epi32);
1830 __m256i perm_5 = _mm256_add_epi32(perm_4, one_epi32);
1831 __m256i perm_6 = _mm256_add_epi32(perm_5, one_epi32);
1832 __m256i perm_7 = _mm256_add_epi32(perm_6, one_epi32);
1833
1834 float* AVS_RESTRICT dst_ptr = dst + x + y_from * dst_pitch;
1835 const float* src_ptr = src + iStart + y_from * src_pitch;
1836
1837 // For partial_load: how many valid floats from iStart (capped at 8 per YMM).
1838 const int remaining = program->source_size - iStart;
1839 const int floats_to_load_a = std::min(remaining, 8);
1840 const int floats_to_load_b = std::max(remaining - 8, 0); // 0 → setzero in _mm256_load_partial_safe
1841
1842 for (int y = y_from; y < y_to; ++y) {
1843 __m256 data_src, data_src2;
1844 if constexpr (partial_load) {
1845 data_src = _mm256_load_partial_safe(src_ptr, floats_to_load_a);
1846 data_src2 = _mm256_load_partial_safe(src_ptr + 8, floats_to_load_b);
1847 }
1848 else {
1849 data_src = _mm256_loadu_ps(src_ptr);
1850 data_src2 = _mm256_loadu_ps(src_ptr + 8);
1851 }
1852
1853 // Gather each tap for all 8 output pixels from the 16-float window.
1854 __m256 data_0 = avx2_permutex2var_ps(data_src, data_src2, perm_0);
1855 __m256 data_1 = avx2_permutex2var_ps(data_src, data_src2, perm_1);
1856 __m256 data_2 = avx2_permutex2var_ps(data_src, data_src2, perm_2);
1857 __m256 data_3 = avx2_permutex2var_ps(data_src, data_src2, perm_3);
1858 __m256 data_4 = avx2_permutex2var_ps(data_src, data_src2, perm_4);
1859 __m256 data_5 = avx2_permutex2var_ps(data_src, data_src2, perm_5);
1860 __m256 data_6 = avx2_permutex2var_ps(data_src, data_src2, perm_6);
1861 __m256 data_7 = avx2_permutex2var_ps(data_src, data_src2, perm_7);
1862
1863 // Two parallel FMA chains to reduce dependency depth.
1864 __m256 result0 = _mm256_mul_ps(data_0, coef_0);
1865 __m256 result1 = _mm256_mul_ps(data_2, coef_2);
1866 result0 = _mm256_fmadd_ps(data_1, coef_1, result0);
1867 result1 = _mm256_fmadd_ps(data_3, coef_3, result1);
1868 result0 = _mm256_fmadd_ps(data_4, coef_4, result0);
1869 result1 = _mm256_fmadd_ps(data_5, coef_5, result1);
1870 result0 = _mm256_fmadd_ps(data_6, coef_6, result0);
1871 result1 = _mm256_fmadd_ps(data_7, coef_7, result1);
1872
1873 _mm256_stream_ps(dst_ptr, _mm256_add_ps(result0, result1));
1874
1875 dst_ptr += dst_pitch;
1876 src_ptr += src_pitch;
1877 }
1878 current_coeff += filter_size * 8;
1879 }; // end of lambda
1880
1881 for (; x < width_safe_mod; x += PIXELS_AT_A_TIME)
1882 do_h_float_core(std::false_type{});
1883 for (; x < width; x += PIXELS_AT_A_TIME)
1884 do_h_float_core(std::true_type{});
1885 }
1886 }
1887 #endif
1888
1889 // Simulating the AVX512 case, where 16-way permutes are possible.
1890 // H, kernel size 4, 2x8 pix version, 16 output pixels.
1891 // Same as plain ks4, but 2x8 pixels instead of 1x8 pixels at a time.
1892 // Since AVX2 only supports 256 bit and 8xfloat permute, we have to simulate 16 pixel permute, with
1893 // handling cross-lane indices 0..15, but since the intrinsic support is only for 0..7 we have to use
1894 // two separate permutes, masks and then blend the results together.
1895
1896 // Structure to hold all precalculated vectors for ONE set of coefficients/taps (e.g., perm_0)
1897 // Since we have 4 taps (perm_0 to perm_3), you would need 4 instances of this structure.
1898 typedef struct {
1899 // Permutation Indices (4 __m256i vectors)
1900 __m256i PL_A; // Low output half, source A indices (0-7)
1901 __m256i PL_B; // Low output half, source B indices (0-7)
1902 __m256i PH_A; // High output half, source A indices (0-7)
1903 __m256i PH_B; // High output half, source B indices (0-7)
1904
1905 // Mask Vectors (2 __m256 vectors for blendv_ps)
1906 __m256 ML_B; // Low output half mask (1s select B, 0s select A)
1907 __m256 MH_B; // High output half mask (1s select B, 0s select A)
1908 } PermuteVectors_AVX2;
1909
1910 ✗ static void precalculate_cross_perm_avx2(
1911 const int* pixel_offset,
1912 int x,
1913 PermuteVectors_AVX2* tap_vectors[4])
1914 {
1915 // The base offset for the first loaded register A is pixel_offset[x + 0]
1916 ✗ const int begin1 = pixel_offset[x + 0];
1917
1918 // Broadcast the constant 8 and the base offset 'begin1'
1919 ✗ const __m256i v_8 = _mm256_set1_epi32(8);
1920 ✗ const __m256i v_begin1 = _mm256_set1_epi32(begin1);
1921
1922 // A constant 0 for index in the ignored lane
1923 ✗ const __m256i v_zero = _mm256_setzero_si256();
1924
1925 // Loop through all 4 taps independently
1926 ✗ for (int tap = 0; tap < 4; ++tap) {
1927
1928 // 1. Prepare the 16 absolute indices (I_k) by adding 'tap'
1929 // This still requires a temporary array or two separate loads/SIMD additions
1930
1931 // Use temporary arrays of 8 elements for simplicity, but could be done directly in SIMD
1932 int I_k_low[8];
1933 int I_k_high[8];
1934 ✗ for (int k = 0; k < 8; ++k) {
1935 ✗ I_k_low[k] = pixel_offset[x + k] + tap;
1936 ✗ I_k_high[k] = pixel_offset[x + k + 8] + tap;
1937 }
1938
1939 // Load the 16 absolute indices, split into two __m256i vectors
1940 ✗ __m256i v_I_low = _mm256_loadu_si256((const __m256i*)I_k_low);
1941 ✗ __m256i v_I_high = _mm256_loadu_si256((const __m256i*)I_k_high);
1942
1943 // --- Calculate Relative Indices (J_k = I_k - begin1) ---
1944
1945 ✗ __m256i v_J_low = _mm256_sub_epi32(v_I_low, v_begin1);
1946 ✗ __m256i v_J_high = _mm256_sub_epi32(v_I_high, v_begin1);
1947
1948 // --- Calculate Mask B (ML_B and MH_B) ---
1949 // Mask: 0xFFFFFFFF if J_k >= 8, 0x00000000 if J_k < 8
1950 // _mm256_cmpgt_epi32(a, b) computes a > b. We want J_k >= 8, so we use J_k > 7.
1951 ✗ const __m256i v_7 = _mm256_set1_epi32(7);
1952
1953 ✗ __m256i v_Mask_low = _mm256_cmpgt_epi32(v_J_low, v_7);
1954 ✗ __m256i v_Mask_high = _mm256_cmpgt_epi32(v_J_high, v_7);
1955
1956 // Store the float mask vectors (Mask B)
1957 ✗ tap_vectors[tap]->ML_B = _mm256_castsi256_ps(v_Mask_low);
1958 ✗ tap_vectors[tap]->MH_B = _mm256_castsi256_ps(v_Mask_high);
1959
1960 // --- Calculate Permutation Indices for Source B (PH_B and PL_B) ---
1961 // Index B is J_k - 8 (only for elements where J_k >= 8)
1962 ✗ __m256i v_Jm8_low = _mm256_sub_epi32(v_J_low, v_8);
1963 ✗ __m256i v_Jm8_high = _mm256_sub_epi32(v_J_high, v_8);
1964
1965 // Select: (Mask B) ? (J_k - 8) : 0
1966 // _mm256_blendv_epi8 can be used as a general purpose blend for 32-bit integers
1967 // Note: The index '0' for the ignored lane doesn't matter, as the corresponding
1968 // output element will be selected from Source A, not B.
1969 ✗ tap_vectors[tap]->PL_B = _mm256_blendv_epi8(v_zero, v_Jm8_low, v_Mask_low);
1970 ✗ tap_vectors[tap]->PH_B = _mm256_blendv_epi8(v_zero, v_Jm8_high, v_Mask_high);
1971
1972 // --- Calculate Permutation Indices for Source A (PH_A and PL_A) ---
1973 // Index A is J_k (only for elements where J_k < 8)
1974 // Select: (Mask B) ? 0 : J_k
1975 // The inverse mask can be created by NOTting the mask (using XOR with all ones, or NOT equivalent)
1976 // Since we want NOT Mask B to select J_k, we use the original mask to select 0.
1977
1978 // A simpler way: J_k already contains the correct index (0-7). We just need to zero it out
1979 // where it's NOT needed (i.e., where Mask B is set).
1980
1981 // Inverse Mask: 0xFFFFFFFF if J_k < 8, 0x00000000 if J_k >= 8
1982 ✗ __m256i v_InvMask_low = _mm256_xor_si256(v_Mask_low, _mm256_set1_epi32(0xFFFFFFFF));
1983 ✗ __m256i v_InvMask_high = _mm256_xor_si256(v_Mask_high, _mm256_set1_epi32(0xFFFFFFFF));
1984
1985 // Select: (Inv Mask) ? J_k : 0
1986 ✗ tap_vectors[tap]->PL_A = _mm256_blendv_epi8(v_zero, v_J_low, v_InvMask_low);
1987 ✗ tap_vectors[tap]->PH_A = _mm256_blendv_epi8(v_zero, v_J_high, v_InvMask_high);
1988 }
1989 ✗ }
1990
1991 // Helper for permutex style horizontal resampling 32 bit float
1992 // Safe partial load for 1-15 floats, padding with zeros to avoid NaN contamination
1993 // Using jump tables instead of multiple if-else, each case is extremely
1994 // optimized looking at the generated assembly..
1995 ✗ static void _mm256_load_512_partial_safe(__m256 &A, __m256 &B, const float* src_ptr, int floats_to_load) {
1996 ✗ if (floats_to_load == 1) {
1997 ✗ A = _mm256_setr_ps(src_ptr[0], 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f);
1998 ✗ B = _mm256_setzero_ps();
1999 }
2000 ✗ else if (floats_to_load == 2) {
2001 ✗ A = _mm256_setr_ps(src_ptr[0], src_ptr[1], 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f);
2002 ✗ B = _mm256_setzero_ps();
2003 }
2004 ✗ else if (floats_to_load == 3) {
2005 ✗ A = _mm256_setr_ps(src_ptr[0], src_ptr[1], src_ptr[2], 0.0f, 0.0f, 0.0f, 0.0f, 0.0f);
2006 ✗ B = _mm256_setzero_ps();
2007 }
2008 ✗ else if (floats_to_load == 4) {
2009 ✗ A = _mm256_setr_ps(src_ptr[0], src_ptr[1], src_ptr[2], src_ptr[3], 0.0f, 0.0f, 0.0f, 0.0f);
2010 ✗ B = _mm256_setzero_ps();
2011 }
2012 ✗ else if (floats_to_load == 5) {
2013 ✗ A = _mm256_setr_ps(src_ptr[0], src_ptr[1], src_ptr[2], src_ptr[3], src_ptr[4], 0.0f, 0.0f, 0.0f);
2014 ✗ B = _mm256_setzero_ps();
2015 }
2016 ✗ else if (floats_to_load == 6) {
2017 ✗ A = _mm256_setr_ps(src_ptr[0], src_ptr[1], src_ptr[2], src_ptr[3], src_ptr[4], src_ptr[5], 0.0f, 0.0f);
2018 ✗ B = _mm256_setzero_ps();
2019 }
2020 ✗ else if (floats_to_load == 7) {
2021 ✗ A = _mm256_setr_ps(src_ptr[0], src_ptr[1], src_ptr[2], src_ptr[3], src_ptr[4], src_ptr[5], src_ptr[6], 0.0f);
2022 ✗ B = _mm256_setzero_ps();
2023 }
2024 ✗ else if (floats_to_load == 8) {
2025 ✗ A = _mm256_loadu_ps(src_ptr);
2026 ✗ B = _mm256_setzero_ps();
2027 }
2028 ✗ else if (floats_to_load == 9) {
2029 ✗ A = _mm256_loadu_ps(src_ptr);
2030 ✗ B = _mm256_setr_ps(src_ptr[8], 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f);
2031 }
2032 ✗ else if (floats_to_load == 10) {
2033 ✗ A = _mm256_loadu_ps(src_ptr);
2034 ✗ B = _mm256_setr_ps(src_ptr[8], src_ptr[9], 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f);
2035 }
2036 ✗ else if (floats_to_load == 11) {
2037 ✗ A = _mm256_loadu_ps(src_ptr);
2038 ✗ B = _mm256_setr_ps(src_ptr[8], src_ptr[9], src_ptr[10], 0.0f, 0.0f, 0.0f, 0.0f, 0.0f);
2039 }
2040 ✗ else if (floats_to_load == 12) {
2041 ✗ A = _mm256_loadu_ps(src_ptr);
2042 ✗ B = _mm256_setr_ps(src_ptr[8], src_ptr[9], src_ptr[10], src_ptr[11], 0.0f, 0.0f, 0.0f, 0.0f);
2043 }
2044 ✗ else if (floats_to_load == 13) {
2045 ✗ A = _mm256_loadu_ps(src_ptr);
2046 ✗ B = _mm256_setr_ps(src_ptr[8], src_ptr[9], src_ptr[10], src_ptr[11], src_ptr[12], 0.0f, 0.0f, 0.0f);
2047 }
2048 ✗ else if (floats_to_load == 14) {
2049 ✗ A = _mm256_loadu_ps(src_ptr);
2050 ✗ B = _mm256_setr_ps(src_ptr[8], src_ptr[9], src_ptr[10], src_ptr[11], src_ptr[12], src_ptr[13], 0.0f, 0.0f);
2051 }
2052 ✗ else if (floats_to_load == 15) {
2053 ✗ A = _mm256_loadu_ps(src_ptr);
2054 ✗ B = _mm256_setr_ps(src_ptr[8], src_ptr[9], src_ptr[10], src_ptr[11], src_ptr[12], src_ptr[13], src_ptr[14], 0.0f);
2055 }
2056 ✗ else if (floats_to_load == 16) { // cannot happen
2057 ✗ A = _mm256_loadu_ps(src_ptr);
2058 ✗ B = _mm256_loadu_ps(src_ptr + 8);
2059 }
2060 else {
2061 ✗ A = _mm256_setzero_ps(); // n/a cannot happen
2062 ✗ B = _mm256_setzero_ps(); // n/a cannot happen
2063 }
2064 ✗ }
2065
2066
2067 // resize_h_planar_float_avx2_xxx_vstripe_ks4 method #2: permutex-based, 16 pixel test version
2068 // like resize_h_planar_float_avx512_permutex_vstripe_ks4
2069 ✗ void resize_h_planar_float_avx2_permutex_vstripe_ks4_pix16(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel)
2070 {
2071 ✗ const int filter_size = program->filter_size; // aligned, practically the coeff table stride
2072
2073 ✗ src_pitch /= sizeof(float);
2074 ✗ dst_pitch /= sizeof(float);
2075
2076 ✗ float* src = (float*)src8;
2077 ✗ float* dst = (float*)dst8;
2078
2079 ✗ constexpr int PIXELS_AT_A_TIME = 2 * 8; // Process eight pixels in parallel in AVX2
2080
2081 // Pre-checked for permutex-based upsampling: the source pixels will surely fit within single 2x8 float loads
2082 // The right edge handling will be done via safe partial loads when needed, loading 2x8 pixels at once
2083 // may not be safe there.
2084
2085 // 'source_overread_beyond_targetx' marks the x position in the target (output) scanline where,
2086 // if we process N pixels at a time (e.g., 8 for AVX2), the filter kernel may overread the source
2087 // buffer near the right edge due to kernel size and pixel offsets. Beyond this value, it is no
2088 // longer safe to read N source pixels at once from pixel_offset[].
2089
2090 // For x positions < source_overread_beyond_targetx, it is safe to load N source pixels at once.
2091 // For x positions >= source_overread_beyond_targetx, we must use a safer loading method (e.g.,
2092 // partial loads with padding) to avoid out-of-bounds memory access.
2093
2094 // permutex is even more special: the safety analysis is performed only for the beginning of each
2095 // block of 16 pixels processed at a time, so only the source loads for the offset position of
2096 // every 16th target pixel are considered. This is 'safelimit_16_pixels_each16th_target'.
2097 // The program's safe limits are pre-calculated during program setup.
2098
2099 ✗ const int width_safe_mod = (program->safelimit_16_pixels_each16th_target.overread_possible ? program->safelimit_16_pixels_each16th_target.source_overread_beyond_targetx : width) / PIXELS_AT_A_TIME * PIXELS_AT_A_TIME;
2100
2101 // Preconditions:
2102 ✗ assert(program->filter_size_real <= 4); // We preload all relevant coefficients (up to 4) before the height loop.
2103
2104 // 'target_size_alignment' ensures we can safely access coefficients using offsets like
2105 // coeff + filter_size*0 to filter_size*7 when processing 16 H pixels at a time
2106 ✗ assert(program->target_size_alignment >= 16);
2107
2108 // Ensure that coefficient loading is safe for 4 float loads,
2109 // if less than 4, padded with zeros till filter_size_alignment.
2110 ✗ assert(program->filter_size_alignment >= 4);
2111
2112 ✗ const int max_scanlines = program->max_scanlines;
2113
2114 // Example setup before the main loop (assuming memory allocation for 4 instances)
2115 PermuteVectors_AVX2 taps[4];
2116 ✗ PermuteVectors_AVX2* tap_pointers[4] = { &taps[0], &taps[1], &taps[2], &taps[3] };
2117
2118 ✗ for (int y_from = 0; y_from < height; y_from += max_scanlines) {
2119 ✗ int y_to = std::min((int)(y_from + max_scanlines), height);
2120 // Reset current_coeff for the start of the stripe
2121 ✗ const float* AVS_RESTRICT current_coeff = program->pixel_coefficient_float; // +iYstart * filter_size;
2122
2123 ✗ int x = 0;
2124
2125 // This 'auto' lambda construct replaces the need of templates
2126 ✗ auto do_h_float_core = [&](auto partial_load) {
2127
2128 // Call precalculation once per x position
2129 ✗ precalculate_cross_perm_avx2(program->pixel_offset.data(), x, tap_pointers);
2130
2131 // Assumes 'filter_size_alignment' <= 4, 'target_size_alignment' >= 16
2132 // Prepare 4 coefs per pixel for 16 pixels in transposed V-form at once before the height loop.
2133
2134 // ---------------------------------------------------------------------------
2135 // 1. Load Coefficients
2136 // ---------------------------------------------------------------------------
2137 // We process 16 pixels total.
2138 // Low Group (Pixels 0-7): Loaded into coef_0..coef_3
2139 // High Group (Pixels 8-15): Loaded into coef_4..coef_7
2140
2141 // filter_size is typically the stride in floats (e.g., 4)
2142
2143 // -- Load Low Group (Pixels 0,1,2,3 and 4,5,6,7 interleaved for Transpose macro) --
2144 ✗ __m256 coef_0 = _mm256_load_2_m128(current_coeff + filter_size * 0, current_coeff + filter_size * 4);
2145 ✗ __m256 coef_1 = _mm256_load_2_m128(current_coeff + filter_size * 1, current_coeff + filter_size * 5);
2146 ✗ __m256 coef_2 = _mm256_load_2_m128(current_coeff + filter_size * 2, current_coeff + filter_size * 6);
2147 ✗ __m256 coef_3 = _mm256_load_2_m128(current_coeff + filter_size * 3, current_coeff + filter_size * 7);
2148
2149 // -- Load High Group (Pixels 8,9,10,11 and 12,13,14,15 interleaved) --
2150 // Note: Offsets are shifted by 8 relative to the Low Group
2151 ✗ __m256 coef_4 = _mm256_load_2_m128(current_coeff + filter_size * 8, current_coeff + filter_size * 12);
2152 ✗ __m256 coef_5 = _mm256_load_2_m128(current_coeff + filter_size * 9, current_coeff + filter_size * 13);
2153 ✗ __m256 coef_6 = _mm256_load_2_m128(current_coeff + filter_size * 10, current_coeff + filter_size * 14);
2154 ✗ __m256 coef_7 = _mm256_load_2_m128(current_coeff + filter_size * 11, current_coeff + filter_size * 15);
2155
2156 // ---------------------------------------------------------------------------
2157 // 2. Transpose
2158 // ---------------------------------------------------------------------------
2159 // After transpose:
2160 // coef_0 -> Tap 0 for Pixels 0-7
2161 // coef_1 -> Tap 1 for Pixels 0-7 ... etc
2162 ✗ _MM_TRANSPOSE8_LANE4_PS(coef_0, coef_1, coef_2, coef_3);
2163
2164 // coef_4 -> Tap 0 for Pixels 8-15
2165 // coef_5 -> Tap 1 for Pixels 8-15 ... etc
2166 ✗ _MM_TRANSPOSE8_LANE4_PS(coef_4, coef_5, coef_6, coef_7);
2167
2168 ✗ const int begin1 = program->pixel_offset[x + 0];
2169 // These indexes are guaranteed to be 0..15 due to the earlier analysis,
2170 // and can be used for the indexing parameter in combiner blendm mask, _mm256_permutevar8x32_ps
2171 ✗ float* AVS_RESTRICT dst_ptr = dst + x + y_from * dst_pitch;
2172 ✗ const float* src_ptr = src + begin1 + y_from * src_pitch;
2173
2174 // for partial_load only
2175 ✗ const int remaining = program->source_size - begin1;
2176 ✗ const int floats_to_load = remaining >= 16 ? 16 : remaining;
2177
2178 ✗ for (int y = y_from; y < y_to; ++y) {
2179
2180 // process scanline y
2181 __m256 A;
2182 __m256 B;
2183
2184 //__m256 data_src;
2185 // We'll need exactly 2x8 floats starting from src+begin1
2186 if constexpr (partial_load) {
2187 // In the potentially unsafe zone (near the right edge of the image), we use a safe loading function
2188 // to prevent reading beyond the allocated source scanline. This handles cases where loading 8 floats
2189 // starting from 'src_ptr + beginX' might exceed the source buffer.
2190 ✗ _mm256_load_512_partial_safe(/*ref*/A, /*ref*/B, src_ptr, floats_to_load);
2191 }
2192 else {
2193 ✗ A = _mm256_loadu_ps(src_ptr);
2194 ✗ B = _mm256_loadu_ps(src_ptr + 8);
2195 // data_src = _mm256_loadu_ps(src_ptr); // load 8 source pixels, can contain garbage beyond the right edge in the last loop
2196 }
2197
2198 // After we load 2x8 source pixels starting from begin1, we can be sure, that pixel_offset[x+0] .. pixel_offset[x+15] + 3 is
2199 // within valid source range. Pre-check chooses permutex method only if all needed pixels fit within these 16 loaded pixels.
2200
2201 // Permute and Blend for Tap 0 (data_0)
2202 ✗ __m256 A_perm_0L = _mm256_permutevar8x32_ps(A, taps[0].PL_A);
2203 ✗ __m256 B_perm_0L = _mm256_permutevar8x32_ps(B, taps[0].PL_B);
2204 ✗ __m256 data_0L = _mm256_blendv_ps(A_perm_0L, B_perm_0L, taps[0].ML_B); // Result: dst[x+0]..dst[x+7]
2205
2206 ✗ __m256 A_perm_0H = _mm256_permutevar8x32_ps(A, taps[0].PH_A);
2207 ✗ __m256 B_perm_0H = _mm256_permutevar8x32_ps(B, taps[0].PH_B);
2208 ✗ __m256 data_0H = _mm256_blendv_ps(A_perm_0H, B_perm_0H, taps[0].MH_B); // Result: dst[x+8]..dst[x+15]
2209
2210 // Repeat for data_1, data_2, data_3...
2211 // Permute and Blend for Tap 1 (data_1)
2212 ✗ __m256 A_perm_1L = _mm256_permutevar8x32_ps(A, taps[1].PL_A);
2213 ✗ __m256 B_perm_1L = _mm256_permutevar8x32_ps(B, taps[1].PL_B);
2214 ✗ __m256 data_1L = _mm256_blendv_ps(A_perm_1L, B_perm_1L, taps[1].ML_B); // Result: dst[x+0]..dst[x+7]
2215
2216 ✗ __m256 A_perm_1H = _mm256_permutevar8x32_ps(A, taps[1].PH_A);
2217 ✗ __m256 B_perm_1H = _mm256_permutevar8x32_ps(B, taps[1].PH_B);
2218 ✗ __m256 data_1H = _mm256_blendv_ps(A_perm_1H, B_perm_1H, taps[1].MH_B); // Result: dst[x+8]..dst[x+15]
2219 // Permute and Blend for Tap 2 (data_2)
2220 ✗ __m256 A_perm_2L = _mm256_permutevar8x32_ps(A, taps[2].PL_A);
2221 ✗ __m256 B_perm_2L = _mm256_permutevar8x32_ps(B, taps[2].PL_B);
2222 ✗ __m256 data_2L = _mm256_blendv_ps(A_perm_2L, B_perm_2L, taps[2].ML_B); // Result: dst[x+0]..dst[x+7]
2223
2224 ✗ __m256 A_perm_2H = _mm256_permutevar8x32_ps(A, taps[2].PH_A);
2225 ✗ __m256 B_perm_2H = _mm256_permutevar8x32_ps(B, taps[2].PH_B);
2226 ✗ __m256 data_2H = _mm256_blendv_ps(A_perm_2H, B_perm_2H, taps[2].MH_B); // Result: dst[x+8]..dst[x+15]
2227 // Permute and Blend for Tap 3 (data_3)
2228 ✗ __m256 A_perm_3L = _mm256_permutevar8x32_ps(A, taps[3].PL_A);
2229 ✗ __m256 B_perm_3L = _mm256_permutevar8x32_ps(B, taps[3].PL_B);
2230 ✗ __m256 data_3L = _mm256_blendv_ps(A_perm_3L, B_perm_3L, taps[3].ML_B); // Result: dst[x+0]..dst[x+7]
2231
2232 ✗ __m256 A_perm_3H = _mm256_permutevar8x32_ps(A, taps[3].PH_A);
2233 ✗ __m256 B_perm_3H = _mm256_permutevar8x32_ps(B, taps[3].PH_B);
2234 ✗ __m256 data_3H = _mm256_blendv_ps(A_perm_3H, B_perm_3H, taps[3].MH_B); // Result: dst[x+8]..dst[x+15]
2235
2236 // perm_0 .. perm_3 lo and hi contain the indexes to permute data_src into the correct order
2237 // for each of the 16 output pixels so they index into 0..15 (guaranteed) range of the source data loaded above
2238 // Since we have only 8-wide permute, we have to do two separate permutes and then blend the results together.
2239
2240 // Assuming these eight data vectors have been calculated via Permute+Blend:
2241 // __m256 data_0L, data_0H;
2242 // __m256 data_1L, data_1H;
2243 // __m256 data_2L, data_2H;
2244 // __m256 data_3L, data_3H;
2245
2246 // ---------------------------------------------------------------------------
2247 // 3. Calculation
2248 // ---------------------------------------------------------------------------
2249
2250 // --- Low Half (Pixels 0-7) ---
2251 // Uses: data_0L..3L and coef_0..3
2252
2253 ✗ __m256 result0L = _mm256_mul_ps(data_0L, coef_0); // Tap 0
2254 ✗ __m256 result1L = _mm256_mul_ps(data_2L, coef_2); // Tap 2
2255
2256 ✗ result0L = _mm256_fmadd_ps(data_1L, coef_1, result0L); // Tap 1
2257 ✗ result1L = _mm256_fmadd_ps(data_3L, coef_3, result1L); // Tap 3
2258
2259 ✗ __m256 final_result_L = _mm256_add_ps(result0L, result1L);
2260
2261
2262 // --- High Half (Pixels 8-15) ---
2263 // Uses: data_0H..3H and coef_4..7
2264 // Note: coef_4 corresponds to Tap 0 for this group, coef_5 to Tap 1, etc.
2265
2266 ✗ __m256 result0H = _mm256_mul_ps(data_0H, coef_4); // Tap 0 (High)
2267 ✗ __m256 result1H = _mm256_mul_ps(data_2H, coef_6); // Tap 2 (High)
2268
2269 ✗ result0H = _mm256_fmadd_ps(data_1H, coef_5, result0H); // Tap 1 (High)
2270 ✗ result1H = _mm256_fmadd_ps(data_3H, coef_7, result1H); // Tap 3 (High)
2271
2272 ✗ __m256 final_result_H = _mm256_add_ps(result0H, result1H);
2273
2274 // ---------------------------------------------------------------------------
2275 // 4. Store
2276 // ---------------------------------------------------------------------------
2277
2278 _mm256_stream_ps(dst_ptr, final_result_L); // dst[0..7]
2279 ✗ _mm256_stream_ps(dst_ptr + 8, final_result_H); // dst[8..15]
2280
2281 ✗ dst_ptr += dst_pitch;
2282 ✗ src_ptr += src_pitch;
2283 }
2284 ✗ current_coeff += filter_size * 16;
2285 ✗ }; // end of lambda
2286
2287 // Process the 'safe zone' where direct full unaligned loads are acceptable.
2288 ✗ for (; x < width_safe_mod; x += PIXELS_AT_A_TIME)
2289 {
2290 ✗ do_h_float_core(std::false_type{}); // partial_load == false, use direct _mm_loadu_ps
2291 }
2292
2293 // Process the potentially 'unsafe zone' near the image edge, using safe loading.
2294 ✗ for (; x < width; x += PIXELS_AT_A_TIME)
2295 {
2296 ✗ do_h_float_core(std::true_type{}); // partial_load == true, use the safer '_mm256_load_partial_safe'
2297 }
2298 }
2299
2300 ✗ }
2301
2302
2303
2304 template<int filtersizemod4>
2305 1 void resize_h_planar_float_avx2_gather_permutex_vstripe_ks4(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel)
2306 {
2307
2308 // FIXME: this analysis could be done once in the dispatcher instead of per call, since the program is constant
2309 1 bool bDoGather = program->resize_h_planar_gather_permutex_vstripe_check(8/*iSamplesInThGroup*/, 8 /*permutex_index_diff_limit*/, 4 /*kernel_size*/);
2310
1/8
void resize_h_planar_float_avx2_gather_permutex_vstripe_ks4<0>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 3 → 4 not taken.
✗ Branch 3 → 5 not taken.
void resize_h_planar_float_avx2_gather_permutex_vstripe_ks4<1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 3 → 4 not taken.
✗ Branch 3 → 5 not taken.
void resize_h_planar_float_avx2_gather_permutex_vstripe_ks4<2>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 3 → 4 taken 1 time.
✗ Branch 3 → 5 not taken.
void resize_h_planar_float_avx2_gather_permutex_vstripe_ks4<3>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 3 → 4 not taken.
✗ Branch 3 → 5 not taken.
 1 if (bDoGather)
2311 {
2312 1 resize_h_planar_float_avx2_transpose_vstripe_ks4<filtersizemod4>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
2313 } // if bDoGather
2314 else
2315 {
2316 ✗ resize_h_planar_float_avx2_permutex_vstripe_ks4(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
2317 }
2318
2319 // versus the original one, which is a generic upsample/downsample method
2320 //resize_h_planar_float_avx_transpose_vstripe_ks4<filtersizemod4>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
2321 1 }
2322
2323 // Instantiate them
2324 template void resize_h_planar_float_avx2_transpose_vstripe_ks4<0>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
2325 template void resize_h_planar_float_avx2_transpose_vstripe_ks4<1>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
2326 template void resize_h_planar_float_avx2_transpose_vstripe_ks4<2>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
2327 template void resize_h_planar_float_avx2_transpose_vstripe_ks4<3>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
2328
2329 // Instantiate them
2330 template void resize_h_planar_float_avx2_gather_permutex_vstripe_ks4<0>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
2331 template void resize_h_planar_float_avx2_gather_permutex_vstripe_ks4<1>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
2332 template void resize_h_planar_float_avx2_gather_permutex_vstripe_ks4<2>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
2333 template void resize_h_planar_float_avx2_gather_permutex_vstripe_ks4<3>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
2334
2335 //--------------------------------------------------------------------------
2336
2337 // AVX2 Horizontal float
2338
2339 // Three helpers, each for processing 4 target pixels from 16, 8 and 4 source pixel/coeff pairs.
2340
2341 // Helper, implemented for AVX2 (simulating zext)
2342 // zero-extend 128-bit float vector to 256-bit float vector
2343 AVS_FORCEINLINE static __m256 _mm256_zextps128_ps256_avx2(__m128 a)
2344 {
2345 448 __m256 zero_v = _mm256_setzero_ps();
2346 448 return _mm256_insertf128_ps(zero_v, a, 0);
2347 }
2348
2349 // 4 target pixels, each from 16 source pixel/coeff pair
2350 // Called only when accessing 16 source pixels and coefficients at a time is safe
2351 // AVX2 OPTIMIZATION: Process 2x8 blocks to simulate the 16-block stride
2352 AVS_FORCEINLINE static void process_pix4_coeff16_h_float_core_avx2(
2353 const float* src,
2354 int begin1, int begin2, int begin3, int begin4,
2355 const float* current_coeff,
2356 int filter_size,
2357 __m256& result1, __m256& result2, __m256& result3, __m256& result4)
2358 {
2359 // --- Block A: First 8 floats ---
2360 ✗ __m256 data_1_a = _mm256_loadu_ps(src + begin1);
2361 ✗ __m256 coeff_1_a = _mm256_loadu_ps(current_coeff);
2362 ✗ result1 = _mm256_fmadd_ps(data_1_a, coeff_1_a, result1);
2363 ✗ __m256 data_1_b = _mm256_loadu_ps(src + begin1 + 8);
2364 ✗ __m256 coeff_1_b = _mm256_loadu_ps(current_coeff + 8);
2365 ✗ result1 = _mm256_fmadd_ps(data_1_b, coeff_1_b, result1);
2366
2367 ✗ __m256 data_2_a = _mm256_loadu_ps(src + begin2);
2368 ✗ __m256 coeff_2_a = _mm256_loadu_ps(current_coeff + 1 * filter_size);
2369 ✗ result2 = _mm256_fmadd_ps(data_2_a, coeff_2_a, result2);
2370 ✗ __m256 data_2_b = _mm256_loadu_ps(src + begin2 + 8);
2371 ✗ __m256 coeff_2_b = _mm256_loadu_ps(current_coeff + 1 * filter_size + 8);
2372 ✗ result2 = _mm256_fmadd_ps(data_2_b, coeff_2_b, result2);
2373
2374 ✗ __m256 data_3_a = _mm256_loadu_ps(src + begin3);
2375 ✗ __m256 coeff_3_a = _mm256_loadu_ps(current_coeff + 2 * filter_size);
2376 ✗ result3 = _mm256_fmadd_ps(data_3_a, coeff_3_a, result3);
2377 ✗ __m256 data_3_b = _mm256_loadu_ps(src + begin3 + 8);
2378 ✗ __m256 coeff_3_b = _mm256_loadu_ps(current_coeff + 2 * filter_size + 8);
2379 ✗ result3 = _mm256_fmadd_ps(data_3_b, coeff_3_b, result3);
2380
2381 ✗ __m256 data_4_a = _mm256_loadu_ps(src + begin4);
2382 ✗ __m256 coeff_4_a = _mm256_loadu_ps(current_coeff + 3 * filter_size);
2383 ✗ result4 = _mm256_fmadd_ps(data_4_a, coeff_4_a, result4);
2384 ✗ __m256 data_4_b = _mm256_loadu_ps(src + begin4 + 8);
2385 ✗ __m256 coeff_4_b = _mm256_loadu_ps(current_coeff + 3 * filter_size + 8);
2386 ✗ result4 = _mm256_fmadd_ps(data_4_b, coeff_4_b, result4);
2387
2388 ✗ }
2389
2390 // 4 target pixels, each from 8 source pixel/coeff pair
2391 // Called only when accessing 8 source pixels and coefficients at a time is safe
2392 AVS_FORCEINLINE static void process_pix4_coeff8_h_float_core_avx2(
2393 const float* src,
2394 int begin1, int begin2, int begin3, int begin4,
2395 const float* current_coeff,
2396 int filter_size,
2397 __m256& result1, __m256& result2, __m256& result3, __m256& result4)
2398 {
2399 // Load 8 source floats for each of the four beginning source offsets
2400 // Load 8 coefficients for each of the four output pixels
2401 ✗ __m256 data_1 = _mm256_loadu_ps(src + begin1);
2402 ✗ __m256 coeff_1 = _mm256_load_ps(current_coeff); // 8 coeffs for pixel 1
2403 ✗ result1 = _mm256_fmadd_ps(data_1, coeff_1, result1);
2404
2405 ✗ __m256 data_2 = _mm256_loadu_ps(src + begin2);
2406 ✗ __m256 coeff_2 = _mm256_load_ps(current_coeff + 1 * filter_size); // 8 coeffs for pixel 2
2407 ✗ result2 = _mm256_fmadd_ps(data_2, coeff_2, result2);
2408
2409 ✗ __m256 data_3 = _mm256_loadu_ps(src + begin3);
2410 ✗ __m256 coeff_3 = _mm256_load_ps(current_coeff + 2 * filter_size); // 8 coeffs for pixel 3
2411 ✗ result3 = _mm256_fmadd_ps(data_3, coeff_3, result3);
2412
2413 ✗ __m256 data_4 = _mm256_loadu_ps(src + begin4);
2414 ✗ __m256 coeff_4 = _mm256_load_ps(current_coeff + 3 * filter_size); // 8 coeffs for pixel 4
2415 ✗ result4 = _mm256_fmadd_ps(data_4, coeff_4, result4);
2416 ✗ }
2417
2418 // 4 target pixels, each from 4 source pixel/coeff pair.
2419 // Called only for first iteration when results are not initialized.
2420 // Otherwise same as process_pix4_coeff8_h_float_core.
2421 // Optimized: Uses 256-bit MUL directly on zero-extended loads.
2422 AVS_FORCEINLINE static void process_pix4_coeff4_h_float_core_first_avx2(
2423 const float* src,
2424 int begin1, int begin2, int begin3, int begin4,
2425 const float* current_coeff,
2426 int filter_size,
2427 __m256& result1, __m256& result2, __m256& result3, __m256& result4)
2428 {
2429 // Pixel 1
2430 20 __m128 data_1 = _mm_loadu_ps(src + begin1);
2431 20 __m128 coeff_1 = _mm_load_ps(current_coeff);
2432 20 __m128 temp_result_1 = _mm_mul_ps(data_1, coeff_1);
2433 20 result1 = _mm256_zextps128_ps256_avx2(temp_result_1);
2434
2435 // Pixel 2
2436 20 __m128 data_2 = _mm_loadu_ps(src + begin2);
2437 40 __m128 coeff_2 = _mm_load_ps(current_coeff + 1 * filter_size);
2438 20 __m128 temp_result_2 = _mm_mul_ps(data_2, coeff_2);
2439 20 result2 = _mm256_zextps128_ps256_avx2(temp_result_2);
2440
2441 // Pixel 3
2442 20 __m128 data_3 = _mm_loadu_ps(src + begin3);
2443 40 __m128 coeff_3 = _mm_load_ps(current_coeff + 2 * filter_size);
2444 20 __m128 temp_result_3 = _mm_mul_ps(data_3, coeff_3);
2445 20 result3 = _mm256_zextps128_ps256_avx2(temp_result_3);
2446
2447 // Pixel 4
2448 20 __m128 data_4 = _mm_loadu_ps(src + begin4);
2449 40 __m128 coeff_4 = _mm_load_ps(current_coeff + 3 * filter_size);
2450 20 __m128 temp_result_4 = _mm_mul_ps(data_4, coeff_4);
2451 20 result4 = _mm256_zextps128_ps256_avx2(temp_result_4);
2452 20 }
2453
2454 #define HAS_COEFF16_STEP
2455
2456 // filtersize_hint: special: 0..4 for 4,8,16,24,32. Generic: -1
2457 // filter_size is an aligned value and always multiple of 8 (prerequisite)
2458 template<bool safe_aligned_mode, int filtersize_hint>
2459 AVS_FORCEINLINE static void process_four_pixels_h_float_pix4of16_ks_4_8_16_avx2(
2460 const float* src_ptr,
2461 int begin1, int begin2, int begin3, int begin4,
2462 float* current_coeff,
2463 int filter_size,
2464 __m256& result1, __m256& result2, __m256& result3, __m256& result4,
2465 int kernel_size)
2466 {
2467
2468 // very special case: filter size <= 4
2469 if constexpr (safe_aligned_mode) {
2470 if constexpr(filtersize_hint == 0) {
2471 // Process 4 target pixels and 4 source pixels/coefficients at a time
2472 // XMM-based loop internally, but returns __m256 with upper 128 cleared
2473 // Do not assume initialized zeros in result1..4, they will be set here.
2474 process_pix4_coeff4_h_float_core_first_avx2(
2475 src_ptr + 0, begin1, begin2, begin3, begin4,
2476 current_coeff + 0,
2477 filter_size,
2478 result1, result2, result3, result4);
2479 20 return;
2480 }
2481 }
2482
2483 // not: when filtersize_hint == -1, it is not covered with filtersize_hint maximum 4 case, which means
2484 // that real filter size is over 32.
2485 // Thus we surely can have 2x16 coeff processing here.
2486
2487 52 int i = 0;
2488 #ifdef HAS_COEFF16_STEP
2489 if constexpr(filtersize_hint == -1) {
2490 // Handle 2x16 coeffs first, since we know that real filter size is over 32 here, because of filtersize_hint == -1
2491 ✗ const int ksmod16_sure = 32;
2492 // this will be unrolled probably
2493 ✗ for (; i < ksmod16_sure; i += 16) {
2494 // Direct AVX2 adaptation: The core function now updates resultX (YMM) directly
2495 ✗ process_pix4_coeff16_h_float_core_avx2(
2496 ✗ src_ptr + i, begin1, begin2, begin3, begin4,
2497 ✗ current_coeff + i,
2498 filter_size,
2499 result1, result2, result3, result4);
2500 }
2501 // processed 32 coeffs from the kernel, do the rest
2502 ✗ const int ksmod16 = safe_aligned_mode ? (filter_size / 16 * 16) : (kernel_size / 16 * 16);
2503 // Process 4 target pixels and 16 source pixels/coefficients at a time
2504 ✗ for (; i < ksmod16; i += 16) {
2505 // Direct AVX2 adaptation: The core function now updates resultX (YMM) directly
2506 ✗ process_pix4_coeff16_h_float_core_avx2(
2507 ✗ src_ptr + i, begin1, begin2, begin3, begin4,
2508 ✗ current_coeff + i,
2509 filter_size,
2510 result1, result2, result3, result4);
2511 }
2512 }
2513 else {
2514 // do by 16 coeffs till possible
2515 if (filtersize_hint >= 2) {
2516 ✗ const int ksmod16 = safe_aligned_mode ? (filter_size / 16 * 16) : (kernel_size / 16 * 16);
2517 // Process 4 target pixels and 16 source pixels/coefficients at a time
2518 ✗ for (; i < ksmod16; i += 16) {
2519 // Direct AVX2 adaptation: The core function now updates resultX (YMM) directly
2520 ✗ process_pix4_coeff16_h_float_core_avx2(
2521 ✗ src_ptr + i, begin1, begin2, begin3, begin4,
2522 ✗ current_coeff + i,
2523 filter_size,
2524 result1, result2, result3, result4);
2525 }
2526 }
2527 }
2528
2529 // filter sizes 16 or 32 can return here
2530 if constexpr (safe_aligned_mode && (filtersize_hint == 2 || filtersize_hint == 4)) {
2531 ✗ return;
2532 }
2533
2534 if constexpr (!safe_aligned_mode) {
2535
4/38
✗ Branch 206 → 207 not taken.
✗ Branch 206 → 208 not taken.
✗ Branch 254 → 255 not taken.
✓ Branch 254 → 256 taken 13 times.
✗ Branch 349 → 350 not taken.
✗ Branch 349 → 351 not taken.
✗ Branch 397 → 398 not taken.
✓ Branch 397 → 399 taken 13 times.
✗ Branch 461 → 462 not taken.
✗ Branch 461 → 463 not taken.
✗ Branch 515 → 516 not taken.
✗ Branch 515 → 517 not taken.
✗ Branch 543 → 544 not taken.
✗ Branch 543 → 545 not taken.
✗ Branch 563 → 564 not taken.
✓ Branch 563 → 565 taken 13 times.
✗ Branch 655 → 656 not taken.
✗ Branch 655 → 657 not taken.
✗ Branch 658 → 659 not taken.
✗ Branch 658 → 660 not taken.
✗ Branch 706 → 707 not taken.
✓ Branch 706 → 708 taken 13 times.
✗ Branch 721 → 722 not taken.
✗ Branch 721 → 723 not taken.
✗ Branch 760 → 761 not taken.
✗ Branch 760 → 762 not taken.
✗ Branch 872 → 873 not taken.
✗ Branch 872 → 874 not taken.
✗ Branch 954 → 955 not taken.
✗ Branch 954 → 956 not taken.
✗ Branch 967 → 968 not taken.
✗ Branch 967 → 969 not taken.
✗ Branch 1066 → 1067 not taken.
✗ Branch 1066 → 1068 not taken.
✗ Branch 1236 → 1237 not taken.
✗ Branch 1236 → 1238 not taken.
✗ Branch 1482 → 1483 not taken.
✗ Branch 1482 → 1484 not taken.
 52 if (i == kernel_size) return; // kernel_size is not known compile time
2536 }
2537 #else
2538 // no coeff16 step
2539 if constexpr (filtersize_hint == -1) {
2540 // Handle 4x8 coeffs first, since we know that real filter size is over 32 here, because of filtersize_hint == -1
2541 const int ksmod8_sure = 32;
2542 // this will be unrolled probably
2543 for (; i < ksmod8_sure; i += 8) {
2544 process_pix4_coeff8_h_float_core_avx2(
2545 src_ptr + i, begin1, begin2, begin3, begin4,
2546 current_coeff + i,
2547 filter_size,
2548 result1, result2, result3, result4);
2549 }
2550 }
2551 #endif
2552
2553 // When to do the coeff8 step if we had the coeff16 step enabled:
2554 // not safe-aligned mode: always. E.g. kernel_size == 28 -> 16 done, now 10 rest, do 8 next
2555 // filtersize_hint == -1: not-compile-time known filtersize (kernel_size / 16 * 16 done, rest follows)
2556 // filtersize_hint == 1 or 3: 0*16 or 1*16 done, now do 1*8
2557 #ifdef HAS_COEFF16_STEP
2558 if (!safe_aligned_mode || filtersize_hint == -1 || filtersize_hint == 1 || filtersize_hint == 3) {
2559 #else
2560 if (true) {
2561 #endif
2562 // 32 bytes contain 8 floats. We will use 256-bit registers (YMM).
2563 52 const int ksmod8 = safe_aligned_mode ? (filter_size / 8 * 8) : (kernel_size / 8 * 8);
2564
2565 // Process 4 target pixels and 8 source pixels/coefficients at a time (YMM-based loop)
2566
4/60
✗ Branch 43 → 17 not taken.
✗ Branch 43 → 44 not taken.
✗ Branch 77 → 51 not taken.
✗ Branch 77 → 78 not taken.
✗ Branch 94 → 68 not taken.
✗ Branch 94 → 95 not taken.
✗ Branch 134 → 108 not taken.
✗ Branch 134 → 135 not taken.
✗ Branch 146 → 120 not taken.
✗ Branch 146 → 147 not taken.
✗ Branch 168 → 142 not taken.
✗ Branch 168 → 169 not taken.
✗ Branch 179 → 153 not taken.
✗ Branch 179 → 180 not taken.
✗ Branch 235 → 209 not taken.
✗ Branch 235 → 236 not taken.
✗ Branch 283 → 257 not taken.
✓ Branch 283 → 284 taken 13 times.
✗ Branch 287 → 261 not taken.
✗ Branch 287 → 288 not taken.
✗ Branch 372 → 346 not taken.
✗ Branch 372 → 373 not taken.
✗ Branch 378 → 352 not taken.
✗ Branch 378 → 379 not taken.
✗ Branch 426 → 400 not taken.
✓ Branch 426 → 427 taken 13 times.
✗ Branch 443 → 417 not taken.
✗ Branch 443 → 444 not taken.
✗ Branch 490 → 464 not taken.
✗ Branch 490 → 491 not taken.
✗ Branch 544 → 518 not taken.
✗ Branch 544 → 545 not taken.
✗ Branch 572 → 546 not taken.
✗ Branch 572 → 573 not taken.
✗ Branch 580 → 554 not taken.
✗ Branch 580 → 581 not taken.
✗ Branch 592 → 566 not taken.
✓ Branch 592 → 593 taken 13 times.
✗ Branch 684 → 658 not taken.
✗ Branch 684 → 685 not taken.
✗ Branch 687 → 661 not taken.
✗ Branch 687 → 688 not taken.
✗ Branch 735 → 709 not taken.
✓ Branch 735 → 736 taken 13 times.
✗ Branch 750 → 724 not taken.
✗ Branch 750 → 751 not taken.
✗ Branch 789 → 763 not taken.
✗ Branch 789 → 790 not taken.
✗ Branch 901 → 875 not taken.
✗ Branch 901 → 902 not taken.
✗ Branch 983 → 957 not taken.
✗ Branch 983 → 984 not taken.
✗ Branch 996 → 970 not taken.
✗ Branch 996 → 997 not taken.
✗ Branch 1095 → 1069 not taken.
✗ Branch 1095 → 1096 not taken.
✗ Branch 1265 → 1239 not taken.
✗ Branch 1265 → 1266 not taken.
✗ Branch 1511 → 1485 not taken.
✗ Branch 1511 → 1512 not taken.
 52 for (; i < ksmod8; i += 8) {
2567 ✗ process_pix4_coeff8_h_float_core_avx2(
2568 ✗ src_ptr + i, begin1, begin2, begin3, begin4,
2569 ✗ current_coeff + i,
2570 filter_size,
2571 result1, result2, result3, result4);
2572 }
2573 }
2574
2575 if constexpr (!safe_aligned_mode) {
2576 // Right edge case.
2577 // Coeffs are zero padded, reading them is no problem.
2578 // But if we read past the end of source then we can get possible NaN contamination.
2579 // Handle the remainder: 1 to 7 source/coefficient elements.
2580 // real_kernel_size is used here, it's guaranteed that reading real_kernel_size elements
2581 // from any pixel_offset[] is safe and ends within the source buffer.
2582 // Optional 4-2-1 processing loop.
2583
2584
4/38
✗ Branch 236 → 237 not taken.
✗ Branch 236 → 238 not taken.
✗ Branch 284 → 285 not taken.
✓ Branch 284 → 286 taken 13 times.
✗ Branch 379 → 380 not taken.
✗ Branch 379 → 381 not taken.
✗ Branch 427 → 428 not taken.
✓ Branch 427 → 429 taken 13 times.
✗ Branch 491 → 492 not taken.
✗ Branch 491 → 493 not taken.
✗ Branch 545 → 546 not taken.
✗ Branch 545 → 547 not taken.
✗ Branch 573 → 574 not taken.
✗ Branch 573 → 575 not taken.
✗ Branch 593 → 594 not taken.
✓ Branch 593 → 595 taken 13 times.
✗ Branch 685 → 686 not taken.
✗ Branch 685 → 687 not taken.
✗ Branch 688 → 689 not taken.
✗ Branch 688 → 690 not taken.
✗ Branch 736 → 737 not taken.
✓ Branch 736 → 738 taken 13 times.
✗ Branch 751 → 752 not taken.
✗ Branch 751 → 753 not taken.
✗ Branch 790 → 791 not taken.
✗ Branch 790 → 792 not taken.
✗ Branch 902 → 903 not taken.
✗ Branch 902 → 904 not taken.
✗ Branch 984 → 985 not taken.
✗ Branch 984 → 986 not taken.
✗ Branch 997 → 998 not taken.
✗ Branch 997 → 999 not taken.
✗ Branch 1096 → 1097 not taken.
✗ Branch 1096 → 1098 not taken.
✗ Branch 1266 → 1267 not taken.
✗ Branch 1266 → 1268 not taken.
✗ Branch 1512 → 1513 not taken.
✗ Branch 1512 → 1514 not taken.
 52 if (i == kernel_size) return;
2585
2586 // --- Define Base Pointers for Source and Coefficients ---
2587 52 const float* src_ptr1 = src_ptr + begin1;
2588 52 const float* src_ptr2 = src_ptr + begin2;
2589 52 const float* src_ptr3 = src_ptr + begin3;
2590 52 const float* src_ptr4 = src_ptr + begin4;
2591
2592 52 float* current_coeff2 = current_coeff + 1 * filter_size;
2593 52 float* current_coeff3 = current_coeff + 2 * filter_size;
2594 52 float* current_coeff4 = current_coeff + 3 * filter_size;
2595
2596 52 const int ksmod4 = kernel_size / 4 * 4;
2597
2598 // -------------------------------------------------------------------
2599 // Mod 4 Block (4 elements for four pixels using __m128)
2600 // -------------------------------------------------------------------
2601
8/38
✗ Branch 238 → 239 not taken.
✗ Branch 238 → 289 not taken.
✓ Branch 286 → 287 taken 8 times.
✓ Branch 286 → 337 taken 5 times.
✗ Branch 381 → 382 not taken.
✗ Branch 381 → 432 not taken.
✓ Branch 429 → 430 taken 8 times.
✓ Branch 429 → 480 taken 5 times.
✗ Branch 493 → 494 not taken.
✗ Branch 493 → 544 not taken.
✗ Branch 547 → 548 not taken.
✗ Branch 547 → 598 not taken.
✗ Branch 575 → 576 not taken.
✗ Branch 575 → 626 not taken.
✓ Branch 595 → 596 taken 8 times.
✓ Branch 595 → 646 taken 5 times.
✗ Branch 687 → 688 not taken.
✗ Branch 687 → 738 not taken.
✗ Branch 690 → 691 not taken.
✗ Branch 690 → 741 not taken.
✓ Branch 738 → 739 taken 8 times.
✓ Branch 738 → 789 taken 5 times.
✗ Branch 753 → 754 not taken.
✗ Branch 753 → 804 not taken.
✗ Branch 792 → 793 not taken.
✗ Branch 792 → 843 not taken.
✗ Branch 904 → 905 not taken.
✗ Branch 904 → 955 not taken.
✗ Branch 986 → 987 not taken.
✗ Branch 986 → 1037 not taken.
✗ Branch 999 → 1000 not taken.
✗ Branch 999 → 1050 not taken.
✗ Branch 1098 → 1099 not taken.
✗ Branch 1098 → 1149 not taken.
✗ Branch 1268 → 1269 not taken.
✗ Branch 1268 → 1319 not taken.
✗ Branch 1514 → 1515 not taken.
✗ Branch 1514 → 1565 not taken.
 52 if (i < ksmod4) {
2602
2603 // Pixel 1
2604 32 __m128 data_1 = _mm_loadu_ps(src_ptr1 + i);
2605 64 __m128 coeff_1 = _mm_load_ps(current_coeff + i);
2606 32 __m128 temp_result_1 = _mm_mul_ps(data_1, coeff_1);
2607 32 result1 = _mm256_add_ps(result1, _mm256_zextps128_ps256_avx2(temp_result_1));
2608
2609 // Pixel 2
2610 32 __m128 data_2 = _mm_loadu_ps(src_ptr2 + i);
2611 64 __m128 coeff_2 = _mm_load_ps(current_coeff2 + i);
2612 32 __m128 temp_result_2 = _mm_mul_ps(data_2, coeff_2);
2613 32 result2 = _mm256_add_ps(result2, _mm256_zextps128_ps256_avx2(temp_result_2));
2614
2615 // Pixel 3
2616 32 __m128 data_3 = _mm_loadu_ps(src_ptr3 + i);
2617 64 __m128 coeff_3 = _mm_load_ps(current_coeff3 + i);
2618 32 __m128 temp_result_3 = _mm_mul_ps(data_3, coeff_3);
2619 32 result3 = _mm256_add_ps(result3, _mm256_zextps128_ps256_avx2(temp_result_3));
2620
2621 // Pixel 4
2622 32 __m128 data_4 = _mm_loadu_ps(src_ptr4 + i);
2623 64 __m128 coeff_4 = _mm_load_ps(current_coeff4 + i);
2624 32 __m128 temp_result_4 = _mm_mul_ps(data_4, coeff_4);
2625 32 result4 = _mm256_add_ps(result4, _mm256_zextps128_ps256_avx2(temp_result_4));
2626
2627 32 i += 4;
2628
4/38
✗ Branch 287 → 288 not taken.
✗ Branch 287 → 289 not taken.
✓ Branch 335 → 336 taken 8 times.
✗ Branch 335 → 337 not taken.
✗ Branch 430 → 431 not taken.
✗ Branch 430 → 432 not taken.
✓ Branch 478 → 479 taken 8 times.
✗ Branch 478 → 480 not taken.
✗ Branch 542 → 543 not taken.
✗ Branch 542 → 544 not taken.
✗ Branch 596 → 597 not taken.
✗ Branch 596 → 598 not taken.
✗ Branch 624 → 625 not taken.
✗ Branch 624 → 626 not taken.
✓ Branch 644 → 645 taken 8 times.
✗ Branch 644 → 646 not taken.
✗ Branch 736 → 737 not taken.
✗ Branch 736 → 738 not taken.
✗ Branch 739 → 740 not taken.
✗ Branch 739 → 741 not taken.
✓ Branch 787 → 788 taken 8 times.
✗ Branch 787 → 789 not taken.
✗ Branch 802 → 803 not taken.
✗ Branch 802 → 804 not taken.
✗ Branch 841 → 842 not taken.
✗ Branch 841 → 843 not taken.
✗ Branch 953 → 954 not taken.
✗ Branch 953 → 955 not taken.
✗ Branch 1035 → 1036 not taken.
✗ Branch 1035 → 1037 not taken.
✗ Branch 1048 → 1049 not taken.
✗ Branch 1048 → 1050 not taken.
✗ Branch 1147 → 1148 not taken.
✗ Branch 1147 → 1149 not taken.
✗ Branch 1317 → 1318 not taken.
✗ Branch 1317 → 1319 not taken.
✗ Branch 1563 → 1564 not taken.
✗ Branch 1563 → 1565 not taken.
 32 if (i == kernel_size) return;
2629 }
2630
2631 20 const int ksmod2 = kernel_size / 2 * 2;
2632
2633 // -------------------------------------------------------------------
2634 // Mod 2 Block (2 elements for four pixels using __m128)
2635 // -------------------------------------------------------------------
2636
4/38
✗ Branch 289 → 290 not taken.
✗ Branch 289 → 310 not taken.
✓ Branch 337 → 338 taken 5 times.
✗ Branch 337 → 358 not taken.
✗ Branch 432 → 433 not taken.
✗ Branch 432 → 453 not taken.
✓ Branch 480 → 481 taken 5 times.
✗ Branch 480 → 501 not taken.
✗ Branch 544 → 545 not taken.
✗ Branch 544 → 565 not taken.
✗ Branch 598 → 599 not taken.
✗ Branch 598 → 619 not taken.
✗ Branch 626 → 627 not taken.
✗ Branch 626 → 647 not taken.
✓ Branch 646 → 647 taken 5 times.
✗ Branch 646 → 667 not taken.
✗ Branch 738 → 739 not taken.
✗ Branch 738 → 759 not taken.
✗ Branch 741 → 742 not taken.
✗ Branch 741 → 762 not taken.
✓ Branch 789 → 790 taken 5 times.
✗ Branch 789 → 810 not taken.
✗ Branch 804 → 805 not taken.
✗ Branch 804 → 825 not taken.
✗ Branch 843 → 844 not taken.
✗ Branch 843 → 864 not taken.
✗ Branch 955 → 956 not taken.
✗ Branch 955 → 976 not taken.
✗ Branch 1037 → 1038 not taken.
✗ Branch 1037 → 1058 not taken.
✗ Branch 1050 → 1051 not taken.
✗ Branch 1050 → 1071 not taken.
✗ Branch 1149 → 1150 not taken.
✗ Branch 1149 → 1170 not taken.
✗ Branch 1319 → 1320 not taken.
✗ Branch 1319 → 1340 not taken.
✗ Branch 1565 → 1566 not taken.
✗ Branch 1565 → 1586 not taken.
 20 if (i < ksmod2) {
2637 // _mm_load_sd (vmovsd) loads 64 bits and zeroes everything else in the register.
2638 // Vector looks like: [0, 0, 0, 0, 0, 0, val1, val0]
2639 // We can use the exact same FMA trick.
2640
2641 160 auto load_2_floats_as_ymm = [](const float* p) {
2642 480 return _mm256_zextps128_ps256_avx2(_mm_castpd_ps(_mm_load_sd(reinterpret_cast<const double*>(p))));
2643 };
2644
2645 20 result1 = _mm256_fmadd_ps(load_2_floats_as_ymm(src_ptr1 + i), load_2_floats_as_ymm(current_coeff + i), result1);
2646 20 result2 = _mm256_fmadd_ps(load_2_floats_as_ymm(src_ptr2 + i), load_2_floats_as_ymm(current_coeff2 + i), result2);
2647 20 result3 = _mm256_fmadd_ps(load_2_floats_as_ymm(src_ptr3 + i), load_2_floats_as_ymm(current_coeff3 + i), result3);
2648 20 result4 = _mm256_fmadd_ps(load_2_floats_as_ymm(src_ptr4 + i), load_2_floats_as_ymm(current_coeff4 + i), result4);
2649
2650 20 i += 2;
2651
4/38
✗ Branch 306 → 307 not taken.
✗ Branch 306 → 308 not taken.
✗ Branch 354 → 355 not taken.
✓ Branch 354 → 356 taken 5 times.
✗ Branch 449 → 450 not taken.
✗ Branch 449 → 451 not taken.
✗ Branch 497 → 498 not taken.
✓ Branch 497 → 499 taken 5 times.
✗ Branch 561 → 562 not taken.
✗ Branch 561 → 563 not taken.
✗ Branch 615 → 616 not taken.
✗ Branch 615 → 617 not taken.
✗ Branch 643 → 644 not taken.
✗ Branch 643 → 645 not taken.
✗ Branch 663 → 664 not taken.
✓ Branch 663 → 665 taken 5 times.
✗ Branch 755 → 756 not taken.
✗ Branch 755 → 757 not taken.
✗ Branch 758 → 759 not taken.
✗ Branch 758 → 760 not taken.
✗ Branch 806 → 807 not taken.
✓ Branch 806 → 808 taken 5 times.
✗ Branch 821 → 822 not taken.
✗ Branch 821 → 823 not taken.
✗ Branch 860 → 861 not taken.
✗ Branch 860 → 862 not taken.
✗ Branch 972 → 973 not taken.
✗ Branch 972 → 974 not taken.
✗ Branch 1054 → 1055 not taken.
✗ Branch 1054 → 1056 not taken.
✗ Branch 1067 → 1068 not taken.
✗ Branch 1067 → 1069 not taken.
✗ Branch 1166 → 1167 not taken.
✗ Branch 1166 → 1168 not taken.
✗ Branch 1336 → 1337 not taken.
✗ Branch 1336 → 1338 not taken.
✗ Branch 1582 → 1583 not taken.
✗ Branch 1582 → 1584 not taken.
 20 if (i == kernel_size) return;
2652 }
2653
2654 // -------------------------------------------------------------------
2655 // Fallback Scalar Operation (1 element remaining)
2656 // -------------------------------------------------------------------
2657
4/38
✗ Branch 310 → 311 not taken.
✗ Branch 310 → 344 not taken.
✓ Branch 358 → 359 taken 5 times.
✗ Branch 358 → 392 not taken.
✗ Branch 453 → 454 not taken.
✗ Branch 453 → 487 not taken.
✓ Branch 501 → 502 taken 5 times.
✗ Branch 501 → 535 not taken.
✗ Branch 565 → 566 not taken.
✗ Branch 565 → 599 not taken.
✗ Branch 619 → 620 not taken.
✗ Branch 619 → 653 not taken.
✗ Branch 647 → 648 not taken.
✗ Branch 647 → 681 not taken.
✓ Branch 667 → 668 taken 5 times.
✗ Branch 667 → 701 not taken.
✗ Branch 759 → 760 not taken.
✗ Branch 759 → 793 not taken.
✗ Branch 762 → 763 not taken.
✗ Branch 762 → 796 not taken.
✓ Branch 810 → 811 taken 5 times.
✗ Branch 810 → 844 not taken.
✗ Branch 825 → 826 not taken.
✗ Branch 825 → 859 not taken.
✗ Branch 864 → 865 not taken.
✗ Branch 864 → 898 not taken.
✗ Branch 976 → 977 not taken.
✗ Branch 976 → 1010 not taken.
✗ Branch 1058 → 1059 not taken.
✗ Branch 1058 → 1092 not taken.
✗ Branch 1071 → 1072 not taken.
✗ Branch 1071 → 1105 not taken.
✗ Branch 1170 → 1171 not taken.
✗ Branch 1170 → 1204 not taken.
✗ Branch 1340 → 1341 not taken.
✗ Branch 1340 → 1374 not taken.
✗ Branch 1586 → 1587 not taken.
✗ Branch 1586 → 1620 not taken.
 20 if (i < kernel_size) {
2658
2659 // Optimized scalar loop for the single remaining element
2660 20 float final_scalar1 = src_ptr1[i] * current_coeff[i];
2661 20 float final_scalar2 = src_ptr2[i] * current_coeff2[i];
2662 20 float final_scalar3 = src_ptr3[i] * current_coeff3[i];
2663 20 float final_scalar4 = src_ptr4[i] * current_coeff4[i];
2664
2665 // Using the helper for the last one to be safe/consistent with scalar ops
2666 60 result1 = _mm256_add_ps(result1, _mm256_zextps128_ps256_avx2(_mm_set_ss(final_scalar1)));
2667 60 result2 = _mm256_add_ps(result2, _mm256_zextps128_ps256_avx2(_mm_set_ss(final_scalar2)));
2668 60 result3 = _mm256_add_ps(result3, _mm256_zextps128_ps256_avx2(_mm_set_ss(final_scalar3)));
2669 60 result4 = _mm256_add_ps(result4, _mm256_zextps128_ps256_avx2(_mm_set_ss(final_scalar4)));
2670 // i is now equal to kernel_size (i++)
2671 }
2672 }
2673 ✗ }
2674
2675
2676 template<bool is_safe, int filtersize_hint>
2677 AVS_FORCEINLINE static void process_sixteen_pixels_h_float_pix16_sub4_ks_4_8_16_avx2(
2678 const float* src, int x, float* current_coeff_base,
2679 int filter_size, // 8, 16, 24, 32 are quasi-constexpr here, others not compile-time known but still aligned to 8
2680 float* dst,
2681 ResamplingProgram* program)
2682 {
2683
2/12
✗ Branch 7 → 8 not taken.
✓ Branch 7 → 9 taken 5 times.
✗ Branch 197 → 198 not taken.
✗ Branch 197 → 199 not taken.
✗ Branch 245 → 246 not taken.
✓ Branch 245 → 247 taken 13 times.
✗ Branch 289 → 290 not taken.
✗ Branch 289 → 291 not taken.
✗ Branch 401 → 402 not taken.
✗ Branch 401 → 403 not taken.
✗ Branch 609 → 610 not taken.
✗ Branch 609 → 611 not taken.
 18 assert(program->filter_size_alignment == 8);
2684
2685 18 float* current_coeff = current_coeff_base + x * filter_size;
2686 18 const int unaligned_kernel_size = program->filter_size_real;
2687 18 const __m256 zero256 = _mm256_setzero_ps();
2688
2689 // --- Block 1: Pixels 0, 1, 2, 3 ---
2690 18 __m256 result0 = zero256;
2691 18 __m256 result1 = zero256;
2692 18 __m256 result2 = zero256;
2693 18 __m256 result3 = zero256;
2694
2695 18 int begin0 = program->pixel_offset[x + 0];
2696 18 int begin1 = program->pixel_offset[x + 1];
2697 18 int begin2 = program->pixel_offset[x + 2];
2698 18 int begin3 = program->pixel_offset[x + 3];
2699
2700 process_four_pixels_h_float_pix4of16_ks_4_8_16_avx2<is_safe, filtersize_hint>(
2701 src, begin0, begin1, begin2, begin3, current_coeff, filter_size,
2702 result0, result1, result2, result3, unaligned_kernel_size);
2703 18 current_coeff += 4 * filter_size;
2704
2705 // --- Block 2: Pixels 4, 5, 6, 7 ---
2706 18 __m256 result4 = zero256;
2707 18 __m256 result5 = zero256;
2708 18 __m256 result6 = zero256;
2709 18 __m256 result7 = zero256;
2710
2711 18 int begin4 = program->pixel_offset[x + 4];
2712 18 int begin5 = program->pixel_offset[x + 5];
2713 18 int begin6 = program->pixel_offset[x + 6];
2714 18 int begin7 = program->pixel_offset[x + 7];
2715
2716 process_four_pixels_h_float_pix4of16_ks_4_8_16_avx2<is_safe, filtersize_hint>(
2717 src, begin4, begin5, begin6, begin7, current_coeff, filter_size,
2718 result4, result5, result6, result7, unaligned_kernel_size);
2719 18 current_coeff += 4 * filter_size;
2720
2721 // ---------------------------------------------------------------------------
2722 // REDUCTION FOR PIXELS 0-7 (Result256_low)
2723 // ---------------------------------------------------------------------------
2724
2725 // Round 1: Reduce pairs (8 vectors -> 4 vectors)
2726 18 __m256 sum01 = _mm256_hadd_ps(result0, result1);
2727 18 __m256 sum23 = _mm256_hadd_ps(result2, result3);
2728 18 __m256 sum45 = _mm256_hadd_ps(result4, result5);
2729 36 __m256 sum67 = _mm256_hadd_ps(result6, result7);
2730
2731 // Round 2: Reduce quads (4 vectors -> 2 vectors)
2732 18 __m256 sum0123 = _mm256_hadd_ps(sum01, sum23);
2733 18 __m256 sum4567 = _mm256_hadd_ps(sum45, sum67);
2734
2735 // Round 3: Final Merge (Add Lower 128-bit to Upper 128-bit)
2736 18 __m128 lo_0123 = _mm256_castps256_ps128(sum0123);
2737 18 __m128 lo_4567 = _mm256_castps256_ps128(sum4567);
2738 18 __m256 result_lo = _mm256_insertf128_ps(_mm256_castps128_ps256(lo_0123), lo_4567, 1);
2739
2740 18 __m128 hi_0123 = _mm256_extractf128_ps(sum0123, 1);
2741 18 __m128 hi_4567 = _mm256_extractf128_ps(sum4567, 1);
2742 18 __m256 result_hi = _mm256_insertf128_ps(_mm256_castps128_ps256(hi_0123), hi_4567, 1);
2743
2744 // Assemble the Low 256-bit result (Pixels 0-7)
2745 18 __m256 result256_low = _mm256_add_ps(result_lo, result_hi);
2746 18 _mm256_stream_ps(reinterpret_cast<float*>(dst + x), result256_low);
2747
2748
2749 // --- Block 3: Pixels 8, 9, 10, 11 ---
2750 18 __m256 result8 = zero256;
2751 18 __m256 result9 = zero256;
2752 18 __m256 result10 = zero256;
2753 18 __m256 result11 = zero256;
2754
2755 18 int begin8 = program->pixel_offset[x + 8];
2756 18 int begin9 = program->pixel_offset[x + 9];
2757 18 int begin10 = program->pixel_offset[x + 10];
2758 18 int begin11 = program->pixel_offset[x + 11];
2759
2760 process_four_pixels_h_float_pix4of16_ks_4_8_16_avx2<is_safe, filtersize_hint>(
2761 src, begin8, begin9, begin10, begin11, current_coeff, filter_size,
2762 result8, result9, result10, result11, unaligned_kernel_size);
2763 18 current_coeff += 4 * filter_size;
2764
2765 // --- Block 4: Pixels 12, 13, 14, 15 ---
2766 18 __m256 result12 = zero256;
2767 18 __m256 result13 = zero256;
2768 18 __m256 result14 = zero256;
2769 18 __m256 result15 = zero256;
2770
2771 18 int begin12 = program->pixel_offset[x + 12];
2772 18 int begin13 = program->pixel_offset[x + 13];
2773 18 int begin14 = program->pixel_offset[x + 14];
2774 18 int begin15 = program->pixel_offset[x + 15];
2775
2776 process_four_pixels_h_float_pix4of16_ks_4_8_16_avx2<is_safe, filtersize_hint>(
2777 src, begin12, begin13, begin14, begin15, current_coeff, filter_size,
2778 result12, result13, result14, result15, unaligned_kernel_size);
2779
2780
2781 // ---------------------------------------------------------------------------
2782 // REDUCTION FOR PIXELS 8-15 (Result256_high)
2783 // ---------------------------------------------------------------------------
2784
2785 // Round 1: Reduce pairs (8 vectors -> 4 vectors)
2786 18 __m256 sum89 = _mm256_hadd_ps(result8, result9);
2787 18 __m256 sum1011 = _mm256_hadd_ps(result10, result11);
2788 18 __m256 sum1213 = _mm256_hadd_ps(result12, result13);
2789 36 __m256 sum1415 = _mm256_hadd_ps(result14, result15);
2790
2791 // Round 2: Reduce quads (4 vectors -> 2 vectors)
2792 18 __m256 sum8_11 = _mm256_hadd_ps(sum89, sum1011);
2793 18 __m256 sum12_15 = _mm256_hadd_ps(sum1213, sum1415);
2794
2795 // Round 3: Final Merge (Add Lower 128-bit to Upper 128-bit)
2796 18 __m128 lo_8_11 = _mm256_castps256_ps128(sum8_11);
2797 18 __m128 lo_12_15 = _mm256_castps256_ps128(sum12_15);
2798 18 __m256 result_lo_high = _mm256_insertf128_ps(_mm256_castps128_ps256(lo_8_11), lo_12_15, 1);
2799
2800 18 __m128 hi_8_11 = _mm256_extractf128_ps(sum8_11, 1);
2801 18 __m128 hi_12_15 = _mm256_extractf128_ps(sum12_15, 1);
2802 18 __m256 result_hi_high = _mm256_insertf128_ps(_mm256_castps128_ps256(hi_8_11), hi_12_15, 1);
2803
2804 // Assemble the High 256-bit result (Pixels 8-15)
2805 18 __m256 result256_high = _mm256_add_ps(result_lo_high, result_hi_high);
2806
2807 // ---------------------------------------------------------------------------
2808 // Stream the two 256-bit results
2809 // ---------------------------------------------------------------------------
2810 18 _mm256_stream_ps(reinterpret_cast<float*>(dst + x + 8), result256_high);
2811 18 }
2812
2813 // filtersizealigned8: special: 0, 1..4, Generic : -1
2814 template<int filtersize_hint>
2815 2 static void internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) {
2816 AVS_UNUSED(bits_per_pixel);
2817 // filter_size is aligned to 8 (prerequisite), contrary that we have a special case for filter size <=4
2818
2819 // We note that when template is used, filter_size is quasi-constexpr if filtersize_hint != -1.
2820 // When filtersize_hint == -1, then program->filter_size is aligned to 8 anyway, but not known at compile time.
2821 2 const int filter_size =
2822 filtersize_hint == 0 ? 8 : // though we'll optimize for 4 internally, coeff buffer is still allocated for 8
2823 (filtersize_hint >= 1) ? filtersize_hint * 8 : program->filter_size; // this latter is always aligned to 8 as well
2824
2825 2 const float* src = (float*)src8;
2826 2 float* dst = (float*)dst8;
2827 2 dst_pitch = dst_pitch / sizeof(float);
2828 2 src_pitch = src_pitch / sizeof(float);
2829
2830 2 constexpr int PIXELS_AT_A_TIME = 16;
2831 // Align safe zone to 16 pixels
2832
1/12
void internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<0>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 2 → 3 taken 2 times.
✗ Branch 2 → 4 not taken.
void internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 2 → 3 not taken.
✗ Branch 2 → 4 not taken.
void internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<2>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 2 → 3 not taken.
✗ Branch 2 → 4 not taken.
void internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<3>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 2 → 3 not taken.
✗ Branch 2 → 4 not taken.
void internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<4>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 2 → 3 not taken.
✗ Branch 2 → 4 not taken.
void internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<-1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 2 → 3 not taken.
✗ Branch 2 → 4 not taken.
 2 const int w_safe_mod16 = (program->safelimit_16_pixels.overread_possible ? program->safelimit_16_pixels.source_overread_beyond_targetx : width) / PIXELS_AT_A_TIME * PIXELS_AT_A_TIME;
2833
2834
2/12
void internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<0>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 871 → 6 taken 9 times.
✓ Branch 871 → 872 taken 2 times.
void internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 823 → 6 not taken.
✗ Branch 823 → 824 not taken.
void internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<2>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 1119 → 6 not taken.
✗ Branch 1119 → 1120 not taken.
void internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<3>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 1231 → 6 not taken.
✗ Branch 1231 → 1232 not taken.
void internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<4>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 1119 → 6 not taken.
✗ Branch 1119 → 1120 not taken.
void internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<-1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 1647 → 6 not taken.
✗ Branch 1647 → 1648 not taken.
 11 for (int y = 0; y < height; y++) {
2835 9 float* current_coeff_base = program->pixel_coefficient_float;
2836
2837 // Process safe aligned pixels
2838
2/12
void internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<0>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 243 → 7 taken 5 times.
✓ Branch 243 → 244 taken 9 times.
void internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 195 → 7 not taken.
✗ Branch 195 → 196 not taken.
void internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<2>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 287 → 7 not taken.
✗ Branch 287 → 288 not taken.
void internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<3>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 399 → 7 not taken.
✗ Branch 399 → 400 not taken.
void internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<4>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 287 → 7 not taken.
✗ Branch 287 → 288 not taken.
void internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<-1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 607 → 7 not taken.
✗ Branch 607 → 608 not taken.
 14 for (int x = 0; x < w_safe_mod16; x += PIXELS_AT_A_TIME) {
2839 process_sixteen_pixels_h_float_pix16_sub4_ks_4_8_16_avx2<true, filtersize_hint>(src, x, current_coeff_base, filter_size, dst, program);
2840 }
2841
2842 // Process up to the actual kernel size (unsafe zone)
2843
2/12
void internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<0>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 869 → 245 taken 13 times.
✓ Branch 869 → 870 taken 9 times.
void internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 821 → 197 not taken.
✗ Branch 821 → 822 not taken.
void internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<2>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 1117 → 289 not taken.
✗ Branch 1117 → 1118 not taken.
void internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<3>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 1229 → 401 not taken.
✗ Branch 1229 → 1230 not taken.
void internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<4>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 1117 → 289 not taken.
✗ Branch 1117 → 1118 not taken.
void internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<-1>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✗ Branch 1645 → 609 not taken.
✗ Branch 1645 → 1646 not taken.
 22 for (int x = w_safe_mod16; x < width; x += PIXELS_AT_A_TIME) {
2844 process_sixteen_pixels_h_float_pix16_sub4_ks_4_8_16_avx2<false, filtersize_hint>(src, x, current_coeff_base, filter_size, dst, program);
2845 }
2846
2847 9 dst += dst_pitch;
2848 9 src += src_pitch;
2849 }
2850 2 }
2851
2852 // Winner implementation: resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16;
2853 2 void resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) {
2854 2 const int filter_size = program->filter_size;
2855 // Expected alignment
2856
1/2
✗ Branch 2 → 3 not taken.
✓ Branch 2 → 4 taken 2 times.
 2 assert(program->filter_size_alignment >= 8);
2857
2858 // Dispatcher template now supports filter_size aligned to 8 (8, 16, 24, 32) and a special case for <=4
2859 // Larger filter sizes will use the generic method (-1) which still benefit from 16-8-4 coeff processing blocks.
2860
1/2
✓ Branch 4 → 5 taken 2 times.
✗ Branch 4 → 8 not taken.
 2 if (filter_size == 1 * 8)
2861
1/2
✓ Branch 5 → 6 taken 2 times.
✗ Branch 5 → 7 not taken.
 2 if (program->filter_size_real <= 4)
2862 2 internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<0>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel); // Internally optimized for 4
2863 else
2864 ✗ internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<1>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel); // Internally optimized for 8
2865 ✗ else if (filter_size == 2 * 8) // Internally optimized for 16
2866 ✗ internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<2>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
2867 ✗ else if (filter_size == 3 * 8) // Internally optimized for 16+8
2868 ✗ internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<3>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
2869 ✗ else if (filter_size == 4 * 8) // Internally optimized for 2*16
2870 ✗ internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<4>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
2871 else // -1: basic method, use program->filter_size, internally optimized for calculating coeffs in N*16 + 8 + 4 + 2 + 1 blocks
2872 ✗ internal_resizer_h_avx2_generic_float_pix16_sub4_ks_4_8_16<-1>(dst8, src8, dst_pitch, src_pitch, program, width, height, bits_per_pixel);
2873 2 }
2874
2875