GCC Code Coverage Report

Directory: avs_core/
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 72.4% 745 / 0 / 1029
Functions: 68.9% 62 / 0 / 90
Branches: 73.3% 110 / 0 / 150
filters/overlay/intel/masked_rowprep_avx2.cpp
Line Branch Exec Source
1 // masked_rowprep_avx2.cpp
2 // AVX2 rowprep implementations + explicit template instantiations.
3 // Compiled with -mavx2 -mfma (GCC/Clang) or /arch:AVX2 (MSVC) via handle_arch_flags(AVX2).
4 //
5 // avx2_pack_* helpers are static — internal to this TU only.
6
7 #if defined(_MSC_VER)
8 #include <intrin.h>
9 #else
10 #include <immintrin.h>
11 #endif
12
13 #include "avs/config.h"
14 #include "../blend_common.h"
15 #include "masked_rowprep_avx2_impl.h" // declarations + simd_magic_div_32_avx2 inline
16 #include <vector>
17 #include <cstdint>
18
19 // ---------------------------------------------------------------------------
20 // Pack helpers — internal to this TU.
21 // ---------------------------------------------------------------------------
22 static AVS_FORCEINLINE __m128i avx2_pack_u16_to_u8(const __m256i& v) {
23 356 return _mm_packus_epi16(_mm256_castsi256_si128(v), _mm256_extracti128_si256(v, 1));
24 }
25 static AVS_FORCEINLINE __m128i avx2_pack_u32_to_u16(const __m256i& v) {
26 215 __m256i packed = _mm256_packus_epi32(v, _mm256_setzero_si256());
27 645 return _mm_unpacklo_epi64(_mm256_castsi256_si128(packed), _mm256_extracti128_si256(packed, 1));
28 }
29
30 // ---------------------------------------------------------------------------
31 // MASK422 — horizontal 2-tap average.
32 // avg[x] = (src[x*2] + src[x*2+1] + 1) >> 1
33 // ---------------------------------------------------------------------------
34 template<typename pixel_t, bool full_opacity>
35 #if defined(GCC) || defined(CLANG)
36 __attribute__((__target__("avx2")))
37 #endif
38 45 static void fill_mask422_avx2(
39 pixel_t* dst, const pixel_t* src, int width,
40 int opacity_i, int half, MagicDiv magic)
41 {
42 45 int x = 0;
43 if constexpr (sizeof(pixel_t) == 1) {
44 20 [[maybe_unused]] const __m256i v_opacity = _mm256_set1_epi16((short)opacity_i);
45 20 [[maybe_unused]] const __m256i v_half16 = _mm256_set1_epi16((short)half);
46 40 [[maybe_unused]] const __m256i v_mdiv = _mm256_set1_epi16((short)magic.div);
47 20 const __m256i mask_lo = _mm256_set1_epi16(0x00FF);
48
4/4
void fill_mask422_avx2<unsigned char, false>(unsigned char*, unsigned char const*, int, int, int, MagicDiv):
✓ Branch 51 → 19 taken 10 times.
✓ Branch 51 → 52 taken 10 times.
void fill_mask422_avx2<unsigned char, true>(unsigned char*, unsigned char const*, int, int, int, MagicDiv):
✓ Branch 43 → 19 taken 10 times.
✓ Branch 43 → 44 taken 10 times.
 40 for (; x <= width - 16; x += 16) {
49 40 __m256i v = _mm256_loadu_si256((const __m256i*)(src + x * 2));
50 20 __m256i even = _mm256_and_si256(v, mask_lo);
51 20 __m256i odd = _mm256_srli_epi16(v, 8);
52 70 __m256i avg = _mm256_srli_epi16(
53 _mm256_add_epi16(_mm256_add_epi16(even, odd), _mm256_set1_epi16(1)), 1);
54 if constexpr (!full_opacity) {
55 20 __m256i scaled = _mm256_add_epi16(_mm256_mullo_epi16(avg, v_opacity), v_half16);
56 30 avg = _mm256_srli_epi16(_mm256_mulhi_epu16(scaled, v_mdiv), magic.shift);
57 }
58 20 _mm_storeu_si128((__m128i*)(dst + x), avx2_pack_u16_to_u8(avg));
59 }
60 } else {
61 // 16-bit: 16 uint16 luma -> 8 uint16 chroma per iteration.
62 25 const __m256i ones = _mm256_set1_epi16(1);
63 25 const __m256i v_pivot16 = _mm256_set1_epi16(-32768);
64 // 65536 corrects the double-bias subtraction from madd, +1 handles the formula's rounding
65 25 const __m256i v_correct32 = _mm256_set1_epi32(65536 + 1);
66
67 25 [[maybe_unused]] const __m256i v_opacity32 = _mm256_set1_epi32(opacity_i);
68 25 [[maybe_unused]] const __m256i v_half32 = _mm256_set1_epi32(half);
69
70
4/4
void fill_mask422_avx2<unsigned short, false>(unsigned short*, unsigned short const*, int, int, int, MagicDiv):
✓ Branch 57 → 17 taken 25 times.
✓ Branch 57 → 58 taken 15 times.
void fill_mask422_avx2<unsigned short, true>(unsigned short*, unsigned short const*, int, int, int, MagicDiv):
✓ Branch 39 → 17 taken 15 times.
✓ Branch 39 → 40 taken 10 times.
 65 for (; x <= width - 8; x += 8) {
71 // Load 16 pixels (32 bytes) of uint16_t data
72 80 __m256i v = _mm256_loadu_si256((const __m256i*)(src + x * 2));
73
74 // unsigned 0..65535 -> signed -32768..32767 safely
75 40 __m256i v_signed = _mm256_add_epi16(v, v_pivot16);
76
77 // Horizontal pair addition: (a - 32768) + (b - 32768) = a + b - 65536
78 40 __m256i sum32 = _mm256_madd_epi16(v_signed, ones);
79
80 // Add back 65536 to undo the bias, add 1 for rounding, then logical shift right
81 55 __m256i avg32 = _mm256_srli_epi32(_mm256_add_epi32(sum32, v_correct32), 1);
82
83 if constexpr (!full_opacity)
84 75 avg32 = simd_magic_div_32_avx2(
85 _mm256_add_epi32(_mm256_mullo_epi32(avg32, v_opacity32), v_half32),
86 25 magic.div, magic.shift);
87 40 _mm_storeu_si128((__m128i*)(dst + x), avx2_pack_u32_to_u16(avg32));
88 }
89 }
90
8/8
void fill_mask422_avx2<unsigned char, false>(unsigned char*, unsigned char const*, int, int, int, MagicDiv):
✓ Branch 56 → 53 taken 10 times.
✓ Branch 56 → 57 taken 10 times.
void fill_mask422_avx2<unsigned char, true>(unsigned char*, unsigned char const*, int, int, int, MagicDiv):
✓ Branch 46 → 45 taken 10 times.
✓ Branch 46 → 47 taken 10 times.
void fill_mask422_avx2<unsigned short, false>(unsigned short*, unsigned short const*, int, int, int, MagicDiv):
✓ Branch 62 → 59 taken 25 times.
✓ Branch 62 → 63 taken 15 times.
void fill_mask422_avx2<unsigned short, true>(unsigned short*, unsigned short const*, int, int, int, MagicDiv):
✓ Branch 42 → 41 taken 10 times.
✓ Branch 42 → 43 taken 10 times.
 100 for (; x < width; x++) {
91 55 const int avg = (src[x * 2] + src[x * 2 + 1] + 1) >> 1;
92 55 dst[x] = full_opacity ? (pixel_t)avg
93 70 : (pixel_t)magic_div_rt<pixel_t>((uint32_t)avg * (uint32_t)opacity_i + (uint32_t)half, magic);
94 }
95 45 }
96
97 // ---------------------------------------------------------------------------
98 // MASK422_MPEG2 — horizontal 3-tap triangle filter with sliding window carry.
99 // avg[x] = (left + 2*src[x*2] + src[x*2+1] + 2) >> 2
100 // ---------------------------------------------------------------------------
101 template<typename pixel_t, bool full_opacity>
102 #if defined(GCC) || defined(CLANG)
103 __attribute__((__target__("avx2")))
104 #endif
105 25 static void fill_mask422_mpeg2_avx2(
106 pixel_t* dst, const pixel_t* src, int width,
107 int opacity_i, int half, MagicDiv magic)
108 {
109 25 int x = 0;
110 if constexpr (sizeof(pixel_t) == 1) {
111 15 [[maybe_unused]] const __m256i v_opacity = _mm256_set1_epi16((short)opacity_i);
112 15 [[maybe_unused]] const __m256i v_half16 = _mm256_set1_epi16((short)half);
113 30 [[maybe_unused]] const __m256i v_mdiv = _mm256_set1_epi16((short)magic.div);
114 15 const __m256i mask_lo = _mm256_set1_epi16(0x00FF);
115 30 __m256i prev_carry = _mm256_castsi128_si256(
116 30 _mm_insert_epi16(_mm_setzero_si128(), src[0], 7));
117
118
4/4
void fill_mask422_mpeg2_avx2<unsigned char, false>(unsigned char*, unsigned char const*, int, int, int, MagicDiv):
✓ Branch 63 → 23 taken 5 times.
✓ Branch 63 → 64 taken 10 times.
void fill_mask422_mpeg2_avx2<unsigned char, true>(unsigned char*, unsigned char const*, int, int, int, MagicDiv):
✓ Branch 55 → 23 taken 5 times.
✓ Branch 55 → 56 taken 5 times.
 25 for (; x <= width - 16; x += 16) {
119 20 __m256i v = _mm256_loadu_si256((const __m256i*)(src + x * 2));
120 10 __m256i even = _mm256_and_si256(v, mask_lo);
121 10 __m256i odd = _mm256_srli_epi16(v, 8);
122
123 10 __m256i shifted_odd = _mm256_permute2x128_si256(odd, prev_carry, 0x02);
124 10 __m256i left = _mm256_alignr_epi8(odd, shifted_odd, 14);
125
126 55 __m256i res = _mm256_srli_epi16(
127 _mm256_add_epi16(
128 _mm256_add_epi16(_mm256_add_epi16(left, _mm256_slli_epi16(even, 1)), odd),
129 _mm256_set1_epi16(2)), 2);
130
131 if constexpr (!full_opacity) {
132 10 __m256i scaled = _mm256_add_epi16(_mm256_mullo_epi16(res, v_opacity), v_half16);
133 15 res = _mm256_srli_epi16(_mm256_mulhi_epu16(scaled, v_mdiv), magic.shift);
134 }
135 10 _mm_storeu_si128((__m128i*)(dst + x), avx2_pack_u16_to_u8(res));
136
137 10 prev_carry = _mm256_castsi128_si256(
138 20 _mm_insert_epi16(_mm_setzero_si128(),
139 _mm_extract_epi16(_mm256_extracti128_si256(odd, 1), 7), 7));
140 }
141 15 int right_val = _mm_extract_epi16(_mm256_castsi256_si128(prev_carry), 7);
142
4/4
void fill_mask422_mpeg2_avx2<unsigned char, false>(unsigned char*, unsigned char const*, int, int, int, MagicDiv):
✓ Branch 70 → 67 taken 25 times.
✓ Branch 70 → 71 taken 10 times.
void fill_mask422_mpeg2_avx2<unsigned char, true>(unsigned char*, unsigned char const*, int, int, int, MagicDiv):
✓ Branch 60 → 59 taken 5 times.
✓ Branch 60 → 61 taken 5 times.
 45 for (; x < width; x++) {
143 30 const int left = right_val;
144 30 const int mid = src[x * 2];
145 30 right_val = src[x * 2 + 1];
146 30 const int avg = (left + 2 * mid + right_val + 2) >> 2;
147 30 dst[x] = full_opacity ? (pixel_t)avg
148 50 : (pixel_t)magic_div_rt<pixel_t>((uint32_t)avg * (uint32_t)opacity_i + (uint32_t)half, magic);
149 }
150 } else {
151 10 [[maybe_unused]] const __m256i v_opacity32 = _mm256_set1_epi32(opacity_i);
152 10 [[maybe_unused]] const __m256i v_half32 = _mm256_set1_epi32(half);
153 20 __m256i prev_carry = _mm256_castsi128_si256(
154 20 _mm_insert_epi32(_mm_setzero_si128(), src[0], 3));
155
156
4/4
void fill_mask422_mpeg2_avx2<unsigned short, false>(unsigned short*, unsigned short const*, int, int, int, MagicDiv):
✓ Branch 69 → 11 taken 5 times.
✓ Branch 69 → 70 taken 5 times.
void fill_mask422_mpeg2_avx2<unsigned short, true>(unsigned short*, unsigned short const*, int, int, int, MagicDiv):
✓ Branch 51 → 11 taken 5 times.
✓ Branch 51 → 52 taken 5 times.
 20 for (; x <= width - 8; x += 8) {
157 20 __m256i v = _mm256_loadu_si256((const __m256i*)(src + x * 2));
158 10 __m256i lo32 = _mm256_cvtepu16_epi32(_mm256_castsi256_si128(v));
159 10 __m256i hi32 = _mm256_cvtepu16_epi32(_mm256_extracti128_si256(v, 1));
160
161 10 __m256i sh_le = _mm256_shuffle_epi32(lo32, 0x88);
162 10 __m256i sh_he = _mm256_shuffle_epi32(hi32, 0x88);
163 10 __m256i even32 = _mm256_permute4x64_epi64(
164 _mm256_unpacklo_epi64(sh_le, sh_he), _MM_SHUFFLE(3, 1, 2, 0));
165
166 10 __m256i sh_lo = _mm256_shuffle_epi32(lo32, 0xDD);
167 10 __m256i sh_ho = _mm256_shuffle_epi32(hi32, 0xDD);
168 10 __m256i odd32 = _mm256_permute4x64_epi64(
169 _mm256_unpacklo_epi64(sh_lo, sh_ho), _MM_SHUFFLE(3, 1, 2, 0));
170
171 10 __m256i shifted_odd32 = _mm256_permute2x128_si256(odd32, prev_carry, 0x02);
172 10 __m256i left = _mm256_alignr_epi8(odd32, shifted_odd32, 12);
173
174 55 __m256i res = _mm256_srli_epi32(
175 _mm256_add_epi32(
176 _mm256_add_epi32(_mm256_add_epi32(left, _mm256_slli_epi32(even32, 1)), odd32),
177 _mm256_set1_epi32(2)), 2);
178
179 if constexpr (!full_opacity)
180 15 res = simd_magic_div_32_avx2(
181 _mm256_add_epi32(_mm256_mullo_epi32(res, v_opacity32), v_half32),
182 5 magic.div, magic.shift);
183
184 10 _mm_storeu_si128((__m128i*)(dst + x), avx2_pack_u32_to_u16(res));
185
186 10 prev_carry = _mm256_castsi128_si256(
187 20 _mm_insert_epi32(_mm_setzero_si128(),
188 _mm_extract_epi32(_mm256_extracti128_si256(odd32, 1), 3), 3));
189 }
190 10 int right_val = _mm_extract_epi32(_mm256_castsi256_si128(prev_carry), 3);
191
4/4
void fill_mask422_mpeg2_avx2<unsigned short, false>(unsigned short*, unsigned short const*, int, int, int, MagicDiv):
✓ Branch 76 → 73 taken 5 times.
✓ Branch 76 → 77 taken 5 times.
void fill_mask422_mpeg2_avx2<unsigned short, true>(unsigned short*, unsigned short const*, int, int, int, MagicDiv):
✓ Branch 56 → 55 taken 5 times.
✓ Branch 56 → 57 taken 5 times.
 20 for (; x < width; x++) {
192 10 const int left = right_val;
193 10 const int mid = src[x * 2];
194 10 right_val = src[x * 2 + 1];
195 10 const int avg = (left + 2 * mid + right_val + 2) >> 2;
196 10 dst[x] = full_opacity ? (pixel_t)avg
197 10 : (pixel_t)magic_div_rt<pixel_t>((uint32_t)avg * (uint32_t)opacity_i + (uint32_t)half, magic);
198 }
199 }
200 25 }
201
202 // ---------------------------------------------------------------------------
203 // MASK420 — 2x2 box average (MPEG-1 placement).
204 // avg[x] = (row0[x*2]+row0[x*2+1]+row1[x*2]+row1[x*2+1]+2) >> 2
205 // ---------------------------------------------------------------------------
206 template<typename pixel_t, bool full_opacity>
207 #if defined(GCC) || defined(CLANG)
208 __attribute__((__target__("avx2")))
209 #endif
210 49 static void fill_mask420_avx2(
211 pixel_t* dst, const pixel_t* row0, int mask_pitch, int width,
212 int opacity_i, int half, MagicDiv magic)
213 {
214 49 const pixel_t* row1 = row0 + mask_pitch;
215 49 int x = 0;
216 if constexpr (sizeof(pixel_t) == 1) {
217 29 [[maybe_unused]] const __m256i v_opacity = _mm256_set1_epi16((short)opacity_i);
218 29 [[maybe_unused]] const __m256i v_half16 = _mm256_set1_epi16((short)half);
219 58 [[maybe_unused]] const __m256i v_mdiv = _mm256_set1_epi16((short)magic.div);
220 29 const __m256i mask_lo = _mm256_set1_epi16(0x00FF);
221
4/4
void fill_mask420_avx2<unsigned char, false>(unsigned char*, unsigned char const*, int, int, int, int, MagicDiv):
✓ Branch 61 → 19 taken 20 times.
✓ Branch 61 → 62 taken 19 times.
void fill_mask420_avx2<unsigned char, true>(unsigned char*, unsigned char const*, int, int, int, int, MagicDiv):
✓ Branch 53 → 19 taken 10 times.
✓ Branch 53 → 54 taken 10 times.
 59 for (; x <= width - 16; x += 16) {
222 30 __m256i r0 = _mm256_loadu_si256((const __m256i*)(row0 + x * 2));
223 60 __m256i r1 = _mm256_loadu_si256((const __m256i*)(row1 + x * 2));
224 30 __m256i e0 = _mm256_and_si256(r0, mask_lo);
225 30 __m256i o0 = _mm256_srli_epi16(r0, 8);
226 30 __m256i e1 = _mm256_and_si256(r1, mask_lo);
227 30 __m256i o1 = _mm256_srli_epi16(r1, 8);
228 160 __m256i avg = _mm256_srli_epi16(
229 _mm256_add_epi16(
230 _mm256_add_epi16(_mm256_add_epi16(e0, o0), _mm256_add_epi16(e1, o1)),
231 _mm256_set1_epi16(2)), 2);
232 if constexpr (!full_opacity) {
233 40 __m256i scaled = _mm256_add_epi16(_mm256_mullo_epi16(avg, v_opacity), v_half16);
234 60 avg = _mm256_srli_epi16(_mm256_mulhi_epu16(scaled, v_mdiv), magic.shift);
235 }
236 30 _mm_storeu_si128((__m128i*)(dst + x), avx2_pack_u16_to_u8(avg));
237 }
238 } else {
239 // uint16_t: unsigned widening to avoid signed overflow in madd_epi16
240 20 [[maybe_unused]] const __m256i v_opacity32 = _mm256_set1_epi32(opacity_i);
241 20 [[maybe_unused]] const __m256i v_half32 = _mm256_set1_epi32(half);
242
4/4
void fill_mask420_avx2<unsigned short, false>(unsigned short*, unsigned short const*, int, int, int, int, MagicDiv):
✓ Branch 69 → 7 taken 15 times.
✓ Branch 69 → 70 taken 10 times.
void fill_mask420_avx2<unsigned short, true>(unsigned short*, unsigned short const*, int, int, int, int, MagicDiv):
✓ Branch 51 → 7 taken 15 times.
✓ Branch 51 → 52 taken 10 times.
 50 for (; x <= width - 8; x += 8) {
243 30 __m256i v0 = _mm256_loadu_si256((const __m256i*)(row0 + x * 2));
244 60 __m256i v1 = _mm256_loadu_si256((const __m256i*)(row1 + x * 2));
245 30 __m256i r0_lo = _mm256_cvtepu16_epi32(_mm256_castsi256_si128(v0));
246 60 __m256i r0_hi = _mm256_cvtepu16_epi32(_mm256_extracti128_si256(v0, 1));
247 30 __m256i r1_lo = _mm256_cvtepu16_epi32(_mm256_castsi256_si128(v1));
248 60 __m256i r1_hi = _mm256_cvtepu16_epi32(_mm256_extracti128_si256(v1, 1));
249 30 __m256i sum_lo = _mm256_add_epi32(r0_lo, r1_lo);
250 30 __m256i sum_hi = _mm256_add_epi32(r0_hi, r1_hi);
251 30 __m256i sh_le = _mm256_shuffle_epi32(sum_lo, 0x88);
252 30 __m256i sh_he = _mm256_shuffle_epi32(sum_hi, 0x88);
253 30 __m256i even32 = _mm256_permute4x64_epi64(
254 _mm256_unpacklo_epi64(sh_le, sh_he), _MM_SHUFFLE(3, 1, 2, 0));
255 30 __m256i sh_lo = _mm256_shuffle_epi32(sum_lo, 0xDD);
256 30 __m256i sh_ho = _mm256_shuffle_epi32(sum_hi, 0xDD);
257 30 __m256i odd32 = _mm256_permute4x64_epi64(
258 _mm256_unpacklo_epi64(sh_lo, sh_ho), _MM_SHUFFLE(3, 1, 2, 0));
259 105 __m256i avg32 = _mm256_srli_epi32(
260 _mm256_add_epi32(_mm256_add_epi32(even32, odd32), _mm256_set1_epi32(2)), 2);
261 if constexpr (!full_opacity)
262 45 avg32 = simd_magic_div_32_avx2(
263 _mm256_add_epi32(_mm256_mullo_epi32(avg32, v_opacity32), v_half32),
264 15 magic.div, magic.shift);
265 30 _mm_storeu_si128((__m128i*)(dst + x), avx2_pack_u32_to_u16(avg32));
266 }
267 }
268
8/8
void fill_mask420_avx2<unsigned char, false>(unsigned char*, unsigned char const*, int, int, int, int, MagicDiv):
✓ Branch 66 → 63 taken 51 times.
✓ Branch 66 → 67 taken 19 times.
void fill_mask420_avx2<unsigned char, true>(unsigned char*, unsigned char const*, int, int, int, int, MagicDiv):
✓ Branch 56 → 55 taken 10 times.
✓ Branch 56 → 57 taken 10 times.
void fill_mask420_avx2<unsigned short, false>(unsigned short*, unsigned short const*, int, int, int, int, MagicDiv):
✓ Branch 74 → 71 taken 10 times.
✓ Branch 74 → 75 taken 10 times.
void fill_mask420_avx2<unsigned short, true>(unsigned short*, unsigned short const*, int, int, int, int, MagicDiv):
✓ Branch 54 → 53 taken 10 times.
✓ Branch 54 → 55 taken 10 times.
 130 for (; x < width; x++) {
269 81 const int avg = ((int)row0[x*2] + row0[x*2+1] + row1[x*2] + row1[x*2+1] + 2) >> 2;
270 81 dst[x] = full_opacity ? (pixel_t)avg
271 122 : (pixel_t)magic_div_rt<pixel_t>((uint32_t)avg * (uint32_t)opacity_i + (uint32_t)half, magic);
272 }
273 49 }
274
275 // ---------------------------------------------------------------------------
276 // MASK420_MPEG2 — horizontal 3-tap triangle filter with vertical 2-row sum and
277 // sliding-window carry. Filter:
278 // pe[x] = row0[x*2] + row1[x*2] (vertical sum of even-indexed pairs)
279 // po[x] = row0[x*2+1] + row1[x*2+1] (vertical sum of odd-indexed pairs)
280 // avg[x] = (po[x-1] + 2*pe[x] + po[x] + 4) >> 3
281 // uint8_t: 16 output pixels / iteration (32-byte load per row → pe/po as uint16)
282 // uint16_t: 8 output pixels / iteration (32-byte load per row → pe/po as uint32)
283 // Cross-lane carry: 1 element = 14-byte alignr (uint8_t), 12-byte (uint16_t).
284 // ---------------------------------------------------------------------------
285 template<typename pixel_t, bool full_opacity>
286 #if defined(GCC) || defined(CLANG)
287 __attribute__((__target__("avx2")))
288 #endif
289 31 static void fill_mask420_mpeg2_avx2(
290 pixel_t* dst, const pixel_t* row0, int mask_pitch, int width,
291 int opacity_i, int half, MagicDiv magic)
292 {
293 31 const pixel_t* row1 = row0 + mask_pitch;
294 31 int x = 0;
295 if constexpr (sizeof(pixel_t) == 1) {
296 21 [[maybe_unused]] const __m256i v_opacity = _mm256_set1_epi16((short)opacity_i);
297 21 [[maybe_unused]] const __m256i v_half16 = _mm256_set1_epi16((short)half);
298 42 [[maybe_unused]] const __m256i v_mdiv = _mm256_set1_epi16((short)magic.div);
299 21 const __m256i mask_lo = _mm256_set1_epi16(0x00FF);
300 21 const int p0 = (int)row0[0] + row1[0];
301 42 __m256i prev_carry = _mm256_castsi128_si256(
302 42 _mm_insert_epi16(_mm_setzero_si128(), p0, 7));
303
304
4/4
void fill_mask420_mpeg2_avx2<unsigned char, false>(unsigned char*, unsigned char const*, int, int, int, int, MagicDiv):
✓ Branch 73 → 23 taken 12 times.
✓ Branch 73 → 74 taken 16 times.
void fill_mask420_mpeg2_avx2<unsigned char, true>(unsigned char*, unsigned char const*, int, int, int, int, MagicDiv):
✓ Branch 65 → 23 taken 5 times.
✓ Branch 65 → 66 taken 5 times.
 38 for (; x <= width - 16; x += 16) {
305 17 __m256i r0 = _mm256_loadu_si256((const __m256i*)(row0 + x * 2));
306 34 __m256i r1 = _mm256_loadu_si256((const __m256i*)(row1 + x * 2));
307 51 __m256i pe = _mm256_add_epi16(_mm256_and_si256(r0, mask_lo),
308 _mm256_and_si256(r1, mask_lo));
309 34 __m256i po = _mm256_add_epi16(_mm256_srli_epi16(r0, 8),
310 _mm256_srli_epi16(r1, 8));
311
312 17 __m256i shifted_odd = _mm256_permute2x128_si256(po, prev_carry, 0x02);
313 17 __m256i left = _mm256_alignr_epi8(po, shifted_odd, 14);
314
315 90 __m256i res = _mm256_srli_epi16(
316 _mm256_add_epi16(
317 _mm256_add_epi16(_mm256_add_epi16(left, _mm256_slli_epi16(pe, 1)), po),
318 _mm256_set1_epi16(4)), 3);
319
320 if constexpr (!full_opacity) {
321 24 __m256i scaled = _mm256_add_epi16(_mm256_mullo_epi16(res, v_opacity), v_half16);
322 36 res = _mm256_srli_epi16(_mm256_mulhi_epu16(scaled, v_mdiv), magic.shift);
323 }
324 17 _mm_storeu_si128((__m128i*)(dst + x), avx2_pack_u16_to_u8(res));
325
326 17 prev_carry = _mm256_castsi128_si256(
327 34 _mm_insert_epi16(_mm_setzero_si128(),
328 _mm_extract_epi16(_mm256_extracti128_si256(po, 1), 7), 7));
329 }
330 21 int right_val = _mm_extract_epi16(_mm256_castsi256_si128(prev_carry), 7);
331
4/4
void fill_mask420_mpeg2_avx2<unsigned char, false>(unsigned char*, unsigned char const*, int, int, int, int, MagicDiv):
✓ Branch 80 → 77 taken 70 times.
✓ Branch 80 → 81 taken 16 times.
void fill_mask420_mpeg2_avx2<unsigned char, true>(unsigned char*, unsigned char const*, int, int, int, int, MagicDiv):
✓ Branch 70 → 69 taken 5 times.
✓ Branch 70 → 71 taken 5 times.
 96 for (; x < width; x++) {
332 75 const int left = right_val;
333 75 const int mid = (int)row0[x*2] + row1[x*2];
334 75 right_val = (int)row0[x*2+1] + row1[x*2+1];
335 75 const int avg = (left + 2 * mid + right_val + 4) >> 3;
336 75 dst[x] = full_opacity ? (pixel_t)avg
337 140 : (pixel_t)magic_div_rt<pixel_t>((uint32_t)avg * (uint32_t)opacity_i + (uint32_t)half, magic);
338 }
339 } else {
340 10 [[maybe_unused]] const __m256i v_opacity32 = _mm256_set1_epi32(opacity_i);
341 10 [[maybe_unused]] const __m256i v_half32 = _mm256_set1_epi32(half);
342 10 const int p0 = (int)row0[0] + row1[0];
343 20 __m256i prev_carry = _mm256_castsi128_si256(
344 10 _mm_insert_epi32(_mm_setzero_si128(), p0, 3));
345
346
4/4
void fill_mask420_mpeg2_avx2<unsigned short, false>(unsigned short*, unsigned short const*, int, int, int, int, MagicDiv):
✓ Branch 81 → 11 taken 5 times.
✓ Branch 81 → 82 taken 5 times.
void fill_mask420_mpeg2_avx2<unsigned short, true>(unsigned short*, unsigned short const*, int, int, int, int, MagicDiv):
✓ Branch 63 → 11 taken 5 times.
✓ Branch 63 → 64 taken 5 times.
 20 for (; x <= width - 8; x += 8) {
347 10 __m256i r0 = _mm256_loadu_si256((const __m256i*)(row0 + x * 2));
348 20 __m256i r1 = _mm256_loadu_si256((const __m256i*)(row1 + x * 2));
349 // Expand each 8-uint16 half to 8 uint32 and sum vertically.
350 40 __m256i plo32 = _mm256_add_epi32(
351 _mm256_cvtepu16_epi32(_mm256_castsi256_si128(r0)),
352 _mm256_cvtepu16_epi32(_mm256_castsi256_si128(r1)));
353 30 __m256i phi32 = _mm256_add_epi32(
354 _mm256_cvtepu16_epi32(_mm256_extracti128_si256(r0, 1)),
355 _mm256_cvtepu16_epi32(_mm256_extracti128_si256(r1, 1)));
356
357 // Deinterleave even/odd (same as fill_mask422_mpeg2_avx2 uint16_t path).
358 10 __m256i sh_le = _mm256_shuffle_epi32(plo32, 0x88);
359 10 __m256i sh_he = _mm256_shuffle_epi32(phi32, 0x88);
360 10 __m256i even32 = _mm256_permute4x64_epi64(
361 _mm256_unpacklo_epi64(sh_le, sh_he), _MM_SHUFFLE(3, 1, 2, 0));
362
363 10 __m256i sh_lo = _mm256_shuffle_epi32(plo32, 0xDD);
364 10 __m256i sh_ho = _mm256_shuffle_epi32(phi32, 0xDD);
365 10 __m256i odd32 = _mm256_permute4x64_epi64(
366 _mm256_unpacklo_epi64(sh_lo, sh_ho), _MM_SHUFFLE(3, 1, 2, 0));
367
368 10 __m256i shifted_odd32 = _mm256_permute2x128_si256(odd32, prev_carry, 0x02);
369 10 __m256i left = _mm256_alignr_epi8(odd32, shifted_odd32, 12);
370
371 55 __m256i res = _mm256_srli_epi32(
372 _mm256_add_epi32(
373 _mm256_add_epi32(_mm256_add_epi32(left, _mm256_slli_epi32(even32, 1)), odd32),
374 _mm256_set1_epi32(4)), 3);
375
376 if constexpr (!full_opacity)
377 15 res = simd_magic_div_32_avx2(
378 _mm256_add_epi32(_mm256_mullo_epi32(res, v_opacity32), v_half32),
379 5 magic.div, magic.shift);
380
381 10 _mm_storeu_si128((__m128i*)(dst + x), avx2_pack_u32_to_u16(res));
382
383 10 prev_carry = _mm256_castsi128_si256(
384 20 _mm_insert_epi32(_mm_setzero_si128(),
385 _mm_extract_epi32(_mm256_extracti128_si256(odd32, 1), 3), 3));
386 }
387 10 int right_val = _mm_extract_epi32(_mm256_castsi256_si128(prev_carry), 3);
388
4/4
void fill_mask420_mpeg2_avx2<unsigned short, false>(unsigned short*, unsigned short const*, int, int, int, int, MagicDiv):
✓ Branch 88 → 85 taken 5 times.
✓ Branch 88 → 89 taken 5 times.
void fill_mask420_mpeg2_avx2<unsigned short, true>(unsigned short*, unsigned short const*, int, int, int, int, MagicDiv):
✓ Branch 68 → 67 taken 5 times.
✓ Branch 68 → 69 taken 5 times.
 20 for (; x < width; x++) {
389 10 const int left = right_val;
390 10 const int mid = (int)row0[x*2] + row1[x*2];
391 10 right_val = (int)row0[x*2+1] + row1[x*2+1];
392 10 const int avg = (left + 2 * mid + right_val + 4) >> 3;
393 10 dst[x] = full_opacity ? (pixel_t)avg
394 10 : (pixel_t)magic_div_rt<pixel_t>((uint32_t)avg * (uint32_t)opacity_i + (uint32_t)half, magic);
395 }
396 }
397 31 }
398
399 // ---------------------------------------------------------------------------
400 // MASK422_TOPLEFT — left co-sited point sample (no averaging).
401 // dst[x] = src[x*2]
402 // ---------------------------------------------------------------------------
403 template<typename pixel_t, bool full_opacity>
404 #if defined(GCC) || defined(CLANG)
405 __attribute__((__target__("avx2")))
406 #endif
407 40 static void fill_mask422_topleft_avx2(
408 pixel_t* dst, const pixel_t* src, int width,
409 int opacity_i, int half, MagicDiv magic)
410 {
411 40 int x = 0;
412 if constexpr (sizeof(pixel_t) == 1) {
413 20 [[maybe_unused]] const __m256i v_opacity = _mm256_set1_epi16((short)opacity_i);
414 20 [[maybe_unused]] const __m256i v_half16 = _mm256_set1_epi16((short)half);
415 40 [[maybe_unused]] const __m256i v_mdiv = _mm256_set1_epi16((short)magic.div);
416 20 const __m256i mask_lo = _mm256_set1_epi16(0x00FF);
417
4/4
void fill_mask422_topleft_avx2<unsigned char, false>(unsigned char*, unsigned char const*, int, int, int, MagicDiv):
✓ Branch 39 → 19 taken 10 times.
✓ Branch 39 → 40 taken 10 times.
void fill_mask422_topleft_avx2<unsigned char, true>(unsigned char*, unsigned char const*, int, int, int, MagicDiv):
✓ Branch 31 → 19 taken 10 times.
✓ Branch 31 → 32 taken 10 times.
 40 for (; x <= width - 16; x += 16) {
418 40 __m256i v = _mm256_loadu_si256((const __m256i*)(src + x * 2));
419 20 __m256i even = _mm256_and_si256(v, mask_lo); // left (even) bytes as uint16
420 if constexpr (!full_opacity) {
421 20 __m256i scaled = _mm256_add_epi16(_mm256_mullo_epi16(even, v_opacity), v_half16);
422 30 even = _mm256_srli_epi16(_mm256_mulhi_epu16(scaled, v_mdiv), magic.shift);
423 }
424 20 _mm_storeu_si128((__m128i*)(dst + x), avx2_pack_u16_to_u8(even));
425 }
426 } else {
427 // 16-bit: grab even-indexed elements (src[x*2], src[x*2+2], ...)
428 20 [[maybe_unused]] const __m256i v_opacity32 = _mm256_set1_epi32(opacity_i);
429 20 [[maybe_unused]] const __m256i v_half32 = _mm256_set1_epi32(half);
430
4/4
void fill_mask422_topleft_avx2<unsigned short, false>(unsigned short*, unsigned short const*, int, int, int, MagicDiv):
✓ Branch 47 → 7 taken 10 times.
✓ Branch 47 → 48 taken 10 times.
void fill_mask422_topleft_avx2<unsigned short, true>(unsigned short*, unsigned short const*, int, int, int, MagicDiv):
✓ Branch 29 → 7 taken 10 times.
✓ Branch 29 → 30 taken 10 times.
 40 for (; x <= width - 8; x += 8) {
431 40 __m256i v = _mm256_loadu_si256((const __m256i*)(src + x * 2));
432 // Deinterleave: keep even-indexed uint16 elements
433 20 __m256i lo32 = _mm256_cvtepu16_epi32(_mm256_castsi256_si128(v));
434 20 __m256i hi32 = _mm256_cvtepu16_epi32(_mm256_extracti128_si256(v, 1));
435 20 __m256i sh_le = _mm256_shuffle_epi32(lo32, 0x88); // [v0,v2,_,_] in lo64 per lane
436 20 __m256i sh_he = _mm256_shuffle_epi32(hi32, 0x88);
437 20 __m256i even32 = _mm256_permute4x64_epi64(
438 _mm256_unpacklo_epi64(sh_le, sh_he), _MM_SHUFFLE(3, 1, 2, 0));
439 if constexpr (!full_opacity)
440 30 even32 = simd_magic_div_32_avx2(
441 _mm256_add_epi32(_mm256_mullo_epi32(even32, v_opacity32), v_half32),
442 10 magic.div, magic.shift);
443 20 _mm_storeu_si128((__m128i*)(dst + x), avx2_pack_u32_to_u16(even32));
444 }
445 }
446
8/8
void fill_mask422_topleft_avx2<unsigned char, false>(unsigned char*, unsigned char const*, int, int, int, MagicDiv):
✓ Branch 44 → 41 taken 10 times.
✓ Branch 44 → 45 taken 10 times.
void fill_mask422_topleft_avx2<unsigned char, true>(unsigned char*, unsigned char const*, int, int, int, MagicDiv):
✓ Branch 34 → 33 taken 10 times.
✓ Branch 34 → 35 taken 10 times.
void fill_mask422_topleft_avx2<unsigned short, false>(unsigned short*, unsigned short const*, int, int, int, MagicDiv):
✓ Branch 52 → 49 taken 10 times.
✓ Branch 52 → 53 taken 10 times.
void fill_mask422_topleft_avx2<unsigned short, true>(unsigned short*, unsigned short const*, int, int, int, MagicDiv):
✓ Branch 32 → 31 taken 10 times.
✓ Branch 32 → 33 taken 10 times.
 80 for (; x < width; x++) {
447 40 const int val = src[x * 2];
448 40 dst[x] = full_opacity ? (pixel_t)val
449 40 : (pixel_t)magic_div_rt<pixel_t>((uint32_t)val * (uint32_t)opacity_i + (uint32_t)half, magic);
450 }
451 40 }
452
453 // ---------------------------------------------------------------------------
454 // MASK420_TOPLEFT — top-left co-sited point sample (top row only, no averaging).
455 // dst[x] = row0[x*2]
456 // ---------------------------------------------------------------------------
457 template<typename pixel_t, bool full_opacity>
458 #if defined(GCC) || defined(CLANG)
459 __attribute__((__target__("avx2")))
460 #endif
461 20 static void fill_mask420_topleft_avx2(
462 pixel_t* dst, const pixel_t* row0, int /*mask_pitch*/, int width,
463 int opacity_i, int half, MagicDiv magic)
464 {
465 // Identical to fill_mask422_topleft_avx2: top row only, left co-sited.
466 20 fill_mask422_topleft_avx2<pixel_t, full_opacity>(dst, row0, width, opacity_i, half, magic);
467 20 }
468
469 // ---------------------------------------------------------------------------
470 // MASK411 — horizontal 4-tap box average.
471 // avg[x] = (src[x*4]+src[x*4+1]+src[x*4+2]+src[x*4+3]+2) >> 2
472 // ---------------------------------------------------------------------------
473 template<typename pixel_t, bool full_opacity>
474 #if defined(GCC) || defined(CLANG)
475 __attribute__((__target__("avx2")))
476 #endif
477 ✗ static void fill_mask411_avx2(
478 pixel_t* dst, const pixel_t* src, int width,
479 int opacity_i, int half, MagicDiv magic)
480 {
481 ✗ int x = 0;
482 if constexpr (sizeof(pixel_t) == 1) {
483 ✗ const __m256i zero = _mm256_setzero_si256();
484 ✗ [[maybe_unused]] const __m256i v_opacity = _mm256_set1_epi16((short)opacity_i);
485 ✗ [[maybe_unused]] const __m256i v_half16 = _mm256_set1_epi16((short)half);
486 ✗ [[maybe_unused]] const __m256i v_mdiv = _mm256_set1_epi16((short)magic.div);
487 ✗ for (; x <= width - 16; x += 16) {
488 ✗ __m256i v0 = _mm256_loadu_si256((const __m256i*)(src + x * 4));
489 ✗ __m256i v1 = _mm256_loadu_si256((const __m256i*)(src + x * 4 + 32));
490 ✗ __m256i p0 = _mm256_hadd_epi16(
491 _mm256_unpacklo_epi8(v0, zero), _mm256_unpackhi_epi8(v0, zero));
492 ✗ __m256i p1 = _mm256_hadd_epi16(
493 _mm256_unpacklo_epi8(v1, zero), _mm256_unpackhi_epi8(v1, zero));
494 ✗ __m256i avg = _mm256_srli_epi16(
495 _mm256_add_epi16(_mm256_hadd_epi16(p0, p1), _mm256_set1_epi16(2)), 2);
496 ✗ avg = _mm256_permute4x64_epi64(avg, _MM_SHUFFLE(3, 1, 2, 0));
497 if constexpr (!full_opacity) {
498 ✗ __m256i scaled = _mm256_add_epi16(_mm256_mullo_epi16(avg, v_opacity), v_half16);
499 ✗ avg = _mm256_srli_epi16(_mm256_mulhi_epu16(scaled, v_mdiv), magic.shift);
500 }
501 ✗ _mm_storeu_si128((__m128i*)(dst + x), avx2_pack_u16_to_u8(avg));
502 }
503 } else {
504 // uint16_t: unsigned widening to avoid signed overflow in madd_epi16
505 ✗ [[maybe_unused]] const __m256i v_opacity32 = _mm256_set1_epi32(opacity_i);
506 ✗ [[maybe_unused]] const __m256i v_half32 = _mm256_set1_epi32(half);
507 ✗ for (; x <= width - 8; x += 8) {
508 ✗ __m256i v0 = _mm256_loadu_si256((const __m256i*)(src + x * 4)); // s0..s15
509 ✗ __m256i v1 = _mm256_loadu_si256((const __m256i*)(src + x * 4 + 16)); // s16..s31
510 ✗ __m256i lo0 = _mm256_cvtepu16_epi32(_mm256_castsi256_si128(v0));
511 ✗ __m256i hi0 = _mm256_cvtepu16_epi32(_mm256_extracti128_si256(v0, 1));
512 ✗ __m256i lo1 = _mm256_cvtepu16_epi32(_mm256_castsi256_si128(v1));
513 ✗ __m256i hi1 = _mm256_cvtepu16_epi32(_mm256_extracti128_si256(v1, 1));
514 ✗ __m256i pair0 = _mm256_hadd_epi32(lo0, hi0); // lo:[s0+s1,s2+s3,s8+s9,s10+s11] hi:[s4+s5,s6+s7,s12+s13,s14+s15]
515 ✗ __m256i pair1 = _mm256_hadd_epi32(lo1, hi1);
516 ✗ __m256i quad0 = _mm256_hadd_epi32(pair0, pair0); // lo:[G0,G2,G0,G2] hi:[G1,G3,G1,G3]
517 ✗ __m256i quad1 = _mm256_hadd_epi32(pair1, pair1); // lo:[G4,G6,G4,G6] hi:[G5,G7,G5,G7]
518 ✗ __m128i g0123 = _mm_unpacklo_epi32(_mm256_castsi256_si128(quad0),
519 _mm256_extracti128_si256(quad0, 1)); // [G0,G1,G2,G3]
520 ✗ __m128i g4567 = _mm_unpacklo_epi32(_mm256_castsi256_si128(quad1),
521 _mm256_extracti128_si256(quad1, 1)); // [G4,G5,G6,G7]
522 ✗ __m256i avg32 = _mm256_srli_epi32(
523 _mm256_add_epi32(_mm256_set_m128i(g4567, g0123), _mm256_set1_epi32(2)), 2);
524 if constexpr (!full_opacity)
525 ✗ avg32 = simd_magic_div_32_avx2(
526 _mm256_add_epi32(_mm256_mullo_epi32(avg32, v_opacity32), v_half32),
527 ✗ magic.div, magic.shift);
528 ✗ _mm_storeu_si128((__m128i*)(dst + x), avx2_pack_u32_to_u16(avg32));
529 }
530 }
531 ✗ for (; x < width; x++) {
532 ✗ const int avg = ((int)src[x*4] + src[x*4+1] + src[x*4+2] + src[x*4+3] + 2) >> 2;
533 ✗ dst[x] = full_opacity ? (pixel_t)avg
534 ✗ : (pixel_t)magic_div_rt<pixel_t>((uint32_t)avg * (uint32_t)opacity_i + (uint32_t)half, magic);
535 }
536 ✗ }
537 // ---------------------------
538 // End of integer mask helpers
539 // ---------------------------
540
541 // ---------------------------
542 // Start of float mask helpers
543 // ---------------------------
544
545 // MASK422 — horizontal 2-tap box average.
546 // avg[x] = (src[x*2] + src[x*2+1]) * 0.5f
547 //
548 // float: 8 output pixels / iteration (16 input floats loaded)
549 // ---------------------------------------------------------------------------
550 template<bool full_opacity>
551 #if defined(GCC) || defined(CLANG)
552 __attribute__((__target__("avx2")))
553 #endif
554 10 static void fill_mask422_float_avx2(
555 float* dst, const float* src, int width, float opacity)
556 {
557 10 int x = 0;
558 10 const __m256 v_opacity = _mm256_set1_ps(opacity);
559 10 const __m256 v_05 = _mm256_set1_ps(0.5f);
560
561
4/4
void fill_mask422_float_avx2<false>(float*, float const*, int, float):
✓ Branch 27 → 7 taken 10 times.
✓ Branch 27 → 28 taken 5 times.
void fill_mask422_float_avx2<true>(float*, float const*, int, float):
✓ Branch 25 → 7 taken 10 times.
✓ Branch 25 → 26 taken 5 times.
 30 for (; x <= width - 8; x += 8) {
562 // 1. Load 16 floats (two 256-bit registers)
563 20 __m256 r_lo = _mm256_loadu_ps(src + x * 2);
564 20 __m256 r_hi = _mm256_loadu_ps(src + x * 2 + 8);
565
566 // 2. De-interleave Even and Odd samples
567 // r_lo: [s0, s1, s2, s3, s4, s5, s6, s7]
568 // r_hi: [s8, s9, s10, s11, s12, s13, s14, s15]
569 20 __m256 even_shuf = _mm256_shuffle_ps(r_lo, r_hi, _MM_SHUFFLE(2, 0, 2, 0));
570 20 __m256 odd_shuf = _mm256_shuffle_ps(r_lo, r_hi, _MM_SHUFFLE(3, 1, 3, 1));
571
572 // Fix lane crossing to get contiguous even and odd vectors
573 40 __m256 even = _mm256_castsi256_ps(
574 _mm256_permute4x64_epi64(_mm256_castps_si256(even_shuf), _MM_SHUFFLE(3, 1, 2, 0))
575 );
576 40 __m256 odd = _mm256_castsi256_ps(
577 _mm256_permute4x64_epi64(_mm256_castps_si256(odd_shuf), _MM_SHUFFLE(3, 1, 2, 0))
578 );
579
580 // 3. Average: (even + odd) * 0.5
581 30 __m256 avg = _mm256_mul_ps(_mm256_add_ps(even, odd), v_05);
582
583 if constexpr (!full_opacity) {
584 10 avg = _mm256_mul_ps(avg, v_opacity);
585 }
586
587 // 4. Store 8 output pixels
588 20 _mm256_storeu_ps(dst + x, avg);
589 }
590
591 // Scalar Tail
592
4/4
void fill_mask422_float_avx2<false>(float*, float const*, int, float):
✓ Branch 30 → 29 taken 5 times.
✓ Branch 30 → 31 taken 5 times.
void fill_mask422_float_avx2<true>(float*, float const*, int, float):
✓ Branch 28 → 27 taken 5 times.
✓ Branch 28 → 29 taken 5 times.
 20 for (; x < width; x++) {
593 10 const float avg = (src[x * 2] + src[x * 2 + 1]) * 0.5f;
594 10 dst[x] = full_opacity ? avg : avg * opacity;
595 }
596 10 }
597
598 // ---------------------------------------------------------------------------
599 // MASK422_MPEG2 — horizontal 3-tap triangle filter with sliding window carry.
600 // avg[x] = (left + 2*src[x*2] + src[x*2+1]) * 0.25f
601 //
602 // float: 8 output pixels / iteration (16 input floats per iteration)
603 // Cross-lane carry: 1 element = 12-byte alignr (4 bytes per float).
604 // ---------------------------------------------------------------------------
605 template<bool full_opacity>
606 #if defined(GCC) || defined(CLANG)
607 __attribute__((__target__("avx2")))
608 #endif
609 ✗ static void fill_mask422_mpeg2_float_avx2(
610 float* dst, const float* src, int width, float opacity)
611 {
612 ✗ int x = 0;
613
614 // Seed the carry (left neighbor) with the first even sample
615 // or as per your boundary requirements.
616 ✗ float right_val = src[0];
617
618 ✗ const __m256 v_opacity = _mm256_set1_ps(opacity);
619 ✗ const __m256 v_025 = _mm256_set1_ps(0.25f);
620
621 // v_prev_carry holds the last 'odd' sample from the previous vector
622 ✗ __m256 v_prev_carry = _mm256_set1_ps(right_val);
623
624 ✗ for (; x <= width - 8; x += 8) {
625 // 1. Load 16 floats (one full source block for 8 output pixels)
626 ✗ __m256 v0 = _mm256_loadu_ps(src + x * 2);
627
628 // 2. De-interleave: Even (src[x*2]) and Odd (src[x*2+1])
629 // s0, s1, s2, s3, s4, s5, s6, s7 | s8, s9, s10, s11, s12, s13, s14, s15
630 ✗ __m256 even = _mm256_permutevar8x32_ps(v0, _mm256_set_epi32(14, 12, 10, 8, 6, 4, 2, 0));
631 ✗ __m256 odd = _mm256_permutevar8x32_ps(v0, _mm256_set_epi32(15, 13, 11, 9, 7, 5, 3, 1));
632
633 // 3. Sliding Window: Construct 'left' neighbor vector
634 // shifted_odd: [prev_o4..o7, o0..o3] - brings previous high into current low
635 ✗ __m256 shifted_odd = _mm256_permute2f128_ps(odd, v_prev_carry, 0x02);
636
637 // left: [prev_o7, o0, o1, o2 | o3, o4, o5, o6]
638 // alignr by 12 bytes shifts the 128-bit window by 3 floats (leaving 1 float carry)
639 ✗ __m256 left = _mm256_castsi256_ps(_mm256_alignr_epi8(
640 _mm256_castps_si256(odd),
641 _mm256_castps_si256(shifted_odd), 12));
642
643 // 4. Filter: avg = (left + 2*even + odd) * 0.25
644 ✗ __m256 avg = _mm256_add_ps(left, _mm256_add_ps(_mm256_add_ps(even, even), odd));
645 ✗ avg = _mm256_mul_ps(avg, v_025);
646
647 if constexpr (!full_opacity) {
648 ✗ avg = _mm256_mul_ps(avg, v_opacity);
649 }
650
651 ✗ _mm256_storeu_ps(dst + x, avg);
652
653 // 5. Carry the last element of 'odd' (index 7) for the next iteration
654 ✗ v_prev_carry = _mm256_permutevar8x32_ps(odd, _mm256_set1_epi32(7));
655 }
656
657 // Extraction for scalar tail
658 ✗ right_val = _mm256_cvtss_f32(v_prev_carry);
659
660 ✗ for (; x < width; x++) {
661 ✗ const float left = right_val;
662 ✗ const float mid = src[x * 2];
663 ✗ right_val = src[x * 2 + 1];
664
665 ✗ const float avg = (left + 2.0f * mid + right_val) * 0.25f;
666 ✗ dst[x] = full_opacity ? avg : avg * opacity;
667 }
668 ✗ }
669
670 // ---------------------------------------------------------------------------
671 // MASK420 — 2x2 box average (MPEG-1 placement).
672 // avg[x] = (row0[x*2] + row0[x*2+1] + row1[x*2] + row1[x*2+1]) * 0.25f
673 //
674 // float: 8 output pixels / iteration (requires 16 input floats per row)
675 // ---------------------------------------------------------------------------
676 template<bool full_opacity>
677 #if defined(GCC) || defined(CLANG)
678 __attribute__((__target__("avx2")))
679 #endif
680 10 static void fill_mask420_float_avx2(
681 float* dst, const float* row0, int mask_pitch, int width, float opacity)
682 {
683 10 const float* row1 = row0 + mask_pitch;
684 10 int x = 0;
685
686 10 const __m256 v_opacity = _mm256_set1_ps(opacity);
687 10 const __m256 v_025 = _mm256_set1_ps(0.25f);
688
689
4/4
void fill_mask420_float_avx2<false>(float*, float const*, int, int, float):
✓ Branch 35 → 7 taken 10 times.
✓ Branch 35 → 36 taken 5 times.
void fill_mask420_float_avx2<true>(float*, float const*, int, int, float):
✓ Branch 33 → 7 taken 10 times.
✓ Branch 33 → 34 taken 5 times.
 30 for (; x <= width - 8; x += 8) {
690 // 1. Load 16 floats from each row (total 32 floats per iteration)
691 20 __m256 r0_lo = _mm256_loadu_ps(row0 + x * 2);
692 20 __m256 r0_hi = _mm256_loadu_ps(row0 + x * 2 + 8);
693 20 __m256 r1_lo = _mm256_loadu_ps(row1 + x * 2);
694 40 __m256 r1_hi = _mm256_loadu_ps(row1 + x * 2 + 8);
695
696 // 2. Vertical Sum: Interleaved [e0, o0, e1, o1, e2, o2, e3, o3...]
697 20 __m256 s_lo = _mm256_add_ps(r0_lo, r1_lo);
698 20 __m256 s_hi = _mm256_add_ps(r0_hi, r1_hi);
699
700 // 3. Horizontal Sum via De-interleave
701 // We shuffle 's_lo' and 's_hi' to separate even and odd indices
702 20 __m256 even_shuf = _mm256_shuffle_ps(s_lo, s_hi, _MM_SHUFFLE(2, 0, 2, 0));
703 20 __m256 odd_shuf = _mm256_shuffle_ps(s_lo, s_hi, _MM_SHUFFLE(3, 1, 3, 1));
704
705 // Fix AVX2 lane crossing: [L_even, H_even] and [L_odd, H_odd]
706 40 __m256 even = _mm256_castsi256_ps(
707 _mm256_permute4x64_epi64(_mm256_castps_si256(even_shuf), _MM_SHUFFLE(3, 1, 2, 0))
708 );
709 40 __m256 odd = _mm256_castsi256_ps(
710 _mm256_permute4x64_epi64(_mm256_castps_si256(odd_shuf), _MM_SHUFFLE(3, 1, 2, 0))
711 );
712
713 // 4. Final Average: (even + odd) * 0.25
714 30 __m256 avg = _mm256_mul_ps(_mm256_add_ps(even, odd), v_025);
715
716 if constexpr (!full_opacity) {
717 10 avg = _mm256_mul_ps(avg, v_opacity);
718 }
719
720 20 _mm256_storeu_ps(dst + x, avg);
721 }
722
723 // Scalar Tail
724
4/4
void fill_mask420_float_avx2<false>(float*, float const*, int, int, float):
✓ Branch 38 → 37 taken 5 times.
✓ Branch 38 → 39 taken 5 times.
void fill_mask420_float_avx2<true>(float*, float const*, int, int, float):
✓ Branch 36 → 35 taken 5 times.
✓ Branch 36 → 37 taken 5 times.
 20 for (; x < width; x++) {
725 10 const float sum = row0[x * 2] + row0[x * 2 + 1] +
726 10 row1[x * 2] + row1[x * 2 + 1];
727 10 const float avg = sum * 0.25f;
728 10 dst[x] = full_opacity ? avg : avg * opacity;
729 }
730 10 }
731
732 // ---------------------------------------------------------------------------
733 // MASK420_MPEG2 — horizontal 3-tap triangle filter with vertical 2-row sum and
734 // sliding-window carry. Filter logic:
735 // pe[x] = row0[x*2] + row1[x*2] (vertical sum of even-indexed pairs)
736 // po[x] = row0[x*2+1] + row1[x*2+1] (vertical sum of odd-indexed pairs)
737 // avg[x] = (po[x-1] + 2*pe[x] + po[x]) * 0.125f
738 //
739 // float: 8 output pixels / iteration (32-byte load per row -> 16 floats).
740 // Cross-lane carry: 1 element = 12-byte alignr (4 bytes per float).
741 // ---------------------------------------------------------------------------
742 template<bool full_opacity>
743 #if defined(GCC) || defined(CLANG)
744 __attribute__((__target__("avx2")))
745 #endif
746 ✗ static void fill_mask420_mpeg2_float_avx2(
747 float* dst, const float* row0, int mask_pitch, int width, float opacity)
748 {
749 ✗ const float* row1 = row0 + mask_pitch;
750 ✗ int x = 0;
751
752 // We need the "odd" sum from the pixel immediately before the SIMD block.
753 // If x=0, this is technically out of bounds or needs a padding strategy.
754 // Assuming p0 is the vertical sum at index -1 or starting logic:
755 ✗ float right_val = row0[0] + row1[0];
756 // Broadcast the last 'odd' sum into the carry register
757 // We only need the very last element of the previous 'odd' vector.
758 ✗ __m256 v_prev_odd = _mm256_set1_ps(right_val);
759 ✗ const __m256 v_opacity = _mm256_set1_ps(opacity);
760
761 ✗ for (; x <= width - 8; x += 8) {
762 // 1. Load 16 floats from each row (2x 256-bit loads)
763 ✗ __m256 r0_lo = _mm256_loadu_ps(row0 + x * 2);
764 ✗ __m256 r0_hi = _mm256_loadu_ps(row0 + x * 2 + 8);
765 ✗ __m256 r1_lo = _mm256_loadu_ps(row1 + x * 2);
766 ✗ __m256 r1_hi = _mm256_loadu_ps(row1 + x * 2 + 8);
767
768 // 2. Vertical Sum (pe and po interleaved: [e0, o0, e1, o1...])
769 ✗ __m256 sum_lo = _mm256_add_ps(r0_lo, r1_lo);
770 ✗ __m256 sum_hi = _mm256_add_ps(r0_hi, r1_hi);
771
772 // 3. De-interleave Even and Odd
773 // Use shuffle to get [e0, e1, e2, e3, e4, e5, e6, e7] and [o0, o1...]
774 // Note: _mm256_shuffle_ps works within 128-bit lanes, so we fix it with a permute.
775 ✗ __m256 even_shuf = _mm256_shuffle_ps(sum_lo, sum_hi, _MM_SHUFFLE(2, 0, 2, 0));
776 ✗ __m256 odd_shuf = _mm256_shuffle_ps(sum_lo, sum_hi, _MM_SHUFFLE(3, 1, 3, 1));
777
778 // Fix lane crossing: [L0, L1, H0, H1] -> [L0, H0, L1, H1]
779 ✗ __m256 even = _mm256_castsi256_ps(
780 _mm256_permute4x64_epi64(_mm256_castps_si256(even_shuf), _MM_SHUFFLE(3, 1, 2, 0))
781 );
782 ✗ __m256 odd = _mm256_castsi256_ps(
783 _mm256_permute4x64_epi64(_mm256_castps_si256(odd_shuf), _MM_SHUFFLE(3, 1, 2, 0))
784 );
785
786 // 4. Construct 'left' vector: [prev_o7, o0, o1, o2, o3, o4, o5, o6]
787 // Slide 'odd' right by one element, bringing in the last element from the previous iteration.
788 ✗ __m256 shifted_odd = _mm256_permute2f128_ps(odd, v_prev_odd, 0x02);
789 ✗ __m256 left = _mm256_castsi256_ps(_mm256_alignr_epi8(
790 _mm256_castps_si256(odd),
791 _mm256_castps_si256(shifted_odd), 12));
792
793 // 5. Triangle Filter: (left + 2*even + odd) / 8
794 ✗ __m256 avg = _mm256_mul_ps(
795 _mm256_add_ps(_mm256_add_ps(left, _mm256_add_ps(even, even)), odd),
796 _mm256_set1_ps(0.125f)
797 );
798
799 if constexpr (!full_opacity) {
800 ✗ avg = _mm256_mul_ps(avg, v_opacity);
801 }
802
803 ✗ _mm256_storeu_ps(dst + x, avg);
804
805 // Update carry for next iteration: the last 'odd' sum becomes the next 'left' start
806 ✗ v_prev_odd = _mm256_set1_ps(((float*)&odd)[7]);
807 }
808
809 // Bridge to scalar tail: Extract the last odd sum processed by SIMD
810 ✗ right_val = _mm256_cvtss_f32(_mm256_permutevar8x32_ps(v_prev_odd, _mm256_setzero_si256()));
811 ✗ for (; x < width; x++) {
812 ✗ const float left = right_val;
813 ✗ const float mid = row0[x * 2] + row1[x * 2];
814 ✗ right_val = row0[x * 2 + 1] + row1[x * 2 + 1];
815 ✗ const float avg = (left + 2 * mid + right_val + 4) * 0.125f; // / 8.0f
816 ✗ dst[x] = full_opacity ? avg : avg * opacity;
817 }
818 ✗ }
819
820
821 // ---------------------------------------------------------------------------
822 // MASK422_TOPLEFT — left co-sited point sample (no averaging).
823 // dst[x] = src[x*2]
824 // ---------------------------------------------------------------------------
825 template<bool full_opacity>
826 #if defined(GCC) || defined(CLANG)
827 __attribute__((__target__("avx2")))
828 #endif
829 ✗ static void fill_mask422_topleft_float_avx2(
830 float* dst, const float* src, int width,
831 float opacity)
832 {
833 ✗ int x = 0;
834 ✗ for (; x < width; x++) {
835 ✗ const float val = src[x * 2];
836 ✗ dst[x] = full_opacity ? val : val * opacity;
837 }
838 ✗ }
839
840 // ---------------------------------------------------------------------------
841 // MASK420_TOPLEFT — top-left co-sited point sample (top row only, no averaging).
842 // dst[x] = row0[x*2]
843 // ---------------------------------------------------------------------------
844 template<bool full_opacity>
845 #if defined(GCC) || defined(CLANG)
846 __attribute__((__target__("avx2")))
847 #endif
848 ✗ static void fill_mask420_topleft_float_avx2(
849 float* dst, const float* row0, int /*mask_pitch*/, int width,
850 float opacity)
851 {
852 // Identical to fill_mask422_topleft_float_avx2: top row only, left co-sited.
853 ✗ fill_mask422_topleft_float_avx2<full_opacity>(dst, row0, width, opacity);
854 ✗ }
855
856 // ---------------------------------------------------------------------------
857 // MASK411 — horizontal 4-tap box average.
858 // avg[x] = (src[x*4]+src[x*4+1]+src[x*4+2]+src[x*4+3]) / 4 (*0.25)
859 // ---------------------------------------------------------------------------
860 template<bool full_opacity>
861 #if defined(GCC) || defined(CLANG)
862 __attribute__((__target__("avx2")))
863 #endif
864 ✗ static void fill_mask411_float_avx2(
865 float* dst, const float* src, int width,
866 float opacity)
867 {
868 ✗ int x = 0;
869 ✗ for (; x < width; x++) {
870 ✗ const float avg = (src[x * 4] + src[x * 4 + 1] + src[x * 4 + 2] + src[x * 4 + 3]) * 0.25f;
871 ✗ dst[x] = full_opacity ? avg : avg * opacity;
872 }
873 ✗ }
874
875 // ---------------------------
876 // End of float mask helpers
877 // ---------------------------
878
879
880 // ---------------------------------------------------------------------------
881 // prepare_effective_mask_for_row_avx2
882 // ---------------------------------------------------------------------------
883 template<MaskMode maskMode, typename pixel_t, bool full_opacity>
884 #if defined(GCC) || defined(CLANG)
885 __attribute__((__target__("avx2")))
886 #endif
887 439 const pixel_t* prepare_effective_mask_for_row_avx2(
888 const pixel_t* maskp,
889 int mask_pitch,
890 int width,
891 std::vector<pixel_t>& buf,
892 int opacity_i,
893 int half,
894 MagicDiv magic)
895 {
896 if constexpr (maskMode == MASK444) {
897 if constexpr (full_opacity) {
898 95 return maskp;
899 } else {
900 154 pixel_t* dst = buf.data();
901 154 int x = 0;
902 if constexpr (sizeof(pixel_t) == 1) {
903 99 const __m256i v_opacity = _mm256_set1_epi16((short)opacity_i);
904 99 const __m256i v_half16 = _mm256_set1_epi16((short)half);
905 99 const __m256i v_mdiv = _mm256_set1_epi16((short)magic.div);
906
2/2
✓ Branch 36 → 16 taken 81 times.
✓ Branch 36 → 37 taken 99 times.
 180 for (; x <= width - 16; x += 16) {
907 243 __m256i v16 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(maskp + x)));
908 81 __m256i scaled = _mm256_add_epi16(_mm256_mullo_epi16(v16, v_opacity), v_half16);
909 243 __m256i res = _mm256_srli_epi16(_mm256_mulhi_epu16(scaled, v_mdiv), magic.shift);
910 81 _mm_storeu_si128((__m128i*)(dst + x), avx2_pack_u16_to_u8(res));
911 }
912 } else {
913 55 const __m256i v_opacity32 = _mm256_set1_epi32(opacity_i);
914 55 const __m256i v_half32 = _mm256_set1_epi32(half);
915
2/2
✓ Branch 42 → 8 taken 105 times.
✓ Branch 42 → 43 taken 55 times.
 160 for (; x <= width - 8; x += 8) {
916 210 __m256i v32 = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)(maskp + x)));
917 315 __m256i res = simd_magic_div_32_avx2(
918 _mm256_add_epi32(_mm256_mullo_epi32(v32, v_opacity32), v_half32),
919 105 magic.div, magic.shift);
920 105 _mm_storeu_si128((__m128i*)(dst + x), avx2_pack_u32_to_u16(res));
921 }
922 }
923
4/4
unsigned char const* prepare_effective_mask_for_row_avx2<(MaskMode)7, unsigned char, false>(unsigned char const*, int, int, std::vector<unsigned char, std::allocator<unsigned char> >&, int, int, MagicDiv):
✓ Branch 41 → 38 taken 643 times.
✓ Branch 41 → 42 taken 99 times.
unsigned short const* prepare_effective_mask_for_row_avx2<(MaskMode)7, unsigned short, false>(unsigned short const*, int, int, std::vector<unsigned short, std::allocator<unsigned short> >&, int, int, MagicDiv):
✓ Branch 47 → 44 taken 155 times.
✓ Branch 47 → 48 taken 55 times.
 952 for (; x < width; x++)
924 798 dst[x] = static_cast<pixel_t>(
925 1596 magic_div_rt<pixel_t>((uint32_t)maskp[x] * (uint32_t)opacity_i + (uint32_t)half, magic));
926 154 return dst;
927 }
928 }
929 else {
930 190 pixel_t* dst = buf.data();
931 if constexpr (maskMode == MASK422)
932 45 fill_mask422_avx2<pixel_t, full_opacity>(dst, maskp, width, opacity_i, half, magic);
933 else if constexpr (maskMode == MASK422_MPEG2)
934 25 fill_mask422_mpeg2_avx2<pixel_t, full_opacity>(dst, maskp, width, opacity_i, half, magic);
935 else if constexpr (maskMode == MASK422_TOPLEFT)
936 20 fill_mask422_topleft_avx2<pixel_t, full_opacity>(dst, maskp, width, opacity_i, half, magic);
937 else if constexpr (maskMode == MASK420)
938 49 fill_mask420_avx2<pixel_t, full_opacity>(dst, maskp, mask_pitch, width, opacity_i, half, magic);
939 else if constexpr (maskMode == MASK420_MPEG2)
940 31 fill_mask420_mpeg2_avx2<pixel_t, full_opacity>(dst, maskp, mask_pitch, width, opacity_i, half, magic);
941 else if constexpr (maskMode == MASK420_TOPLEFT)
942 20 fill_mask420_topleft_avx2<pixel_t, full_opacity>(dst, maskp, mask_pitch, width, opacity_i, half, magic);
943 else if constexpr (maskMode == MASK411)
944 ✗ fill_mask411_avx2<pixel_t, full_opacity>(dst, maskp, width, opacity_i, half, magic);
945 190 return dst;
946 }
947 }
948
949 template<MaskMode maskMode, bool full_opacity>
950 #if defined(GCC) || defined(CLANG)
951 __attribute__((__target__("avx2")))
952 #endif
953 30 AVS_FORCEINLINE const float* prepare_effective_mask_for_row_float_avx2(
954 const float* maskp,
955 int mask_pitch,
956 int width,
957 std::vector<float>& buf,
958 float opacity)
959 {
960 if constexpr (maskMode == MASK444) {
961 if constexpr (full_opacity) {
962 5 return maskp;
963 }
964 else {
965 5 float* dst = buf.data();
966 5 int x = 0;
967 5 const __m256 v_opacity = _mm256_set1_ps(opacity);
968
2/2
✓ Branch 12 → 6 taken 5 times.
✓ Branch 12 → 13 taken 5 times.
 10 for (; x <= width - 8; x += 8) {
969 // just put back opacity * mask
970 10 __m256 v16 = _mm256_loadu_ps(maskp + x);
971 5 __m256 scaled = _mm256_mul_ps(v16, v_opacity);
972 5 _mm256_storeu_ps(dst + x, scaled);
973 }
974
2/2
✓ Branch 22 → 14 taken 5 times.
✓ Branch 22 → 23 taken 5 times.
 10 for (; x <= width - 4; x += 4) {
975 // just put back opacity * mask
976 10 __m128 v16 = _mm_loadu_ps(maskp + x);
977 5 __m128 scaled = _mm_mul_ps(v16, _mm_set1_ps(opacity));
978 5 _mm_storeu_ps(dst + x, scaled);
979 }
980
2/2
✓ Branch 25 → 24 taken 5 times.
✓ Branch 25 → 26 taken 5 times.
 10 for (; x < width; x++)
981 5 dst[x] = maskp[x] * opacity;
982 5 return dst;
983 }
984 }
985 else {
986 20 float* dst = buf.data();
987 if constexpr (maskMode == MASK422)
988 10 fill_mask422_float_avx2<full_opacity>(dst, maskp, width, opacity);
989 else if constexpr (maskMode == MASK422_MPEG2)
990 ✗ fill_mask422_mpeg2_float_avx2<full_opacity>(dst, maskp, width, opacity);
991 else if constexpr (maskMode == MASK422_TOPLEFT)
992 ✗ fill_mask422_topleft_float_avx2<full_opacity>(dst, maskp, width, opacity);
993 else if constexpr (maskMode == MASK420)
994 10 fill_mask420_float_avx2<full_opacity>(dst, maskp, mask_pitch, width, opacity);
995 else if constexpr (maskMode == MASK420_MPEG2)
996 ✗ fill_mask420_mpeg2_float_avx2<full_opacity>(dst, maskp, mask_pitch, width, opacity);
997 else if constexpr (maskMode == MASK420_TOPLEFT)
998 ✗ fill_mask420_topleft_float_avx2<full_opacity>(dst, maskp, mask_pitch, width, opacity);
999 else if constexpr (maskMode == MASK411)
1000 ✗ fill_mask411_float_avx2<full_opacity>(dst, maskp, width, opacity);
1001 20 return dst;
1002 }
1003 }
1004
1005 // ---------------------------------------------------------------------------
1006 // Explicit instantiations
1007 // ---------------------------------------------------------------------------
1008
1009 // prepare_effective_mask_for_row_avx2
1010 #define INST_PREP_AVX2(mm, pt) \
1011 template const pt* prepare_effective_mask_for_row_avx2<mm, pt, true> (const pt*, int, int, std::vector<pt>&, int, int, MagicDiv); \
1012 template const pt* prepare_effective_mask_for_row_avx2<mm, pt, false>(const pt*, int, int, std::vector<pt>&, int, int, MagicDiv);
1013 INST_PREP_AVX2(MASK444, uint8_t) INST_PREP_AVX2(MASK444, uint16_t)
1014 INST_PREP_AVX2(MASK420, uint8_t) INST_PREP_AVX2(MASK420, uint16_t)
1015 INST_PREP_AVX2(MASK420_MPEG2, uint8_t) INST_PREP_AVX2(MASK420_MPEG2, uint16_t)
1016 INST_PREP_AVX2(MASK420_TOPLEFT, uint8_t) INST_PREP_AVX2(MASK420_TOPLEFT, uint16_t)
1017 INST_PREP_AVX2(MASK422, uint8_t) INST_PREP_AVX2(MASK422, uint16_t)
1018 INST_PREP_AVX2(MASK422_MPEG2, uint8_t) INST_PREP_AVX2(MASK422_MPEG2, uint16_t)
1019 INST_PREP_AVX2(MASK422_TOPLEFT, uint8_t) INST_PREP_AVX2(MASK422_TOPLEFT, uint16_t)
1020 INST_PREP_AVX2(MASK411, uint8_t) INST_PREP_AVX2(MASK411, uint16_t)
1021 #undef INST_PREP_AVX2
1022
1023 // prepare_effective_mask_for_row_avx2
1024 #define INST_PREP_AVX2(mm) \
1025 template const float* prepare_effective_mask_for_row_float_avx2<mm, true> (const float*, int, int, std::vector<float>&, float); \
1026 template const float* prepare_effective_mask_for_row_float_avx2<mm, false>(const float*, int, int, std::vector<float>&, float);
1027 INST_PREP_AVX2(MASK444)
1028 INST_PREP_AVX2(MASK420)
1029 INST_PREP_AVX2(MASK420_MPEG2)
1030 INST_PREP_AVX2(MASK420_TOPLEFT)
1031 INST_PREP_AVX2(MASK422)
1032 INST_PREP_AVX2(MASK422_MPEG2)
1033 INST_PREP_AVX2(MASK422_TOPLEFT)
1034 INST_PREP_AVX2(MASK411)
1035 #undef INST_PREP_AVX2
1036
1037