GCC Code Coverage Report

Directory: avs_core/
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 68.8% 687 / 0 / 998
Functions: 68.9% 62 / 0 / 90
Branches: 69.0% 109 / 0 / 158
filters/overlay/intel/masked_rowprep_sse41.cpp
Line Branch Exec Source
1 // masked_rowprep_sse41.cpp
2 // SSE4.1 rowprep implementations + explicit template instantiations.
3 // Compiled with -msse4.1 (GCC/Clang) or /arch:SSE2 (MSVC) via handle_arch_flags(SSE41).
4 //
5 // simd_magic_div_32 lives in masked_rowprep_sse41.h (inline, needed by merge impl).
6 // All fill_mask*_sse41 helpers are static (internal to this TU).
7
8 #if defined(_MSC_VER)
9 #include <intrin.h>
10 #else
11 #include <smmintrin.h>
12 #endif
13
14 #include "masked_rowprep_sse41.h" // own declarations + simd_magic_div_32 inline
15
16 #include <vector>
17 #include <cstdint>
18
19 // ---------------------------------------------------------------------------
20 // Internal helper — deinterleave 16 uint8 → two int16 vectors (even/odd bytes).
21 // Only used within this TU by the fill_mask* functions below.
22 // ---------------------------------------------------------------------------
23 #if defined(GCC) || defined(CLANG)
24 __attribute__((__target__("sse4.1")))
25 #endif
26 static AVS_FORCEINLINE void deinterleave_u8_epi16(
27 __m128i v, __m128i& even_out, __m128i& odd_out)
28 {
29 360 even_out = _mm_and_si128(v, _mm_set1_epi16(0x00FF));
30 180 odd_out = _mm_srli_epi16(v, 8);
31 180 }
32
33 // ---------------------------------------------------------------------------
34 // MASK422 — horizontal 2-tap average, no inter-row state.
35 // avg[x] = (src[x*2] + src[x*2+1] + 1) >> 1
36 // dst[x] = full_opacity ? avg : (avg * opacity_i + half) / max
37 // ---------------------------------------------------------------------------
38 template<typename pixel_t, bool full_opacity>
39 #if defined(GCC) || defined(CLANG)
40 __attribute__((__target__("sse4.1")))
41 #endif
42 45 static void fill_mask422_sse41(
43 pixel_t* dst, const pixel_t* src, int width,
44 int opacity_i, int half, MagicDiv magic)
45 {
46 45 int x = 0;
47 if constexpr (sizeof(pixel_t) == 1) {
48 // 8-bit: 16 luma bytes → 8 chroma bytes per iteration
49 20 [[maybe_unused]] const __m128i v_opacity = _mm_set1_epi16((short)opacity_i);
50 20 [[maybe_unused]] const __m128i v_half16 = _mm_set1_epi16((short)half);
51 20 [[maybe_unused]] const __m128i v_mdiv = _mm_set1_epi16((short)magic.div);
52
4/4
void fill_mask422_sse41<unsigned char, false>(unsigned char*, unsigned char const*, int, int, int, MagicDiv):
✓ Branch 48 → 15 taken 20 times.
✓ Branch 48 → 49 taken 10 times.
void fill_mask422_sse41<unsigned char, true>(unsigned char*, unsigned char const*, int, int, int, MagicDiv):
✓ Branch 40 → 15 taken 20 times.
✓ Branch 40 → 41 taken 10 times.
 60 for (; x <= width - 8; x += 8) {
53 80 __m128i v = _mm_loadu_si128((const __m128i*)(src + x * 2));
54 __m128i even, odd;
55 deinterleave_u8_epi16(v, even, odd);
56 160 __m128i avg = _mm_srli_epi16(_mm_add_epi16(_mm_add_epi16(even, odd), _mm_set1_epi16(1)), 1);
57 if constexpr (!full_opacity) {
58 // (avg * opacity + half) / max — all fit in uint16
59 20 __m128i scaled = _mm_add_epi16(_mm_mullo_epi16(avg, v_opacity), v_half16);
60 60 avg = _mm_srli_epi16(_mm_mulhi_epu16(scaled, v_mdiv), magic.shift);
61 }
62 40 _mm_storel_epi64((__m128i*)(dst + x), _mm_packus_epi16(avg, avg));
63 }
64 } else {
65 // 16-bit: 8 uint16 luma -> 4 uint16 chroma per iteration.
66 // Use signed pivot to utilize madd_epi16 safely
67 25 const __m128i v_ones = _mm_set1_epi16(1);
68 25 const __m128i v_pivot16 = _mm_set1_epi16(-32768);
69 // 65536 corrects the double-bias subtraction, +1 handles the formula's rounding
70 25 const __m128i v_correct32 = _mm_set1_epi32(65536 + 1);
71
72 25 [[maybe_unused]] const __m128i v_opacity32 = _mm_set1_epi32(opacity_i);
73 25 [[maybe_unused]] const __m128i v_half32 = _mm_set1_epi32(half);
74
75
4/4
void fill_mask422_sse41<unsigned short, false>(unsigned short*, unsigned short const*, int, int, int, MagicDiv):
✓ Branch 63 → 23 taken 50 times.
✓ Branch 63 → 64 taken 15 times.
void fill_mask422_sse41<unsigned short, true>(unsigned short*, unsigned short const*, int, int, int, MagicDiv):
✓ Branch 37 → 23 taken 30 times.
✓ Branch 37 → 38 taken 10 times.
 105 for (; x <= width - 4; x += 4) {
76 160 __m128i v = _mm_loadu_si128((const __m128i*)(src + x * 2));
77
78 // unsigned 0..65535 -> signed -32768..32767
79 80 __m128i v_signed = _mm_add_epi16(v, v_pivot16);
80
81 // Pairwise add: (a - 32768) + (b - 32768) = a + b - 65536
82 80 __m128i sum32 = _mm_madd_epi16(v_signed, v_ones);
83
84 // Add 65536 back + 1 for rounding, then shift
85 110 __m128i avg32 = _mm_srli_epi32(_mm_add_epi32(sum32, v_correct32), 1);
86
87 if constexpr (!full_opacity)
88 150 avg32 = simd_magic_div_32(
89 _mm_add_epi32(_mm_mullo_epi32(avg32, v_opacity32), v_half32),
90 50 magic.div, magic.shift);
91 80 _mm_storel_epi64((__m128i*)(dst + x), _mm_packus_epi32(avg32, avg32));
92 }
93 }
94
8/8
void fill_mask422_sse41<unsigned char, false>(unsigned char*, unsigned char const*, int, int, int, MagicDiv):
✓ Branch 53 → 50 taken 10 times.
✓ Branch 53 → 54 taken 10 times.
void fill_mask422_sse41<unsigned char, true>(unsigned char*, unsigned char const*, int, int, int, MagicDiv):
✓ Branch 43 → 42 taken 10 times.
✓ Branch 43 → 44 taken 10 times.
void fill_mask422_sse41<unsigned short, false>(unsigned short*, unsigned short const*, int, int, int, MagicDiv):
✓ Branch 68 → 65 taken 25 times.
✓ Branch 68 → 69 taken 15 times.
void fill_mask422_sse41<unsigned short, true>(unsigned short*, unsigned short const*, int, int, int, MagicDiv):
✓ Branch 40 → 39 taken 10 times.
✓ Branch 40 → 41 taken 10 times.
 100 for (; x < width; x++) {
95 55 const int avg = (src[x * 2] + src[x * 2 + 1] + 1) >> 1;
96 55 dst[x] = full_opacity ? (pixel_t)avg
97 70 : (pixel_t)magic_div_rt<pixel_t>((uint32_t)avg * (uint32_t)opacity_i + (uint32_t)half, magic);
98 }
99 45 }
100
101 // ---------------------------------------------------------------------------
102 // MASK422_MPEG2 — horizontal 3-tap triangle filter with sliding window.
103 // avg[x] = (left + 2*src[x*2] + src[x*2+1] + 2) >> 2
104 // dst[x] = full_opacity ? avg : (avg * opacity_i + half) / max
105 // ---------------------------------------------------------------------------
106 template<typename pixel_t, bool full_opacity>
107 #if defined(GCC) || defined(CLANG)
108 __attribute__((__target__("sse4.1")))
109 #endif
110 20 static void fill_mask422_mpeg2_sse41(
111 pixel_t* dst, const pixel_t* src, int width,
112 int opacity_i, int half, MagicDiv magic)
113 {
114 20 int x = 0;
115 if constexpr (sizeof(pixel_t) == 1) {
116 10 [[maybe_unused]] const __m128i v_opacity = _mm_set1_epi16((short)opacity_i);
117 10 [[maybe_unused]] const __m128i v_half16 = _mm_set1_epi16((short)half);
118 10 [[maybe_unused]] const __m128i v_mdiv = _mm_set1_epi16((short)magic.div);
119 10 __m128i prev_carry = _mm_insert_epi16(_mm_setzero_si128(), src[0], 7);
120
4/4
void fill_mask422_mpeg2_sse41<unsigned char, false>(unsigned char*, unsigned char const*, int, int, int, MagicDiv):
✓ Branch 56 → 17 taken 10 times.
✓ Branch 56 → 57 taken 5 times.
void fill_mask422_mpeg2_sse41<unsigned char, true>(unsigned char*, unsigned char const*, int, int, int, MagicDiv):
✓ Branch 48 → 17 taken 10 times.
✓ Branch 48 → 49 taken 5 times.
 30 for (; x <= width - 8; x += 8) {
121 40 __m128i v = _mm_loadu_si128((const __m128i*)(src + x * 2));
122 __m128i even, odd;
123 deinterleave_u8_epi16(v, even, odd);
124 20 __m128i left = _mm_alignr_epi8(odd, prev_carry, 14);
125 120 __m128i res = _mm_srli_epi16(
126 _mm_add_epi16(
127 _mm_add_epi16(_mm_add_epi16(left, _mm_slli_epi16(even, 1)), odd),
128 _mm_set1_epi16(2)), 2);
129 if constexpr (!full_opacity) {
130 10 __m128i scaled = _mm_add_epi16(_mm_mullo_epi16(res, v_opacity), v_half16);
131 30 res = _mm_srli_epi16(_mm_mulhi_epu16(scaled, v_mdiv), magic.shift);
132 }
133 20 _mm_storel_epi64((__m128i*)(dst + x), _mm_packus_epi16(res, res));
134 40 prev_carry = _mm_insert_epi16(_mm_setzero_si128(), _mm_extract_epi16(odd, 7), 7);
135 }
136 10 int right_val = _mm_extract_epi16(prev_carry, 7);
137
4/4
void fill_mask422_mpeg2_sse41<unsigned char, false>(unsigned char*, unsigned char const*, int, int, int, MagicDiv):
✓ Branch 61 → 58 taken 5 times.
✓ Branch 61 → 62 taken 5 times.
void fill_mask422_mpeg2_sse41<unsigned char, true>(unsigned char*, unsigned char const*, int, int, int, MagicDiv):
✓ Branch 51 → 50 taken 5 times.
✓ Branch 51 → 52 taken 5 times.
 20 for (; x < width; x++) {
138 10 const int left = right_val;
139 10 const int mid = src[x * 2];
140 10 right_val = src[x * 2 + 1];
141 10 const int avg = (left + 2 * mid + right_val + 2) >> 2;
142 10 dst[x] = full_opacity ? (pixel_t)avg
143 10 : (pixel_t)magic_div_rt<pixel_t>((uint32_t)avg * (uint32_t)opacity_i + (uint32_t)half, magic);
144 }
145 } else {
146 10 [[maybe_unused]] const __m128i v_opacity32 = _mm_set1_epi32(opacity_i);
147 10 [[maybe_unused]] const __m128i v_half32 = _mm_set1_epi32(half);
148 10 __m128i prev_carry = _mm_insert_epi32(_mm_setzero_si128(), src[0], 3);
149
4/4
void fill_mask422_mpeg2_sse41<unsigned short, false>(unsigned short*, unsigned short const*, int, int, int, MagicDiv):
✓ Branch 69 → 13 taken 10 times.
✓ Branch 69 → 70 taken 5 times.
void fill_mask422_mpeg2_sse41<unsigned short, true>(unsigned short*, unsigned short const*, int, int, int, MagicDiv):
✓ Branch 43 → 13 taken 10 times.
✓ Branch 43 → 44 taken 5 times.
 30 for (; x <= width - 4; x += 4) {
150 40 __m128i v = _mm_loadu_si128((const __m128i*)(src + x * 2));
151 20 __m128i lo32 = _mm_cvtepu16_epi32(v);
152 20 __m128i hi32 = _mm_cvtepu16_epi32(_mm_srli_si128(v, 8));
153 40 __m128i even32 = _mm_unpacklo_epi64(
154 20 _mm_shuffle_epi32(lo32, _MM_SHUFFLE(2, 0, 2, 0)),
155 20 _mm_shuffle_epi32(hi32, _MM_SHUFFLE(2, 0, 2, 0)));
156 40 __m128i odd32 = _mm_unpacklo_epi64(
157 20 _mm_shuffle_epi32(lo32, _MM_SHUFFLE(3, 1, 3, 1)),
158 20 _mm_shuffle_epi32(hi32, _MM_SHUFFLE(3, 1, 3, 1)));
159 20 __m128i left = _mm_alignr_epi8(odd32, prev_carry, 12);
160 110 __m128i res = _mm_srli_epi32(
161 _mm_add_epi32(
162 _mm_add_epi32(_mm_add_epi32(left, _mm_slli_epi32(even32, 1)), odd32),
163 _mm_set1_epi32(2)), 2);
164 if constexpr (!full_opacity)
165 30 res = simd_magic_div_32(
166 _mm_add_epi32(_mm_mullo_epi32(res, v_opacity32), v_half32),
167 10 magic.div, magic.shift);
168 20 _mm_storel_epi64((__m128i*)(dst + x), _mm_packus_epi32(res, res));
169 40 prev_carry = _mm_insert_epi32(_mm_setzero_si128(), _mm_extract_epi32(odd32, 3), 3);
170 }
171 10 int right_val = _mm_extract_epi32(prev_carry, 3);
172
4/4
void fill_mask422_mpeg2_sse41<unsigned short, false>(unsigned short*, unsigned short const*, int, int, int, MagicDiv):
✓ Branch 74 → 71 taken 5 times.
✓ Branch 74 → 75 taken 5 times.
void fill_mask422_mpeg2_sse41<unsigned short, true>(unsigned short*, unsigned short const*, int, int, int, MagicDiv):
✓ Branch 46 → 45 taken 5 times.
✓ Branch 46 → 47 taken 5 times.
 20 for (; x < width; x++) {
173 10 const int left = right_val;
174 10 const int mid = src[x * 2];
175 10 right_val = src[x * 2 + 1];
176 10 const int avg = (left + 2 * mid + right_val + 2) >> 2;
177 10 dst[x] = full_opacity ? (pixel_t)avg
178 10 : (pixel_t)magic_div_rt<pixel_t>((uint32_t)avg * (uint32_t)opacity_i + (uint32_t)half, magic);
179 }
180 }
181 20 }
182
183 // ---------------------------------------------------------------------------
184 // MASK420 — 2×2 box average (MPEG-1 placement). No inter-row state.
185 // avg[x] = (row0[x*2]+row0[x*2+1]+row1[x*2]+row1[x*2+1]+2) >> 2
186 // dst[x] = full_opacity ? avg : (avg * opacity_i + half) / max
187 // ---------------------------------------------------------------------------
188 template<typename pixel_t, bool full_opacity>
189 #if defined(GCC) || defined(CLANG)
190 __attribute__((__target__("sse4.1")))
191 #endif
192 45 static void fill_mask420_sse41(
193 pixel_t* dst, const pixel_t* row0, int mask_pitch, int width,
194 int opacity_i, int half, MagicDiv magic)
195 {
196 45 const pixel_t* row1 = row0 + mask_pitch;
197 45 int x = 0;
198 if constexpr (sizeof(pixel_t) == 1) {
199 25 [[maybe_unused]] const __m128i v_opacity = _mm_set1_epi16((short)opacity_i);
200 25 [[maybe_unused]] const __m128i v_half16 = _mm_set1_epi16((short)half);
201 25 [[maybe_unused]] const __m128i v_mdiv = _mm_set1_epi16((short)magic.div);
202
4/4
void fill_mask420_sse41<unsigned char, false>(unsigned char*, unsigned char const*, int, int, int, int, MagicDiv):
✓ Branch 63 → 15 taken 40 times.
✓ Branch 63 → 64 taken 15 times.
void fill_mask420_sse41<unsigned char, true>(unsigned char*, unsigned char const*, int, int, int, int, MagicDiv):
✓ Branch 55 → 15 taken 20 times.
✓ Branch 55 → 56 taken 10 times.
 85 for (; x <= width - 8; x += 8) {
203 60 __m128i r0 = _mm_loadu_si128((const __m128i*)(row0 + x * 2));
204 120 __m128i r1 = _mm_loadu_si128((const __m128i*)(row1 + x * 2));
205 __m128i e0, o0, e1, o1;
206 deinterleave_u8_epi16(r0, e0, o0);
207 deinterleave_u8_epi16(r1, e1, o1);
208 360 __m128i avg = _mm_srli_epi16(
209 _mm_add_epi16(_mm_add_epi16(_mm_add_epi16(e0, o0), _mm_add_epi16(e1, o1)), _mm_set1_epi16(2)), 2);
210 if constexpr (!full_opacity) {
211 40 __m128i scaled = _mm_add_epi16(_mm_mullo_epi16(avg, v_opacity), v_half16);
212 120 avg = _mm_srli_epi16(_mm_mulhi_epu16(scaled, v_mdiv), magic.shift);
213 }
214 60 _mm_storel_epi64((__m128i*)(dst + x), _mm_packus_epi16(avg, avg));
215 }
216 } else {
217 // 16-bit: unsigned widening to avoid signed overflow in madd_epi16
218 20 [[maybe_unused]] const __m128i v_opacity32 = _mm_set1_epi32(opacity_i);
219 20 [[maybe_unused]] const __m128i v_half32 = _mm_set1_epi32(half);
220
4/4
void fill_mask420_sse41<unsigned short, false>(unsigned short*, unsigned short const*, int, int, int, int, MagicDiv):
✓ Branch 71 → 11 taken 30 times.
✓ Branch 71 → 72 taken 10 times.
void fill_mask420_sse41<unsigned short, true>(unsigned short*, unsigned short const*, int, int, int, int, MagicDiv):
✓ Branch 45 → 11 taken 30 times.
✓ Branch 45 → 46 taken 10 times.
 80 for (; x <= width - 4; x += 4) {
221 60 __m128i v0 = _mm_loadu_si128((const __m128i*)(row0 + x * 2)); // 8 uint16
222 120 __m128i v1 = _mm_loadu_si128((const __m128i*)(row1 + x * 2));
223 60 __m128i lo0 = _mm_cvtepu16_epi32(v0);
224 120 __m128i hi0 = _mm_cvtepu16_epi32(_mm_srli_si128(v0, 8));
225 60 __m128i lo1 = _mm_cvtepu16_epi32(v1);
226 120 __m128i hi1 = _mm_cvtepu16_epi32(_mm_srli_si128(v1, 8));
227 60 __m128i sum_lo = _mm_add_epi32(lo0, lo1);
228 60 __m128i sum_hi = _mm_add_epi32(hi0, hi1);
229 120 __m128i even32 = _mm_unpacklo_epi64(
230 60 _mm_shuffle_epi32(sum_lo, _MM_SHUFFLE(2, 0, 2, 0)),
231 60 _mm_shuffle_epi32(sum_hi, _MM_SHUFFLE(2, 0, 2, 0)));
232 60 __m128i odd32 = _mm_unpacklo_epi64(
233 60 _mm_shuffle_epi32(sum_lo, _MM_SHUFFLE(3, 1, 3, 1)),
234 60 _mm_shuffle_epi32(sum_hi, _MM_SHUFFLE(3, 1, 3, 1)));
235 210 __m128i avg32 = _mm_srli_epi32(
236 _mm_add_epi32(_mm_add_epi32(even32, odd32), _mm_set1_epi32(2)), 2);
237 if constexpr (!full_opacity)
238 90 avg32 = simd_magic_div_32(
239 _mm_add_epi32(_mm_mullo_epi32(avg32, v_opacity32), v_half32),
240 30 magic.div, magic.shift);
241 60 _mm_storel_epi64((__m128i*)(dst + x), _mm_packus_epi32(avg32, avg32));
242 }
243 }
244
8/8
void fill_mask420_sse41<unsigned char, false>(unsigned char*, unsigned char const*, int, int, int, int, MagicDiv):
✓ Branch 68 → 65 taken 35 times.
✓ Branch 68 → 69 taken 15 times.
void fill_mask420_sse41<unsigned char, true>(unsigned char*, unsigned char const*, int, int, int, int, MagicDiv):
✓ Branch 58 → 57 taken 10 times.
✓ Branch 58 → 59 taken 10 times.
void fill_mask420_sse41<unsigned short, false>(unsigned short*, unsigned short const*, int, int, int, int, MagicDiv):
✓ Branch 76 → 73 taken 10 times.
✓ Branch 76 → 77 taken 10 times.
void fill_mask420_sse41<unsigned short, true>(unsigned short*, unsigned short const*, int, int, int, int, MagicDiv):
✓ Branch 48 → 47 taken 10 times.
✓ Branch 48 → 49 taken 10 times.
 110 for (; x < width; x++) {
245 65 const int avg = ((int)row0[x*2] + row0[x*2+1] + row1[x*2] + row1[x*2+1] + 2) >> 2;
246 65 dst[x] = full_opacity ? (pixel_t)avg
247 90 : (pixel_t)magic_div_rt<pixel_t>((uint32_t)avg * (uint32_t)opacity_i + (uint32_t)half, magic);
248 }
249 45 }
250
251 // ---------------------------------------------------------------------------
252 // MASK420_MPEG2 — 2-row vertical sum + horizontal 3-tap triangle filter.
253 // avg[x] = (left + 2*P[x*2] + P[x*2+1] + 4) >> 3 where P[k] = row0[k]+row1[k]
254 // dst[x] = full_opacity ? avg : (avg * opacity_i + half) / max
255 // ---------------------------------------------------------------------------
256 template<typename pixel_t, bool full_opacity>
257 #if defined(GCC) || defined(CLANG)
258 __attribute__((__target__("sse4.1")))
259 #endif
260 27 static void fill_mask420_mpeg2_sse41(
261 pixel_t* dst, const pixel_t* row0, int mask_pitch, int width,
262 int opacity_i, int half, MagicDiv magic)
263 {
264 27 const pixel_t* row1 = row0 + mask_pitch;
265 27 int x = 0;
266 if constexpr (sizeof(pixel_t) == 1) {
267 17 [[maybe_unused]] const __m128i v_opacity = _mm_set1_epi16((short)opacity_i);
268 17 [[maybe_unused]] const __m128i v_half16 = _mm_set1_epi16((short)half);
269 17 [[maybe_unused]] const __m128i v_mdiv = _mm_set1_epi16((short)magic.div);
270 17 const int p0 = (int)row0[0] + row1[0];
271 17 __m128i prev_carry = _mm_insert_epi16(_mm_setzero_si128(), p0, 7);
272
273
4/4
void fill_mask420_mpeg2_sse41<unsigned char, false>(unsigned char*, unsigned char const*, int, int, int, int, MagicDiv):
✓ Branch 89 → 17 taken 24 times.
✓ Branch 89 → 90 taken 12 times.
void fill_mask420_mpeg2_sse41<unsigned char, true>(unsigned char*, unsigned char const*, int, int, int, int, MagicDiv):
✓ Branch 81 → 17 taken 10 times.
✓ Branch 81 → 82 taken 5 times.
 51 for (; x <= width - 8; x += 8) {
274 34 __m128i r0 = _mm_loadu_si128((const __m128i*)(row0 + x * 2));
275 68 __m128i r1 = _mm_loadu_si128((const __m128i*)(row1 + x * 2));
276 170 __m128i plo = _mm_add_epi16(_mm_unpacklo_epi8(r0, _mm_setzero_si128()),
277 _mm_unpacklo_epi8(r1, _mm_setzero_si128()));
278 170 __m128i phi = _mm_add_epi16(_mm_unpackhi_epi8(r0, _mm_setzero_si128()),
279 _mm_unpackhi_epi8(r1, _mm_setzero_si128()));
280
281 34 const __m128i shuf_even = _mm_setr_epi8(0,1, 4,5, 8,9, 12,13, -1,-1,-1,-1,-1,-1,-1,-1);
282 34 const __m128i shuf_odd = _mm_setr_epi8(2,3, 6,7, 10,11, 14,15, -1,-1,-1,-1,-1,-1,-1,-1);
283 102 __m128i pe = _mm_unpacklo_epi64(_mm_shuffle_epi8(plo, shuf_even),
284 _mm_shuffle_epi8(phi, shuf_even));
285 68 __m128i po = _mm_unpacklo_epi64(_mm_shuffle_epi8(plo, shuf_odd),
286 _mm_shuffle_epi8(phi, shuf_odd));
287
288 34 __m128i left = _mm_alignr_epi8(po, prev_carry, 14);
289 204 __m128i res = _mm_srli_epi16(
290 _mm_add_epi16(
291 _mm_add_epi16(_mm_add_epi16(left, _mm_slli_epi16(pe, 1)), po),
292 _mm_set1_epi16(4)), 3);
293 if constexpr (!full_opacity) {
294 24 __m128i scaled = _mm_add_epi16(_mm_mullo_epi16(res, v_opacity), v_half16);
295 72 res = _mm_srli_epi16(_mm_mulhi_epu16(scaled, v_mdiv), magic.shift);
296 }
297 34 _mm_storel_epi64((__m128i*)(dst + x), _mm_packus_epi16(res, res));
298 68 prev_carry = _mm_insert_epi16(_mm_setzero_si128(), _mm_extract_epi16(po, 7), 7);
299 }
300 17 int right_val = _mm_extract_epi16(prev_carry, 7);
301
4/4
void fill_mask420_mpeg2_sse41<unsigned char, false>(unsigned char*, unsigned char const*, int, int, int, int, MagicDiv):
✓ Branch 94 → 91 taken 54 times.
✓ Branch 94 → 95 taken 12 times.
void fill_mask420_mpeg2_sse41<unsigned char, true>(unsigned char*, unsigned char const*, int, int, int, int, MagicDiv):
✓ Branch 84 → 83 taken 5 times.
✓ Branch 84 → 85 taken 5 times.
 76 for (; x < width; x++) {
302 59 const int left = right_val;
303 59 const int mid = (int)row0[x*2] + row1[x*2];
304 59 right_val = (int)row0[x*2+1] + row1[x*2+1];
305 59 const int avg = (left + 2 * mid + right_val + 4) >> 3;
306 59 dst[x] = full_opacity ? (pixel_t)avg
307 108 : (pixel_t)magic_div_rt<pixel_t>((uint32_t)avg * (uint32_t)opacity_i + (uint32_t)half, magic);
308 }
309 } else {
310 10 [[maybe_unused]] const __m128i v_opacity32 = _mm_set1_epi32(opacity_i);
311 10 [[maybe_unused]] const __m128i v_half32 = _mm_set1_epi32(half);
312 10 const int p0 = (int)row0[0] + row1[0];
313 10 __m128i prev_carry = _mm_insert_epi32(_mm_setzero_si128(), p0, 3);
314
315
4/4
void fill_mask420_mpeg2_sse41<unsigned short, false>(unsigned short*, unsigned short const*, int, int, int, int, MagicDiv):
✓ Branch 79 → 13 taken 10 times.
✓ Branch 79 → 80 taken 5 times.
void fill_mask420_mpeg2_sse41<unsigned short, true>(unsigned short*, unsigned short const*, int, int, int, int, MagicDiv):
✓ Branch 53 → 13 taken 10 times.
✓ Branch 53 → 54 taken 5 times.
 30 for (; x <= width - 4; x += 4) {
316 20 __m128i v0 = _mm_loadu_si128((const __m128i*)(row0 + x * 2));
317 40 __m128i v1 = _mm_loadu_si128((const __m128i*)(row1 + x * 2));
318 40 __m128i plo = _mm_add_epi32(_mm_cvtepu16_epi32(v0), _mm_cvtepu16_epi32(v1));
319 60 __m128i phi = _mm_add_epi32(_mm_cvtepu16_epi32(_mm_srli_si128(v0, 8)),
320 _mm_cvtepu16_epi32(_mm_srli_si128(v1, 8)));
321 40 __m128i pe = _mm_unpacklo_epi64(
322 20 _mm_shuffle_epi32(plo, _MM_SHUFFLE(2, 0, 2, 0)),
323 20 _mm_shuffle_epi32(phi, _MM_SHUFFLE(2, 0, 2, 0)));
324 40 __m128i po = _mm_unpacklo_epi64(
325 20 _mm_shuffle_epi32(plo, _MM_SHUFFLE(3, 1, 3, 1)),
326 20 _mm_shuffle_epi32(phi, _MM_SHUFFLE(3, 1, 3, 1)));
327
328 20 __m128i left = _mm_alignr_epi8(po, prev_carry, 12);
329 110 __m128i res = _mm_srli_epi32(
330 _mm_add_epi32(
331 _mm_add_epi32(_mm_add_epi32(left, _mm_slli_epi32(pe, 1)), po),
332 _mm_set1_epi32(4)), 3);
333 if constexpr (!full_opacity)
334 30 res = simd_magic_div_32(
335 _mm_add_epi32(_mm_mullo_epi32(res, v_opacity32), v_half32),
336 10 magic.div, magic.shift);
337 20 _mm_storel_epi64((__m128i*)(dst + x), _mm_packus_epi32(res, res));
338 40 prev_carry = _mm_insert_epi32(_mm_setzero_si128(), _mm_extract_epi32(po, 3), 3);
339 }
340 10 int right_val = _mm_extract_epi32(prev_carry, 3);
341
4/4
void fill_mask420_mpeg2_sse41<unsigned short, false>(unsigned short*, unsigned short const*, int, int, int, int, MagicDiv):
✓ Branch 84 → 81 taken 5 times.
✓ Branch 84 → 85 taken 5 times.
void fill_mask420_mpeg2_sse41<unsigned short, true>(unsigned short*, unsigned short const*, int, int, int, int, MagicDiv):
✓ Branch 56 → 55 taken 5 times.
✓ Branch 56 → 57 taken 5 times.
 20 for (; x < width; x++) {
342 10 const int left = right_val;
343 10 const int mid = (int)row0[x*2] + row1[x*2];
344 10 right_val = (int)row0[x*2+1] + row1[x*2+1];
345 10 const int avg = (left + 2 * mid + right_val + 4) >> 3;
346 10 dst[x] = full_opacity ? (pixel_t)avg
347 10 : (pixel_t)magic_div_rt<pixel_t>((uint32_t)avg * (uint32_t)opacity_i + (uint32_t)half, magic);
348 }
349 }
350 27 }
351
352 // ---------------------------------------------------------------------------
353 // MASK422_TOPLEFT — left co-sited point sample (no averaging).
354 // dst[x] = src[x*2]
355 // dst[x] = full_opacity ? dst[x] : (dst[x] * opacity_i + half) / max
356 // ---------------------------------------------------------------------------
357 template<typename pixel_t, bool full_opacity>
358 #if defined(GCC) || defined(CLANG)
359 __attribute__((__target__("sse4.1")))
360 #endif
361 40 static void fill_mask422_topleft_sse41(
362 pixel_t* dst, const pixel_t* src, int width,
363 int opacity_i, int half, MagicDiv magic)
364 {
365 40 int x = 0;
366 if constexpr (sizeof(pixel_t) == 1) {
367 20 [[maybe_unused]] const __m128i v_opacity = _mm_set1_epi16((short)opacity_i);
368 20 [[maybe_unused]] const __m128i v_half16 = _mm_set1_epi16((short)half);
369 20 [[maybe_unused]] const __m128i v_mdiv = _mm_set1_epi16((short)magic.div);
370
4/4
void fill_mask422_topleft_sse41<unsigned char, false>(unsigned char*, unsigned char const*, int, int, int, MagicDiv):
✓ Branch 35 → 15 taken 20 times.
✓ Branch 35 → 36 taken 10 times.
void fill_mask422_topleft_sse41<unsigned char, true>(unsigned char*, unsigned char const*, int, int, int, MagicDiv):
✓ Branch 27 → 15 taken 20 times.
✓ Branch 27 → 28 taken 10 times.
 60 for (; x <= width - 8; x += 8) {
371 80 __m128i v = _mm_loadu_si128((const __m128i*)(src + x * 2));
372 80 __m128i even = _mm_and_si128(v, _mm_set1_epi16(0x00FF)); // left (even) bytes
373 if constexpr (!full_opacity) {
374 20 __m128i scaled = _mm_add_epi16(_mm_mullo_epi16(even, v_opacity), v_half16);
375 60 even = _mm_srli_epi16(_mm_mulhi_epu16(scaled, v_mdiv), magic.shift);
376 }
377 40 _mm_storel_epi64((__m128i*)(dst + x), _mm_packus_epi16(even, even));
378 }
379 } else {
380 // 16-bit: grab even-indexed uint16 elements (src[x*2], src[x*2+2], ...)
381 // Load 8 uint16: [a,b,c,d,e,f,g,h] → want [a,c,e,g]
382 20 const __m128i shuf = _mm_setr_epi8(0,1, 4,5, 8,9, 12,13, -1,-1,-1,-1,-1,-1,-1,-1);
383 20 [[maybe_unused]] const __m128i v_opacity32 = _mm_set1_epi32(opacity_i);
384 20 [[maybe_unused]] const __m128i v_half32 = _mm_set1_epi32(half);
385
4/4
void fill_mask422_topleft_sse41<unsigned short, false>(unsigned short*, unsigned short const*, int, int, int, MagicDiv):
✓ Branch 51 → 15 taken 20 times.
✓ Branch 51 → 52 taken 10 times.
void fill_mask422_topleft_sse41<unsigned short, true>(unsigned short*, unsigned short const*, int, int, int, MagicDiv):
✓ Branch 21 → 15 taken 20 times.
✓ Branch 21 → 22 taken 10 times.
 60 for (; x <= width - 4; x += 4) {
386 80 __m128i v = _mm_loadu_si128((const __m128i*)(src + x * 2));
387 40 __m128i even = _mm_shuffle_epi8(v, shuf); // [a,c,e,g,0,...] as 4 uint16
388 if constexpr (!full_opacity) {
389 20 __m128i even32 = _mm_cvtepu16_epi32(even);
390 60 even32 = simd_magic_div_32(
391 _mm_add_epi32(_mm_mullo_epi32(even32, v_opacity32), v_half32),
392 20 magic.div, magic.shift);
393 20 even = _mm_packus_epi32(even32, even32);
394 }
395 40 _mm_storel_epi64((__m128i*)(dst + x), even);
396 }
397 }
398
8/8
void fill_mask422_topleft_sse41<unsigned char, false>(unsigned char*, unsigned char const*, int, int, int, MagicDiv):
✓ Branch 40 → 37 taken 10 times.
✓ Branch 40 → 41 taken 10 times.
void fill_mask422_topleft_sse41<unsigned char, true>(unsigned char*, unsigned char const*, int, int, int, MagicDiv):
✓ Branch 30 → 29 taken 10 times.
✓ Branch 30 → 31 taken 10 times.
void fill_mask422_topleft_sse41<unsigned short, false>(unsigned short*, unsigned short const*, int, int, int, MagicDiv):
✓ Branch 56 → 53 taken 10 times.
✓ Branch 56 → 57 taken 10 times.
void fill_mask422_topleft_sse41<unsigned short, true>(unsigned short*, unsigned short const*, int, int, int, MagicDiv):
✓ Branch 24 → 23 taken 10 times.
✓ Branch 24 → 25 taken 10 times.
 80 for (; x < width; x++) {
399 40 const int val = src[x * 2];
400 40 dst[x] = full_opacity ? (pixel_t)val
401 40 : (pixel_t)magic_div_rt<pixel_t>((uint32_t)val * (uint32_t)opacity_i + (uint32_t)half, magic);
402 }
403 40 }
404
405 // ---------------------------------------------------------------------------
406 // MASK420_TOPLEFT — top-left co-sited point sample (top row only, no averaging).
407 // dst[x] = row0[x*2]
408 // ---------------------------------------------------------------------------
409 template<typename pixel_t, bool full_opacity>
410 #if defined(GCC) || defined(CLANG)
411 __attribute__((__target__("sse4.1")))
412 #endif
413 20 static void fill_mask420_topleft_sse41(
414 pixel_t* dst, const pixel_t* row0, int /*mask_pitch*/, int width,
415 int opacity_i, int half, MagicDiv magic)
416 {
417 // Identical to fill_mask422_topleft_sse41: top row only, left co-sited.
418 20 fill_mask422_topleft_sse41<pixel_t, full_opacity>(dst, row0, width, opacity_i, half, magic);
419 20 }
420
421 // ---------------------------------------------------------------------------
422 // MASK411 — horizontal 4-tap box average.
423 // avg[x] = (src[x*4]+src[x*4+1]+src[x*4+2]+src[x*4+3]+2) >> 2
424 // dst[x] = full_opacity ? avg : (avg * opacity_i + half) / max
425 // ---------------------------------------------------------------------------
426 template<typename pixel_t, bool full_opacity>
427 #if defined(GCC) || defined(CLANG)
428 __attribute__((__target__("sse4.1")))
429 #endif
430 ✗ static void fill_mask411_sse41(
431 pixel_t* dst, const pixel_t* src, int width,
432 int opacity_i, int half, MagicDiv magic)
433 {
434 ✗ int x = 0;
435 if constexpr (sizeof(pixel_t) == 1) {
436 ✗ const __m128i zero = _mm_setzero_si128();
437 ✗ [[maybe_unused]] const __m128i v_opacity = _mm_set1_epi16((short)opacity_i);
438 ✗ [[maybe_unused]] const __m128i v_half16 = _mm_set1_epi16((short)half);
439 ✗ [[maybe_unused]] const __m128i v_mdiv = _mm_set1_epi16((short)magic.div);
440 ✗ for (; x <= width - 8; x += 8) {
441 ✗ __m128i v0 = _mm_loadu_si128((const __m128i*)(src + x * 4));
442 ✗ __m128i v1 = _mm_loadu_si128((const __m128i*)(src + x * 4 + 16));
443 ✗ __m128i p0 = _mm_hadd_epi16(_mm_unpacklo_epi8(v0, zero), _mm_unpackhi_epi8(v0, zero));
444 ✗ __m128i p1 = _mm_hadd_epi16(_mm_unpacklo_epi8(v1, zero), _mm_unpackhi_epi8(v1, zero));
445 ✗ __m128i avg = _mm_srli_epi16(_mm_add_epi16(_mm_hadd_epi16(p0, p1), _mm_set1_epi16(2)), 2);
446 if constexpr (!full_opacity) {
447 ✗ __m128i scaled = _mm_add_epi16(_mm_mullo_epi16(avg, v_opacity), v_half16);
448 ✗ avg = _mm_srli_epi16(_mm_mulhi_epu16(scaled, v_mdiv), magic.shift);
449 }
450 ✗ _mm_storel_epi64((__m128i*)(dst + x), _mm_packus_epi16(avg, avg));
451 }
452 } else {
453 // 16-bit: unsigned widening to avoid signed overflow in madd_epi16
454 ✗ [[maybe_unused]] const __m128i v_opacity32 = _mm_set1_epi32(opacity_i);
455 ✗ [[maybe_unused]] const __m128i v_half32 = _mm_set1_epi32(half);
456 ✗ for (; x <= width - 4; x += 4) {
457 ✗ __m128i v0 = _mm_loadu_si128((const __m128i*)(src + x * 4)); // s0..s7
458 ✗ __m128i v1 = _mm_loadu_si128((const __m128i*)(src + x * 4 + 8)); // s8..s15
459 ✗ __m128i e0 = _mm_cvtepu16_epi32(v0);
460 ✗ __m128i e0h = _mm_cvtepu16_epi32(_mm_srli_si128(v0, 8));
461 ✗ __m128i e1 = _mm_cvtepu16_epi32(v1);
462 ✗ __m128i e1h = _mm_cvtepu16_epi32(_mm_srli_si128(v1, 8));
463 ✗ __m128i p01 = _mm_hadd_epi32(e0, e0h); // [s0+s1, s2+s3, s4+s5, s6+s7]
464 ✗ __m128i p23 = _mm_hadd_epi32(e1, e1h); // [s8+s9, s10+s11, s12+s13, s14+s15]
465 ✗ __m128i avg32 = _mm_srli_epi32(
466 _mm_add_epi32(_mm_hadd_epi32(p01, p23), _mm_set1_epi32(2)), 2);
467 if constexpr (!full_opacity)
468 ✗ avg32 = simd_magic_div_32(
469 _mm_add_epi32(_mm_mullo_epi32(avg32, v_opacity32), v_half32),
470 ✗ magic.div, magic.shift);
471 ✗ _mm_storel_epi64((__m128i*)(dst + x), _mm_packus_epi32(avg32, avg32));
472 }
473 }
474 ✗ for (; x < width; x++) {
475 ✗ const int avg = ((int)src[x*4] + src[x*4+1] + src[x*4+2] + src[x*4+3] + 2) >> 2;
476 ✗ dst[x] = full_opacity ? (pixel_t)avg
477 ✗ : (pixel_t)magic_div_rt<pixel_t>((uint32_t)avg * (uint32_t)opacity_i + (uint32_t)half, magic);
478 }
479 ✗ }
480
481 // ---------------------------
482 // Start of float mask helpers
483 // ---------------------------
484
485 // MASK422 — horizontal 2-tap box average.
486 // avg[x] = (src[x*2] + src[x*2+1]) * 0.5f
487 // ---------------------------------------------------------------------------
488 // MASK422 — horizontal 2-tap box average.
489 // float: 4 output pixels / iteration (8 input floats loaded)
490 // ---------------------------------------------------------------------------
491 template<bool full_opacity>
492 #if defined(GCC) || defined(CLANG)
493 __attribute__((__target__("sse4.1")))
494 #endif
495 10 static void fill_mask422_float_sse41(
496 float* dst, const float* src, int width, float opacity)
497 {
498 10 int x = 0;
499 10 const __m128 v_opacity = _mm_set1_ps(opacity);
500 10 const __m128 v_05 = _mm_set1_ps(0.5f);
501
502
4/4
void fill_mask422_float_sse41<false>(float*, float const*, int, float):
✓ Branch 19 → 7 taken 20 times.
✓ Branch 19 → 20 taken 5 times.
void fill_mask422_float_sse41<true>(float*, float const*, int, float):
✓ Branch 17 → 7 taken 20 times.
✓ Branch 17 → 18 taken 5 times.
 50 for (; x <= width - 4; x += 4) {
503 // 1. Load 8 floats into two 128-bit registers
504 40 __m128 r0 = _mm_loadu_ps(src + x * 2); // [e0, o0, e1, o1]
505 40 __m128 r1 = _mm_loadu_ps(src + x * 2 + 4); // [e2, o2, e3, o3]
506
507 // 2. De-interleave Even and Odd samples
508 // _mm_shuffle_ps(a, b, mask) takes 2 from a and 2 from b
509 // Mask (2, 0, 2, 0) -> a[0], a[2], b[0], b[2]
510 40 __m128 even = _mm_shuffle_ps(r0, r1, _MM_SHUFFLE(2, 0, 2, 0)); // [e0, e1, e2, e3]
511 40 __m128 odd = _mm_shuffle_ps(r0, r1, _MM_SHUFFLE(3, 1, 3, 1)); // [o0, o1, o2, o3]
512
513 // 3. Average: (even + odd) * 0.5
514 60 __m128 avg = _mm_mul_ps(_mm_add_ps(even, odd), v_05);
515
516 if constexpr (!full_opacity) {
517 20 avg = _mm_mul_ps(avg, v_opacity);
518 }
519
520 40 _mm_storeu_ps(dst + x, avg);
521 }
522
523
4/4
void fill_mask422_float_sse41<false>(float*, float const*, int, float):
✓ Branch 22 → 21 taken 5 times.
✓ Branch 22 → 23 taken 5 times.
void fill_mask422_float_sse41<true>(float*, float const*, int, float):
✓ Branch 20 → 19 taken 5 times.
✓ Branch 20 → 21 taken 5 times.
 20 for (; x < width; x++) {
524 10 const float avg = (src[x * 2] + src[x * 2 + 1]) * 0.5f;
525 10 dst[x] = full_opacity ? avg : avg * opacity;
526 }
527 10 }
528
529 // ---------------------------------------------------------------------------
530 // MASK422_MPEG2 — horizontal 3-tap triangle filter with sliding window carry.
531 // avg[x] = (left + 2*src[x*2] + src[x*2+1]) * 0.25f
532 // float: 4 output pixels / iteration (8 input floats loaded)
533 // ---------------------------------------------------------------------------
534 template<bool full_opacity>
535 #if defined(GCC) || defined(CLANG)
536 __attribute__((__target__("sse4.1")))
537 #endif
538 ✗ static void fill_mask422_mpeg2_float_sse41(
539 float* dst, const float* src, int width, float opacity)
540 {
541 ✗ int x = 0;
542 ✗ float right_val = src[0];
543
544 ✗ const __m128 v_opacity = _mm_set1_ps(opacity);
545 ✗ const __m128 v_025 = _mm_set1_ps(0.25f);
546
547 // v_prev_odd holds the odd samples from the previous iteration
548 ✗ __m128 v_prev_odd = _mm_set1_ps(right_val);
549
550 ✗ for (; x <= width - 4; x += 4) {
551 // 1. Load 8 floats
552 ✗ __m128 r0 = _mm_loadu_ps(src + x * 2);
553 ✗ __m128 r1 = _mm_loadu_ps(src + x * 2 + 4);
554
555 // 2. De-interleave
556 ✗ __m128 even = _mm_shuffle_ps(r0, r1, _MM_SHUFFLE(2, 0, 2, 0));
557 ✗ __m128 odd = _mm_shuffle_ps(r0, r1, _MM_SHUFFLE(3, 1, 3, 1));
558
559 // 3. Sliding Window Carry
560 // We want left = [p_o3, o0, o1, o2]
561 // alignr(curr, prev, 12 bytes) pulls 1 float from 'prev' and 3 from 'curr'
562 ✗ __m128 left = _mm_castsi128_ps(_mm_alignr_epi8(
563 _mm_castps_si128(odd),
564 _mm_castps_si128(v_prev_odd), 12));
565
566 // 4. Filter: avg = (left + 2*even + odd) * 0.25
567 ✗ __m128 avg = _mm_add_ps(left, _mm_add_ps(_mm_add_ps(even, even), odd));
568 ✗ avg = _mm_mul_ps(avg, v_025);
569
570 if constexpr (!full_opacity) {
571 ✗ avg = _mm_mul_ps(avg, v_opacity);
572 }
573
574 ✗ _mm_storeu_ps(dst + x, avg);
575
576 // 5. Update Carry: Simply store the current odd register
577 ✗ v_prev_odd = odd;
578 }
579
580 // Extraction: Get index 3 (the last odd sample) for the scalar tail
581 ✗ right_val = _mm_cvtss_f32(_mm_shuffle_ps(v_prev_odd, v_prev_odd, _MM_SHUFFLE(3, 3, 3, 3)));
582
583 ✗ for (; x < width; x++) {
584 ✗ const float left = right_val;
585 ✗ const float mid = src[x * 2];
586 ✗ right_val = src[x * 2 + 1];
587
588 ✗ const float avg = (left + 2.0f * mid + right_val) * 0.25f;
589 ✗ dst[x] = full_opacity ? avg : avg * opacity;
590 }
591 ✗ }
592
593 // ---------------------------------------------------------------------------
594 // MASK420 — 2x2 box average (MPEG-1 placement).
595 // SSE4.1 float: 4 output pixels / iteration (8 input floats per row)
596 // ---------------------------------------------------------------------------
597 template<bool full_opacity>
598 #if defined(GCC) || defined(CLANG)
599 __attribute__((__target__("sse4.1")))
600 #endif
601 10 static void fill_mask420_float_sse41(
602 float* dst, const float* row0, int mask_pitch, int width, float opacity)
603 {
604 10 const float* row1 = row0 + mask_pitch;
605 10 int x = 0;
606
607 10 const __m128 v_opacity = _mm_set1_ps(opacity);
608 10 const __m128 v_025 = _mm_set1_ps(0.25f);
609
610
4/4
void fill_mask420_float_sse41<false>(float*, float const*, int, int, float):
✓ Branch 27 → 7 taken 20 times.
✓ Branch 27 → 28 taken 5 times.
void fill_mask420_float_sse41<true>(float*, float const*, int, int, float):
✓ Branch 25 → 7 taken 20 times.
✓ Branch 25 → 26 taken 5 times.
 50 for (; x <= width - 4; x += 4) {
611 // 1. Load 8 floats from each row (2x 128-bit loads)
612 40 __m128 r0_0 = _mm_loadu_ps(row0 + x * 2);
613 40 __m128 r0_1 = _mm_loadu_ps(row0 + x * 2 + 4);
614 40 __m128 r1_0 = _mm_loadu_ps(row1 + x * 2);
615 80 __m128 r1_1 = _mm_loadu_ps(row1 + x * 2 + 4);
616
617 // 2. Vertical Sum
618 40 __m128 s0 = _mm_add_ps(r0_0, r1_0); // [e0, o0, e1, o1]
619 40 __m128 s1 = _mm_add_ps(r0_1, r1_1); // [e2, o2, e3, o3]
620
621 // 3. Horizontal Sum via De-interleave
622 // Shuffle picks [e0, e1] from s0 and [e2, e3] from s1
623 40 __m128 even = _mm_shuffle_ps(s0, s1, _MM_SHUFFLE(2, 0, 2, 0));
624 40 __m128 odd = _mm_shuffle_ps(s0, s1, _MM_SHUFFLE(3, 1, 3, 1));
625
626 // 4. Final Average: (even + odd) * 0.25
627 60 __m128 avg = _mm_mul_ps(_mm_add_ps(even, odd), v_025);
628
629 if constexpr (!full_opacity) {
630 20 avg = _mm_mul_ps(avg, v_opacity);
631 }
632
633 40 _mm_storeu_ps(dst + x, avg);
634 }
635
636 // Scalar Tail
637
4/4
void fill_mask420_float_sse41<false>(float*, float const*, int, int, float):
✓ Branch 30 → 29 taken 5 times.
✓ Branch 30 → 31 taken 5 times.
void fill_mask420_float_sse41<true>(float*, float const*, int, int, float):
✓ Branch 28 → 27 taken 5 times.
✓ Branch 28 → 29 taken 5 times.
 20 for (; x < width; x++) {
638 10 const float sum = row0[x * 2] + row0[x * 2 + 1] + row1[x * 2] + row1[x * 2 + 1];
639 10 const float avg = sum * 0.25f;
640 10 dst[x] = full_opacity ? avg : avg * opacity;
641 }
642 10 }
643
644 // ---------------------------------------------------------------------------
645 // MASK420_MPEG2 — horizontal 3-tap triangle filter with vertical 2-row sum.
646 // avg[x] = (po[x-1] + 2*pe[x] + po[x]) * 0.125f
647 // SSE4.1 float: 4 output pixels / iteration (8 input floats per row)
648 // ---------------------------------------------------------------------------
649 template<bool full_opacity>
650 #if defined(GCC) || defined(CLANG)
651 __attribute__((__target__("sse4.1")))
652 #endif
653 ✗ static void fill_mask420_mpeg2_float_sse41(
654 float* dst, const float* row0, int mask_pitch, int width, float opacity)
655 {
656 ✗ const float* row1 = row0 + mask_pitch;
657 ✗ int x = 0;
658
659 ✗ float right_val = row0[0] + row1[0];
660 ✗ __m128 v_prev_odd = _mm_set1_ps(right_val);
661 ✗ const __m128 v_opacity = _mm_set1_ps(opacity);
662 ✗ const __m128 v_0125 = _mm_set1_ps(0.125f);
663
664 ✗ for (; x <= width - 4; x += 4) {
665 // 1. Load 8 floats from each row
666 ✗ __m128 r0_0 = _mm_loadu_ps(row0 + x * 2);
667 ✗ __m128 r0_1 = _mm_loadu_ps(row0 + x * 2 + 4);
668 ✗ __m128 r1_0 = _mm_loadu_ps(row1 + x * 2);
669 ✗ __m128 r1_1 = _mm_loadu_ps(row1 + x * 2 + 4);
670
671 // 2. Vertical Sum (pe and po interleaved)
672 ✗ __m128 s0 = _mm_add_ps(r0_0, r1_0);
673 ✗ __m128 s1 = _mm_add_ps(r0_1, r1_1);
674
675 // 3. De-interleave Even and Odd
676 ✗ __m128 even = _mm_shuffle_ps(s0, s1, _MM_SHUFFLE(2, 0, 2, 0));
677 ✗ __m128 odd = _mm_shuffle_ps(s0, s1, _MM_SHUFFLE(3, 1, 3, 1));
678
679 // 4. Construct 'left' vector: [prev_o3, o0, o1, o2]
680 // alignr 12 bytes = shift right by 3 floats (leaving 1 from v_prev_odd)
681 ✗ __m128 left = _mm_castsi128_ps(_mm_alignr_epi8(
682 _mm_castps_si128(odd),
683 _mm_castps_si128(v_prev_odd), 12));
684
685 // 5. Triangle Filter: (left + 2*even + odd) * 0.125
686 ✗ __m128 avg = _mm_mul_ps(
687 _mm_add_ps(_mm_add_ps(left, _mm_add_ps(even, even)), odd),
688 v_0125
689 );
690
691 if constexpr (!full_opacity) {
692 ✗ avg = _mm_mul_ps(avg, v_opacity);
693 }
694
695 ✗ _mm_storeu_ps(dst + x, avg);
696
697 // Update carry: store the current odd register
698 ✗ v_prev_odd = odd;
699 }
700
701 // Bridge to scalar tail: Extract index 3 (last odd sample)
702 ✗ right_val = _mm_cvtss_f32(_mm_shuffle_ps(v_prev_odd, v_prev_odd, _MM_SHUFFLE(3, 3, 3, 3)));
703
704 ✗ for (; x < width; x++) {
705 ✗ const float left = right_val;
706 ✗ const float mid = row0[x * 2] + row1[x * 2];
707 ✗ right_val = row0[x * 2 + 1] + row1[x * 2 + 1];
708 ✗ const float avg = (left + 2.0f * mid + right_val) * 0.125f;
709 ✗ dst[x] = full_opacity ? avg : avg * opacity;
710 }
711 ✗ }
712
713 // ---------------------------------------------------------------------------
714 // MASK422_TOPLEFT — left co-sited point sample (no averaging).
715 // dst[x] = src[x*2]
716 // SSE4.1 float: 4 output pixels / iteration (8 input floats loaded)
717 // ---------------------------------------------------------------------------
718 template<bool full_opacity>
719 #if defined(GCC) || defined(CLANG)
720 __attribute__((__target__("sse4.1")))
721 #endif
722 ✗ static void fill_mask422_topleft_float_sse41(
723 float* dst, const float* src, int width, float opacity)
724 {
725 ✗ int x = 0;
726 ✗ const __m128 v_opacity = _mm_set1_ps(opacity);
727
728 ✗ for (; x <= width - 4; x += 4) {
729 // 1. Load 8 floats
730 ✗ __m128 r0 = _mm_loadu_ps(src + x * 2); // [e0, o0, e1, o1]
731 ✗ __m128 r1 = _mm_loadu_ps(src + x * 2 + 4); // [e2, o2, e3, o3]
732
733 // 2. Shuffle to pick only the Even samples
734 ✗ __m128 even = _mm_shuffle_ps(r0, r1, _MM_SHUFFLE(2, 0, 2, 0)); // [e0, e1, e2, e3]
735
736 if constexpr (!full_opacity) {
737 ✗ even = _mm_mul_ps(even, v_opacity);
738 }
739
740 ✗ _mm_storeu_ps(dst + x, even);
741 }
742
743 // Scalar Tail
744 ✗ for (; x < width; x++) {
745 ✗ const float val = src[x * 2];
746 ✗ dst[x] = full_opacity ? val : val * opacity;
747 }
748 ✗ }
749
750 // ---------------------------------------------------------------------------
751 // MASK420_TOPLEFT — top-left co-sited point sample (top row only, no averaging).
752 // dst[x] = row0[x*2]
753 // ---------------------------------------------------------------------------
754 template<bool full_opacity>
755 ✗ static void fill_mask420_topleft_float_sse41(
756 float* dst, const float* row0, int /*mask_pitch*/, int width, float opacity)
757 {
758 // 420_TOPLEFT is identical to 422_TOPLEFT for the top row
759 ✗ fill_mask422_topleft_float_sse41<full_opacity>(dst, row0, width, opacity);
760 ✗ }
761
762 // ---------------------------------------------------------------------------
763 // MASK411 — horizontal 4-tap box average.
764 // avg[x] = (src[x*4]+src[x*4+1]+src[x*4+2]+src[x*4+3]) / 4 (*0.25)
765 // SSE4.1 float: 4 output pixels / iteration (16 input floats loaded)
766 // ---------------------------------------------------------------------------
767 template<bool full_opacity>
768 #if defined(GCC) || defined(CLANG)
769 __attribute__((__target__("sse4.1")))
770 #endif
771 ✗ static void fill_mask411_float_sse41(
772 float* dst, const float* src, int width, float opacity)
773 {
774 ✗ int x = 0;
775 ✗ const __m128 v_opacity = _mm_set1_ps(opacity);
776 ✗ const __m128 v_025 = _mm_set1_ps(0.25f);
777
778 ✗ for (; x <= width - 4; x += 4) {
779 // 1. Load 16 floats (4 input blocks)
780 ✗ __m128 r0 = _mm_loadu_ps(src + x * 4); // [s0, s1, s2, s3]
781 ✗ __m128 r1 = _mm_loadu_ps(src + x * 4 + 4); // [s4, s5, s6, s7]
782 ✗ __m128 r2 = _mm_loadu_ps(src + x * 4 + 8); // [s8, s9, s10, s11]
783 ✗ __m128 r3 = _mm_loadu_ps(src + x * 4 + 12); // [s12, s13, s14, s15]
784
785 // 2. Horizontal sum pass 1: [s0+s1, s2+s3, s4+s5, s6+s7]
786 ✗ __m128 h01 = _mm_hadd_ps(r0, r1);
787 ✗ __m128 h23 = _mm_hadd_ps(r2, r3);
788
789 // 3. Horizontal sum pass 2: Sum the pairs to get 4x-pixel sums
790 // [(s0..s3), (s4..s7), (s8..s11), (s12..s15)]
791 ✗ __m128 sum = _mm_hadd_ps(h01, h23);
792
793 ✗ __m128 avg = _mm_mul_ps(sum, v_025);
794
795 if constexpr (!full_opacity) {
796 ✗ avg = _mm_mul_ps(avg, v_opacity);
797 }
798
799 ✗ _mm_storeu_ps(dst + x, avg);
800 }
801
802 // Scalar Tail
803 ✗ for (; x < width; x++) {
804 ✗ const float avg = (src[x * 4] + src[x * 4 + 1] + src[x * 4 + 2] + src[x * 4 + 3]) * 0.25f;
805 ✗ dst[x] = full_opacity ? avg : avg * opacity;
806 }
807 ✗ }
808
809 // ---------------------------------------------------------------------------
810 // prepare_effective_mask_for_row_sse41
811 // full_opacity == true (default): MASK444 returns maskp; others fill buf with
812 // spatial averages only.
813 // full_opacity == false: opacity baked in for every mode including MASK444.
814 // opacity_i, half, magic are ignored when full_opacity == true.
815 // ---------------------------------------------------------------------------
816 template<MaskMode maskMode, typename pixel_t, bool full_opacity>
817 #if defined(GCC) || defined(CLANG)
818 __attribute__((__target__("sse4.1")))
819 #endif
820 352 const pixel_t* prepare_effective_mask_for_row_sse41(
821 const pixel_t* maskp,
822 int mask_pitch,
823 int width,
824 std::vector<pixel_t>& buf,
825 int opacity_i,
826 int half,
827 MagicDiv magic)
828 {
829 if constexpr (maskMode == MASK444) {
830 if constexpr (full_opacity) {
831 77 return maskp;
832 } else {
833 // Copy row with opacity scaling into buf
834 98 pixel_t* dst = buf.data();
835 98 int x = 0;
836 if constexpr (sizeof(pixel_t) == 1) {
837 43 const __m128i v_opacity = _mm_set1_epi16((short)opacity_i);
838 43 const __m128i v_half16 = _mm_set1_epi16((short)half);
839 43 const __m128i v_mdiv = _mm_set1_epi16((short)magic.div);
840
2/2
✓ Branch 36 → 16 taken 162 times.
✓ Branch 36 → 37 taken 43 times.
 205 for (; x <= width - 8; x += 8) {
841 324 __m128i v = _mm_loadl_epi64((const __m128i*)(maskp + x));
842 162 __m128i v16 = _mm_cvtepu8_epi16(v);
843 162 __m128i scaled = _mm_add_epi16(_mm_mullo_epi16(v16, v_opacity), v_half16);
844 486 __m128i res = _mm_srli_epi16(_mm_mulhi_epu16(scaled, v_mdiv), magic.shift);
845 162 _mm_storel_epi64((__m128i*)(dst + x), _mm_packus_epi16(res, res));
846 }
847 } else {
848 55 const __m128i v_opacity32 = _mm_set1_epi32(opacity_i);
849 55 const __m128i v_half32 = _mm_set1_epi32(half);
850
2/2
✓ Branch 50 → 12 taken 210 times.
✓ Branch 50 → 51 taken 55 times.
 265 for (; x <= width - 4; x += 4) {
851 420 __m128i v32 = _mm_cvtepu16_epi32(_mm_loadl_epi64((const __m128i*)(maskp + x)));
852 630 __m128i res = simd_magic_div_32(
853 _mm_add_epi32(_mm_mullo_epi32(v32, v_opacity32), v_half32),
854 210 magic.div, magic.shift);
855 210 _mm_storel_epi64((__m128i*)(dst + x), _mm_packus_epi32(res, res));
856 }
857 }
858
4/4
unsigned char const* prepare_effective_mask_for_row_sse41<(MaskMode)7, unsigned char, false>(unsigned char const*, int, int, std::vector<unsigned char, std::allocator<unsigned char> >&, int, int, MagicDiv):
✓ Branch 41 → 38 taken 195 times.
✓ Branch 41 → 42 taken 43 times.
unsigned short const* prepare_effective_mask_for_row_sse41<(MaskMode)7, unsigned short, false>(unsigned short const*, int, int, std::vector<unsigned short, std::allocator<unsigned short> >&, int, int, MagicDiv):
✓ Branch 55 → 52 taken 155 times.
✓ Branch 55 → 56 taken 55 times.
 448 for (; x < width; x++)
859 350 dst[x] = static_cast<pixel_t>(
860 700 magic_div_rt<pixel_t>((uint32_t)maskp[x] * (uint32_t)opacity_i + (uint32_t)half, magic));
861 98 return dst;
862 }
863 }
864 else {
865 177 pixel_t* dst = buf.data();
866 if constexpr (maskMode == MASK422)
867 45 fill_mask422_sse41<pixel_t, full_opacity>(dst, maskp, width, opacity_i, half, magic);
868 else if constexpr (maskMode == MASK422_MPEG2)
869 20 fill_mask422_mpeg2_sse41<pixel_t, full_opacity>(dst, maskp, width, opacity_i, half, magic);
870 else if constexpr (maskMode == MASK422_TOPLEFT)
871 20 fill_mask422_topleft_sse41<pixel_t, full_opacity>(dst, maskp, width, opacity_i, half, magic);
872 else if constexpr (maskMode == MASK420)
873 45 fill_mask420_sse41<pixel_t, full_opacity>(dst, maskp, mask_pitch, width, opacity_i, half, magic);
874 else if constexpr (maskMode == MASK420_MPEG2)
875 27 fill_mask420_mpeg2_sse41<pixel_t, full_opacity>(dst, maskp, mask_pitch, width, opacity_i, half, magic);
876 else if constexpr (maskMode == MASK420_TOPLEFT)
877 20 fill_mask420_topleft_sse41<pixel_t, full_opacity>(dst, maskp, mask_pitch, width, opacity_i, half, magic);
878 else if constexpr (maskMode == MASK411)
879 ✗ fill_mask411_sse41<pixel_t, full_opacity>(dst, maskp, width, opacity_i, half, magic);
880 177 return dst;
881 }
882 }
883
884 template<MaskMode maskMode, bool full_opacity>
885 #if defined(GCC) || defined(CLANG)
886 __attribute__((__target__("sse4.1")))
887 #endif
888 30 AVS_FORCEINLINE const float* prepare_effective_mask_for_row_float_sse41(
889 const float* maskp,
890 int mask_pitch,
891 int width,
892 std::vector<float>& buf,
893 float opacity)
894 {
895 if constexpr (maskMode == MASK444) {
896 if constexpr (full_opacity) {
897 5 return maskp;
898 }
899 else {
900 5 float* dst = buf.data();
901 5 int x = 0;
902 5 const __m128 v_opacity = _mm_set1_ps(opacity);
903
2/2
✓ Branch 12 → 6 taken 15 times.
✓ Branch 12 → 13 taken 5 times.
 20 for (; x <= width - 4; x += 4) {
904 // just put back opacity * mask
905 30 __m128 v16 = _mm_loadu_ps(maskp + x);
906 15 __m128 scaled = _mm_mul_ps(v16, v_opacity);
907 15 _mm_storeu_ps(dst + x, scaled);
908 }
909
1/2
✗ Branch 22 → 14 not taken.
✓ Branch 22 → 23 taken 5 times.
 5 for (; x <= width - 4; x += 4) {
910 // just put back opacity * mask
911 ✗ __m128 v16 = _mm_loadu_ps(maskp + x);
912 ✗ __m128 scaled = _mm_mul_ps(v16, _mm_set1_ps(opacity));
913 ✗ _mm_storeu_ps(dst + x, scaled);
914 }
915
2/2
✓ Branch 25 → 24 taken 5 times.
✓ Branch 25 → 26 taken 5 times.
 10 for (; x < width; x++)
916 5 dst[x] = maskp[x] * opacity;
917 5 return dst;
918 }
919 }
920 else {
921 20 float* dst = buf.data();
922 if constexpr (maskMode == MASK422)
923 10 fill_mask422_float_sse41<full_opacity>(dst, maskp, width, opacity);
924 else if constexpr (maskMode == MASK422_MPEG2)
925 ✗ fill_mask422_mpeg2_float_sse41<full_opacity>(dst, maskp, width, opacity);
926 else if constexpr (maskMode == MASK422_TOPLEFT)
927 ✗ fill_mask422_topleft_float_sse41<full_opacity>(dst, maskp, width, opacity);
928 else if constexpr (maskMode == MASK420)
929 10 fill_mask420_float_sse41<full_opacity>(dst, maskp, mask_pitch, width, opacity);
930 else if constexpr (maskMode == MASK420_MPEG2)
931 ✗ fill_mask420_mpeg2_float_sse41<full_opacity>(dst, maskp, mask_pitch, width, opacity);
932 else if constexpr (maskMode == MASK420_TOPLEFT)
933 ✗ fill_mask420_topleft_float_sse41<full_opacity>(dst, maskp, mask_pitch, width, opacity);
934 else if constexpr (maskMode == MASK411)
935 ✗ fill_mask411_float_sse41<full_opacity>(dst, maskp, width, opacity);
936 20 return dst;
937 }
938 }
939
940
941 // ---------------------------------------------------------------------------
942 // Explicit instantiations
943 // ---------------------------------------------------------------------------
944
945 // prepare_effective_mask_for_row_sse41
946 #define INST_PREP_SSE41(mm, pt) \
947 template const pt* prepare_effective_mask_for_row_sse41<mm, pt, true> (const pt*, int, int, std::vector<pt>&, int, int, MagicDiv); \
948 template const pt* prepare_effective_mask_for_row_sse41<mm, pt, false>(const pt*, int, int, std::vector<pt>&, int, int, MagicDiv);
949 INST_PREP_SSE41(MASK444, uint8_t) INST_PREP_SSE41(MASK444, uint16_t)
950 INST_PREP_SSE41(MASK420, uint8_t) INST_PREP_SSE41(MASK420, uint16_t)
951 INST_PREP_SSE41(MASK420_MPEG2, uint8_t) INST_PREP_SSE41(MASK420_MPEG2, uint16_t)
952 INST_PREP_SSE41(MASK420_TOPLEFT, uint8_t) INST_PREP_SSE41(MASK420_TOPLEFT, uint16_t)
953 INST_PREP_SSE41(MASK422, uint8_t) INST_PREP_SSE41(MASK422, uint16_t)
954 INST_PREP_SSE41(MASK422_MPEG2, uint8_t) INST_PREP_SSE41(MASK422_MPEG2, uint16_t)
955 INST_PREP_SSE41(MASK422_TOPLEFT, uint8_t) INST_PREP_SSE41(MASK422_TOPLEFT, uint16_t)
956 INST_PREP_SSE41(MASK411, uint8_t) INST_PREP_SSE41(MASK411, uint16_t)
957 #undef INST_PREP_SSE41
958
959 // prepare_effective_mask_for_row_float_sse41
960 #define INST_PREP_SSE41(mm) \
961 template const float* prepare_effective_mask_for_row_float_sse41<mm, true> (const float*, int, int, std::vector<float>&, float); \
962 template const float* prepare_effective_mask_for_row_float_sse41<mm, false>(const float*, int, int, std::vector<float>&, float);
963 INST_PREP_SSE41(MASK444)
964 INST_PREP_SSE41(MASK420)
965 INST_PREP_SSE41(MASK420_MPEG2)
966 INST_PREP_SSE41(MASK420_TOPLEFT)
967 INST_PREP_SSE41(MASK422)
968 INST_PREP_SSE41(MASK422_MPEG2)
969 INST_PREP_SSE41(MASK422_TOPLEFT)
970 INST_PREP_SSE41(MASK411)
971 #undef INST_PREP_SSE41
972
973