GCC Code Coverage Report

Directory: avs_core/
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 100.0% 468 / 0 / 468
Functions: 100.0% 20 / 0 / 20
Branches: 92.9% 78 / 0 / 84
filters/intel/planeswap_avx2.cpp
Line Branch Exec Source
1 // AviSynth+. Copyright 2026- AviSynth+ Project
2 // https://avs-plus.net
3 // http://avisynth.nl
4 // This program is free software; you can redistribute it and/or modify
5 // it under the terms of the GNU General Public License as published by
6 // the Free Software Foundation; either version 2 of the License, or
7 // (at your option) any later version.
8 //
9 // This program is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 // GNU General Public License for more details.
13 //
14 // You should have received a copy of the GNU General Public License
15 // along with this program; if not, write to the Free Software
16 // Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA, or visit
17 // http://www.gnu.org/copyleft/gpl.html .
18 //
19 // Linking Avisynth statically or dynamically with other modules is making a
20 // combined work based on Avisynth. Thus, the terms and conditions of the GNU
21 // General Public License cover the whole combination.
22 //
23 // As a special exception, the copyright holders of Avisynth give you
24 // permission to link Avisynth with independent modules that communicate with
25 // Avisynth solely through the interfaces defined in avisynth.h, regardless of the license
26 // terms of these independent modules, and to copy and distribute the
27 // resulting combined work under terms of your choice, provided that
28 // every copy of the combined work is accompanied by a complete copy of
29 // the source code of Avisynth (the version of Avisynth used to produce the
30 // combined work), being distributed under the terms of the GNU General
31 // Public License plus this exception. An independent module is a module
32 // which is not derived from or based on Avisynth, such as 3rd-party filters,
33 // import and export plugins, or graphical user interfaces.
34
35 #include <avs/config.h>
36 #ifdef AVS_WINDOWS
37 #include <avs/win.h>
38 #else
39 #include <avs/posix.h>
40 #endif
41
42 #include "../planeswap.h"
43 #include "planeswap_avx2.h"
44
45 #if defined(_MSC_VER)
46 #include <intrin.h> // MSVC
47 #else
48 #include <x86intrin.h> // GCC/MinGW/Clang/LLVM
49 #endif
50 #include <immintrin.h>
51
52 #include <stdint.h>
53 #include <type_traits>
54
55 // _mm256_set_m128i is not universally available as an intrinsic; define fallback.
56 #ifndef _mm256_set_m128i
57 #define _mm256_set_m128i(hi, lo) \
58 _mm256_inserti128_si256(_mm256_castsi128_si256(lo), (hi), 1)
59 #endif
60
61 // ---------------------------------------------------------------------------
62 // Packed RGB32 channel extraction — AVX2, 32 pixels per main iteration.
63 //
64 // Source layout (per pixel, 4 bytes): B G R A (channel_index: B=0 G=1 R=2 A=3)
65 // Source rows are bottom-up; srcp must already point at the last (bottom) row.
66 // Pitch is positive; we step backwards (srcp -= src_pitch) each row.
67 //
68 // Strategy:
69 // 1. vpshufb: move target byte to byte 0 of each 32-bit dword, zero bytes 1-3.
70 // (Mask is the same for both 128-bit lanes — pixel layout is identical.)
71 // 2. vpackssdw × 2: pack 4+4 dwords → 8 words (per register pair).
72 // 3. vpackuswb: pack the two word registers → 32 bytes, still interleaved
73 // in 4-byte groups due to in-lane packing.
74 // 4. vpermd with indices [0,4,1,5,2,6,3,7]: restore sequential pixel order.
75 //
76 // Width is guaranteed a multiple of 16 (64-byte pitch / 4 bytes per pixel).
77 // Main loop handles multiples of 32; SSE2/SSSE3 tail handles the remaining 16.
78 // ---------------------------------------------------------------------------
79 template<int channel_index>
80 8 void extract_packed_rgb32_channel_avx2(
81 const BYTE* srcp, BYTE* dstp, int src_pitch, int dst_pitch, int width, int height)
82 {
83 // Per-lane shuffle mask (repeated for both lanes):
84 // channel byte → position 0 of each 4-byte pixel dword; bytes 1-3 → 0x80 (zero).
85 8 constexpr int c = channel_index;
86 8 const __m256i shuf256 = _mm256_set_epi8(
87 (char)0x80,(char)0x80,(char)0x80,(char)(c+12),
88 (char)0x80,(char)0x80,(char)0x80,(char)(c+ 8),
89 (char)0x80,(char)0x80,(char)0x80,(char)(c+ 4),
90 (char)0x80,(char)0x80,(char)0x80,(char)(c+ 0),
91 (char)0x80,(char)0x80,(char)0x80,(char)(c+12),
92 (char)0x80,(char)0x80,(char)0x80,(char)(c+ 8),
93 (char)0x80,(char)0x80,(char)0x80,(char)(c+ 4),
94 (char)0x80,(char)0x80,(char)0x80,(char)(c+ 0)
95 );
96 // Dword permutation to fix pack interleaving:
97 // After packs+packus, dword order is [px0-3, px8-11, px16-19, px24-27, px4-7, px12-15, px20-23, px28-31].
98 // Indices [0,4,1,5,2,6,3,7] restore sequential order.
99 8 const __m256i perm = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
100
101 // SSE tail: same shuffle but 128-bit (16 pixels, processed with SSE packs — no lane issue).
102 8 const __m128i shuf128 = _mm_set_epi8(
103 (char)0x80,(char)0x80,(char)0x80,(char)(c+12),
104 (char)0x80,(char)0x80,(char)0x80,(char)(c+ 8),
105 (char)0x80,(char)0x80,(char)0x80,(char)(c+ 4),
106 (char)0x80,(char)0x80,(char)0x80,(char)(c+ 0)
107 );
108
109
8/8
void extract_packed_rgb32_channel_avx2<0>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 64 → 9 taken 12 times.
✓ Branch 64 → 65 taken 2 times.
void extract_packed_rgb32_channel_avx2<1>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 64 → 9 taken 12 times.
✓ Branch 64 → 65 taken 2 times.
void extract_packed_rgb32_channel_avx2<2>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 64 → 9 taken 12 times.
✓ Branch 64 → 65 taken 2 times.
void extract_packed_rgb32_channel_avx2<3>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 64 → 9 taken 12 times.
✓ Branch 64 → 65 taken 2 times.
 56 for (int y = 0; y < height; ++y) {
110 48 int x = 0;
111 // AVX2 main loop: 32 pixels (128 source bytes → 32 output bytes) per iteration.
112
8/8
void extract_packed_rgb32_channel_avx2<0>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 36 → 10 taken 12 times.
✓ Branch 36 → 37 taken 12 times.
void extract_packed_rgb32_channel_avx2<1>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 36 → 10 taken 12 times.
✓ Branch 36 → 37 taken 12 times.
void extract_packed_rgb32_channel_avx2<2>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 36 → 10 taken 12 times.
✓ Branch 36 → 37 taken 12 times.
void extract_packed_rgb32_channel_avx2<3>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 36 → 10 taken 12 times.
✓ Branch 36 → 37 taken 12 times.
 96 for (; x <= width - 32; x += 32) {
113 48 const BYTE* s = srcp + x * 4;
114 48 __m256i v0 = _mm256_load_si256(reinterpret_cast<const __m256i*>(s + 0)); // px 0-7
115 48 __m256i v1 = _mm256_load_si256(reinterpret_cast<const __m256i*>(s + 32)); // px 8-15
116 48 __m256i v2 = _mm256_load_si256(reinterpret_cast<const __m256i*>(s + 64)); // px 16-23
117 96 __m256i v3 = _mm256_load_si256(reinterpret_cast<const __m256i*>(s + 96)); // px 24-31
118 48 v0 = _mm256_shuffle_epi8(v0, shuf256);
119 48 v1 = _mm256_shuffle_epi8(v1, shuf256);
120 48 v2 = _mm256_shuffle_epi8(v2, shuf256);
121 48 v3 = _mm256_shuffle_epi8(v3, shuf256);
122 // Pack dwords → words → bytes (in-lane; order fixed by vpermd below).
123 48 __m256i p01 = _mm256_packs_epi32(v0, v1);
124 48 __m256i p23 = _mm256_packs_epi32(v2, v3);
125 48 __m256i packed = _mm256_packus_epi16(p01, p23);
126 48 packed = _mm256_permutevar8x32_epi32(packed, perm);
127 48 _mm256_store_si256(reinterpret_cast<__m256i*>(dstp + x), packed);
128 }
129 // SSE tail: at most 16 pixels remain (width is a multiple of 16).
130
8/8
void extract_packed_rgb32_channel_avx2<0>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 62 → 38 taken 12 times.
✓ Branch 62 → 63 taken 12 times.
void extract_packed_rgb32_channel_avx2<1>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 62 → 38 taken 12 times.
✓ Branch 62 → 63 taken 12 times.
void extract_packed_rgb32_channel_avx2<2>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 62 → 38 taken 12 times.
✓ Branch 62 → 63 taken 12 times.
void extract_packed_rgb32_channel_avx2<3>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 62 → 38 taken 12 times.
✓ Branch 62 → 63 taken 12 times.
 96 for (; x < width; x += 16) {
131 48 const BYTE* s = srcp + x * 4;
132 48 __m128i t0 = _mm_load_si128(reinterpret_cast<const __m128i*>(s + 0));
133 48 __m128i t1 = _mm_load_si128(reinterpret_cast<const __m128i*>(s + 16));
134 48 __m128i t2 = _mm_load_si128(reinterpret_cast<const __m128i*>(s + 32));
135 96 __m128i t3 = _mm_load_si128(reinterpret_cast<const __m128i*>(s + 48));
136 48 t0 = _mm_shuffle_epi8(t0, shuf128);
137 48 t1 = _mm_shuffle_epi8(t1, shuf128);
138 96 t2 = _mm_shuffle_epi8(t2, shuf128);
139 48 t3 = _mm_shuffle_epi8(t3, shuf128);
140 144 _mm_store_si128(reinterpret_cast<__m128i*>(dstp + x),
141 _mm_packus_epi16(_mm_packs_epi32(t0, t1), _mm_packs_epi32(t2, t3)));
142 }
143 48 srcp -= src_pitch;
144 48 dstp += dst_pitch;
145 }
146 8 }
147
148 template void extract_packed_rgb32_channel_avx2<0>(const BYTE*, BYTE*, int, int, int, int);
149 template void extract_packed_rgb32_channel_avx2<1>(const BYTE*, BYTE*, int, int, int, int);
150 template void extract_packed_rgb32_channel_avx2<2>(const BYTE*, BYTE*, int, int, int, int);
151 template void extract_packed_rgb32_channel_avx2<3>(const BYTE*, BYTE*, int, int, int, int);
152
153 // ---------------------------------------------------------------------------
154 // Packed RGB64 channel extraction — AVX2, 16 pixels per main iteration.
155 //
156 // Source layout (per pixel, 8 bytes): B G R A (uint16_t each; B=0 G=1 R=2 A=3)
157 // Source rows are bottom-up; srcp must already point at the last (bottom) row.
158 //
159 // Per 16-byte lane (2 pixels): channel word at bytes 2c, 2c+1 (px0) and 2c+8, 2c+9 (px1).
160 //
161 // Strategy:
162 // 1. vpshufb: extract the two uint16 channel words from each 2-pixel pair
163 // into bytes 0-3 of each lane, zero the rest.
164 // Per lane result: [C0lo,C0hi, C1lo,C1hi, 0,...,0]
165 // 2. vpunpckldq × 2: interleave dwords from matching lanes across two registers.
166 // 3. vpunpcklqdq: take the valid 64-bit halves of each lane.
167 // After steps 2-3 we have all 16 channel words in the register but still
168 // interleaved in dword-pair groups, mirroring the RGB32 pattern.
169 // 4. vpermd with indices [0,4,1,5,2,6,3,7]: restore sequential pixel order.
170 //
171 // Width is guaranteed a multiple of 8 (64-byte pitch / 8 bytes per pixel).
172 // Main loop handles multiples of 16; SSE tail handles the remaining 0 or 8 pixels.
173 // ---------------------------------------------------------------------------
174 template<int channel_index>
175 8 void extract_packed_rgb64_channel_avx2(
176 const BYTE* srcp, BYTE* dstp, int src_pitch, int dst_pitch, int width, int height)
177 {
178 8 constexpr int c = channel_index;
179 // Shuffle mask for one 16-byte lane (2 pixels × 8 bytes each):
180 // target channel is at byte offset 2c (lo) and 2c+1 (hi) within each 8-byte pixel.
181 // Extract words from pixel 0 (offset 0) and pixel 1 (offset 8) to bytes 0-3.
182 8 const __m256i shuf256 = _mm256_set_epi8(
183 (char)0x80,(char)0x80,(char)0x80,(char)0x80,
184 (char)0x80,(char)0x80,(char)0x80,(char)0x80,
185 (char)0x80,(char)0x80,(char)0x80,(char)0x80,
186 (char)(2*c+9),(char)(2*c+8),(char)(2*c+1),(char)(2*c+0),
187 (char)0x80,(char)0x80,(char)0x80,(char)0x80,
188 (char)0x80,(char)0x80,(char)0x80,(char)0x80,
189 (char)0x80,(char)0x80,(char)0x80,(char)0x80,
190 (char)(2*c+9),(char)(2*c+8),(char)(2*c+1),(char)(2*c+0)
191 );
192 8 const __m256i perm = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0);
193
194 // SSE tail mask (same pattern, 128-bit).
195 8 const __m128i shuf128 = _mm_set_epi8(
196 (char)0x80,(char)0x80,(char)0x80,(char)0x80,
197 (char)0x80,(char)0x80,(char)0x80,(char)0x80,
198 (char)0x80,(char)0x80,(char)0x80,(char)0x80,
199 (char)(2*c+9),(char)(2*c+8),(char)(2*c+1),(char)(2*c+0)
200 );
201
202
8/8
void extract_packed_rgb64_channel_avx2<0>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 64 → 9 taken 12 times.
✓ Branch 64 → 65 taken 2 times.
void extract_packed_rgb64_channel_avx2<1>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 64 → 9 taken 12 times.
✓ Branch 64 → 65 taken 2 times.
void extract_packed_rgb64_channel_avx2<2>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 64 → 9 taken 12 times.
✓ Branch 64 → 65 taken 2 times.
void extract_packed_rgb64_channel_avx2<3>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 64 → 9 taken 12 times.
✓ Branch 64 → 65 taken 2 times.
 56 for (int y = 0; y < height; ++y) {
203 48 int x = 0;
204 // AVX2 main loop: 16 pixels (128 source bytes → 32 output bytes) per iteration.
205
8/8
void extract_packed_rgb64_channel_avx2<0>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 36 → 10 taken 12 times.
✓ Branch 36 → 37 taken 12 times.
void extract_packed_rgb64_channel_avx2<1>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 36 → 10 taken 12 times.
✓ Branch 36 → 37 taken 12 times.
void extract_packed_rgb64_channel_avx2<2>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 36 → 10 taken 12 times.
✓ Branch 36 → 37 taken 12 times.
void extract_packed_rgb64_channel_avx2<3>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 36 → 10 taken 12 times.
✓ Branch 36 → 37 taken 12 times.
 96 for (; x <= width - 16; x += 16) {
206 48 const BYTE* s = srcp + x * 8;
207 48 __m256i v0 = _mm256_load_si256(reinterpret_cast<const __m256i*>(s + 0)); // px 0-3
208 48 __m256i v1 = _mm256_load_si256(reinterpret_cast<const __m256i*>(s + 32)); // px 4-7
209 48 __m256i v2 = _mm256_load_si256(reinterpret_cast<const __m256i*>(s + 64)); // px 8-11
210 96 __m256i v3 = _mm256_load_si256(reinterpret_cast<const __m256i*>(s + 96)); // px 12-15
211 // After shuffle, each lane: [C_even_lo, C_even_hi, C_odd_lo, C_odd_hi, 0...0]
212 // i.e., dword 0 of each lane holds the 2 channel uint16 values for that pixel pair.
213 48 v0 = _mm256_shuffle_epi8(v0, shuf256);
214 48 v1 = _mm256_shuffle_epi8(v1, shuf256);
215 48 v2 = _mm256_shuffle_epi8(v2, shuf256);
216 48 v3 = _mm256_shuffle_epi8(v3, shuf256);
217 // Interleave valid dwords from matching lanes: result dwords = [{C0,C1},{C4,C5},0,0 | {C2,C3},{C6,C7},0,0]
218 48 __m256i p01 = _mm256_unpacklo_epi32(v0, v1);
219 48 __m256i p23 = _mm256_unpacklo_epi32(v2, v3);
220 // Take the valid 64-bit half from each lane of both registers.
221 // lo lane: [{C0,C1},{C4,C5}, {C8,C9},{C12,C13}]
222 // hi lane: [{C2,C3},{C6,C7}, {C10,C11},{C14,C15}]
223 48 __m256i gathered = _mm256_unpacklo_epi64(p01, p23);
224 // Fix interleaving: dwords = [{C0,C1},{C4,C5},{C8,C9},{C12,C13}, {C2,C3},{C6,C7},{C10,C11},{C14,C15}]
225 // Want sequential: [{C0,C1},{C2,C3},{C4,C5},{C6,C7}, {C8,C9},{C10,C11},{C12,C13},{C14,C15}]
226 48 gathered = _mm256_permutevar8x32_epi32(gathered, perm);
227 48 _mm256_store_si256(reinterpret_cast<__m256i*>(dstp + x * 2), gathered);
228 }
229 // SSE tail: at most 8 pixels remain (width is a multiple of 8).
230
8/8
void extract_packed_rgb64_channel_avx2<0>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 62 → 38 taken 12 times.
✓ Branch 62 → 63 taken 12 times.
void extract_packed_rgb64_channel_avx2<1>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 62 → 38 taken 12 times.
✓ Branch 62 → 63 taken 12 times.
void extract_packed_rgb64_channel_avx2<2>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 62 → 38 taken 12 times.
✓ Branch 62 → 63 taken 12 times.
void extract_packed_rgb64_channel_avx2<3>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 62 → 38 taken 12 times.
✓ Branch 62 → 63 taken 12 times.
 96 for (; x < width; x += 8) {
231 48 const BYTE* s = srcp + x * 8;
232 48 __m128i t0 = _mm_load_si128(reinterpret_cast<const __m128i*>(s + 0)); // px 0-1
233 48 __m128i t1 = _mm_load_si128(reinterpret_cast<const __m128i*>(s + 16)); // px 2-3
234 48 __m128i t2 = _mm_load_si128(reinterpret_cast<const __m128i*>(s + 32)); // px 4-5
235 96 __m128i t3 = _mm_load_si128(reinterpret_cast<const __m128i*>(s + 48)); // px 6-7
236 48 t0 = _mm_shuffle_epi8(t0, shuf128);
237 48 t1 = _mm_shuffle_epi8(t1, shuf128);
238 48 t2 = _mm_shuffle_epi8(t2, shuf128);
239 48 t3 = _mm_shuffle_epi8(t3, shuf128);
240 // t0..t3 each: [{C_even, C_odd}, 0, 0, 0] as dwords
241 48 __m128i q01 = _mm_unpacklo_epi32(t0, t1); // [{C0,C1},{C2,C3}, 0, 0]
242 48 __m128i q23 = _mm_unpacklo_epi32(t2, t3); // [{C4,C5},{C6,C7}, 0, 0]
243 48 _mm_store_si128(reinterpret_cast<__m128i*>(dstp + x * 2),
244 _mm_unpacklo_epi64(q01, q23)); // [{C0,C1},{C2,C3},{C4,C5},{C6,C7}]
245 }
246 48 srcp -= src_pitch;
247 48 dstp += dst_pitch;
248 }
249 8 }
250
251 template void extract_packed_rgb64_channel_avx2<0>(const BYTE*, BYTE*, int, int, int, int);
252 template void extract_packed_rgb64_channel_avx2<1>(const BYTE*, BYTE*, int, int, int, int);
253 template void extract_packed_rgb64_channel_avx2<2>(const BYTE*, BYTE*, int, int, int, int);
254 template void extract_packed_rgb64_channel_avx2<3>(const BYTE*, BYTE*, int, int, int, int);
255
256 // ---------------------------------------------------------------------------
257 // Packed RGB24/RGB48 single-channel extraction — AVX2.
258 //
259 // pixel_t = uint8_t → RGB24, 3 bytes/pixel. 32 pixels per main iteration.
260 // pixel_t = uint16_t → RGB48, 6 bytes/pixel. 16 pixels per main iteration.
261 // channel_index: B=0 G=1 R=2 (no alpha in these formats)
262 //
263 // Source rows are bottom-up; srcp must already point at the last (bottom) row.
264 //
265 // Both formats share the same deinterleave skeleton because 3-channel pixels
266 // pack perfectly into 48-byte groups (16 RGB24 pixels or 8 RGB48 pixels) and
267 // the same three cross-boundary realignment shifts (12, 8, 4) apply to both.
268 //
269 // Strategy — process two 48-byte half-groups per iteration using AVX2 lanes:
270 // 1. Load 6 × 16-byte SSE registers (3 per half-group) into 3 × __m256i, one
271 // half per 128-bit lane. Both lanes execute the identical sequence below.
272 // 2. vpshufb with a compile-time mask deinterleaves each aligned 12-byte window
273 // (4 pixels / 2 pixels for u8/u16) so that:
274 // bytes 0- 3: B values (or B uint16 pairs for RGB48)
275 // bytes 4- 7: G values
276 // bytes 8-11: R values
277 // bytes 12-15: cross-boundary overflow (used by valignr, discarded after)
278 // 3. Three vpalignr shifts create four aligned windows from the three registers:
279 // window 1: BGRA_1[0:15] (no shift)
280 // window 2: BGRA_1[12:15] ++ BGRA_2[0:11] (alignr 12)
281 // window 3: BGRA_2[ 8:15] ++ BGRA_3[0: 7] (alignr 8)
282 // window 4: BGRA_3[4:15] shifted (srli 4)
283 // 4. unpacklo/hi_epi32 × 2 + unpacklo/hi_epi64 (selected by channel_index at
284 // compile time via constexpr if) combine the four packs into 32 output bytes.
285 //
286 // Remainder: if width is not a multiple of pixels_per_iter, one final iteration
287 // with unaligned loads/stores covers the last pixels_per_iter pixels, overlapping
288 // the already-written end of the destination (safe — values are idempotent).
289 // Caller must ensure width >= pixels_per_iter (32 for RGB24, 16 for RGB48).
290 // ---------------------------------------------------------------------------
291 template<typename pixel_t, int channel_index>
292 12 void extract_packed_rgb_noalpha_channel_avx2(
293 const BYTE* srcp, BYTE* dstp, int src_pitch, int dst_pitch, int width, int height)
294 {
295 static_assert(channel_index >= 0 && channel_index <= 2,
296 "RGB24/48 have no alpha; channel_index must be 0 (B), 1 (G), or 2 (R)");
297
298 // pixels_per_iter = 32 (RGB24) or 16 (RGB48); both consume exactly 96 source bytes.
299 12 constexpr int pixels_per_iter = (sizeof(pixel_t) == 1) ? 32 : 16;
300 12 const int wmod = (width / pixels_per_iter) * pixels_per_iter;
301
302 // Shuffle mask — same pattern for both 128-bit lanes.
303 //
304 // RGB24 (uint8_t): 16-byte window = 4 complete BGR pixels (12 bytes) + 4 overflow bytes.
305 // Deinterleave: B→bytes 0-3, G→bytes 4-7, R→bytes 8-11, overflow at 12-15.
306 // Byte source indices: B at 0,3,6,9; G at 1,4,7,10; R at 2,5,8,11; overflow at 12-15.
307 //
308 // RGB48 (uint16_t): 16-byte window = 2 complete BGR pixels (12 bytes) + 4 overflow bytes.
309 // Deinterleave: B uint16→bytes 0-3, G uint16→bytes 4-7, R uint16→bytes 8-11.
310 // Byte source indices: B0 at 0-1, B1 at 6-7; G0 at 2-3, G1 at 8-9; R0 at 4-5, R1 at 10-11.
311 __m256i mask;
312 if constexpr (sizeof(pixel_t) == 1)
313 6 mask = _mm256_set_epi8(
314 15,14,13,12, 11,8,5,2, 10,7,4,1, 9,6,3,0, // hi lane (same pattern)
315 15,14,13,12, 11,8,5,2, 10,7,4,1, 9,6,3,0); // lo lane
316 else
317 6 mask = _mm256_set_epi8(
318 15,14,13,12, 11,10,5,4, 9,8,3,2, 7,6,1,0, // hi lane
319 15,14,13,12, 11,10,5,4, 9,8,3,2, 7,6,1,0); // lo lane
320
321 // Inner kernel: given three __m256i covering two 48-byte half-groups (one per lane),
322 // deinterleave and return 32 bytes of the requested channel.
323 156 auto kernel = [&](__m256i v1, __m256i v2, __m256i v3) -> __m256i {
324 // Window 1: v1 as-is.
325 144 auto pack_lo = _mm256_shuffle_epi8(v1, mask);
326 // Window 2: BGRA_1[12:15] ++ BGRA_2[0:11] — per lane.
327 144 v1 = _mm256_alignr_epi8(v2, v1, 12);
328 144 auto pack_hi = _mm256_shuffle_epi8(v1, mask);
329 // Window 3: BGRA_2[8:15] ++ BGRA_3[0:7].
330 144 v2 = _mm256_alignr_epi8(v3, v2, 8);
331 144 auto pack_lo2 = _mm256_shuffle_epi8(v2, mask);
332 // Window 4: BGRA_3 shifted right 4 bytes — suffix of v3 aligned to window start.
333 144 v3 = _mm256_srli_si256(v3, 4);
334 288 auto pack_hi2 = _mm256_shuffle_epi8(v3, mask);
335
336 // After the four shuffles each pack register contains per lane:
337 // dword 0 = channel-B values (4 × u8 or 2 × u16)
338 // dword 1 = channel-G values
339 // dword 2 = channel-R values
340 // dword 3 = overflow / don't-care
341 //
342 // unpacklo_epi32 selects dwords 0 and 1 (B and G groups);
343 // unpackhi_epi32 selects dwords 2 and 3 (R and overflow).
344 // The final unpacklo/hi_epi64 picks the correct 64-bit half (B vs G, or R).
345 if constexpr (channel_index == 0) { // B: lo dword of each pack
346 48 auto q1 = _mm256_unpacklo_epi32(pack_lo, pack_hi); // [B_w1,B_w2, G_w1,G_w2] per lane
347 48 auto q2 = _mm256_unpacklo_epi32(pack_lo2, pack_hi2); // [B_w3,B_w4, G_w3,G_w4]
348 48 return _mm256_unpacklo_epi64(q1, q2); // [B_w1,B_w2,B_w3,B_w4] per lane
349 } else if constexpr (channel_index == 1) { // G: hi half of the lo-dword pairs
350 48 auto q1 = _mm256_unpacklo_epi32(pack_lo, pack_hi);
351 48 auto q2 = _mm256_unpacklo_epi32(pack_lo2, pack_hi2);
352 48 return _mm256_unpackhi_epi64(q1, q2);
353 } else { // R: lo dword of the hi-dword pairs
354 48 auto q1 = _mm256_unpackhi_epi32(pack_lo, pack_hi); // [R_w1,R_w2, overflow] per lane
355 48 auto q2 = _mm256_unpackhi_epi32(pack_lo2, pack_hi2);
356 48 return _mm256_unpacklo_epi64(q1, q2); // R values, overflow discarded
357 }
358 };
359
360
12/12
void extract_packed_rgb_noalpha_channel_avx2<unsigned char, 0>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 51 → 5 taken 12 times.
✓ Branch 51 → 52 taken 2 times.
void extract_packed_rgb_noalpha_channel_avx2<unsigned char, 1>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 51 → 5 taken 12 times.
✓ Branch 51 → 52 taken 2 times.
void extract_packed_rgb_noalpha_channel_avx2<unsigned char, 2>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 51 → 5 taken 12 times.
✓ Branch 51 → 52 taken 2 times.
void extract_packed_rgb_noalpha_channel_avx2<unsigned short, 0>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 51 → 5 taken 12 times.
✓ Branch 51 → 52 taken 2 times.
void extract_packed_rgb_noalpha_channel_avx2<unsigned short, 1>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 51 → 5 taken 12 times.
✓ Branch 51 → 52 taken 2 times.
void extract_packed_rgb_noalpha_channel_avx2<unsigned short, 2>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 51 → 5 taken 12 times.
✓ Branch 51 → 52 taken 2 times.
 84 for (int y = 0; y < height; ++y) {
361 // Main loop — aligned loads (source row starts 64-byte aligned; x*3*sizeof advances
362 // by multiples of 96 bytes = 6×16, so all six loads remain 16-byte aligned).
363 72 int x = 0;
364
12/12
void extract_packed_rgb_noalpha_channel_avx2<unsigned char, 0>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 27 → 6 taken 12 times.
✓ Branch 27 → 28 taken 12 times.
void extract_packed_rgb_noalpha_channel_avx2<unsigned char, 1>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 27 → 6 taken 12 times.
✓ Branch 27 → 28 taken 12 times.
void extract_packed_rgb_noalpha_channel_avx2<unsigned char, 2>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 27 → 6 taken 12 times.
✓ Branch 27 → 28 taken 12 times.
void extract_packed_rgb_noalpha_channel_avx2<unsigned short, 0>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 27 → 6 taken 12 times.
✓ Branch 27 → 28 taken 12 times.
void extract_packed_rgb_noalpha_channel_avx2<unsigned short, 1>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 27 → 6 taken 12 times.
✓ Branch 27 → 28 taken 12 times.
void extract_packed_rgb_noalpha_channel_avx2<unsigned short, 2>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 27 → 6 taken 12 times.
✓ Branch 27 → 28 taken 12 times.
 144 for (; x < wmod; x += pixels_per_iter) {
365 72 const BYTE* s = srcp + x * 3 * (int)sizeof(pixel_t);
366 // Pack lo half (bytes 0-47) into lo 128-bit lane and hi half (bytes 48-95) into hi lane.
367 216 auto v1 = _mm256_set_m128i(
368 _mm_load_si128(reinterpret_cast<const __m128i*>(s + 48 + 0)),
369 _mm_load_si128(reinterpret_cast<const __m128i*>(s + 0)));
370 216 auto v2 = _mm256_set_m128i(
371 _mm_load_si128(reinterpret_cast<const __m128i*>(s + 48 + 16)),
372 _mm_load_si128(reinterpret_cast<const __m128i*>(s + 16)));
373 216 auto v3 = _mm256_set_m128i(
374 _mm_load_si128(reinterpret_cast<const __m128i*>(s + 48 + 32)),
375 _mm_load_si128(reinterpret_cast<const __m128i*>(s + 32)));
376 72 _mm256_store_si256(
377 72 reinterpret_cast<__m256i*>(dstp + x * (int)sizeof(pixel_t)),
378 kernel(v1, v2, v3));
379 }
380 // Remainder — at most (pixels_per_iter - 1) pixels left; handle by re-processing the
381 // last pixels_per_iter pixels with unaligned loads/stores (may overlap previous writes).
382
6/12
void extract_packed_rgb_noalpha_channel_avx2<unsigned char, 0>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 28 → 29 taken 12 times.
✗ Branch 28 → 50 not taken.
void extract_packed_rgb_noalpha_channel_avx2<unsigned char, 1>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 28 → 29 taken 12 times.
✗ Branch 28 → 50 not taken.
void extract_packed_rgb_noalpha_channel_avx2<unsigned char, 2>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 28 → 29 taken 12 times.
✗ Branch 28 → 50 not taken.
void extract_packed_rgb_noalpha_channel_avx2<unsigned short, 0>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 28 → 29 taken 12 times.
✗ Branch 28 → 50 not taken.
void extract_packed_rgb_noalpha_channel_avx2<unsigned short, 1>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 28 → 29 taken 12 times.
✗ Branch 28 → 50 not taken.
void extract_packed_rgb_noalpha_channel_avx2<unsigned short, 2>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 28 → 29 taken 12 times.
✗ Branch 28 → 50 not taken.
 72 if (wmod < width) {
383 72 x = width - pixels_per_iter;
384 72 const BYTE* s = srcp + x * 3 * (int)sizeof(pixel_t);
385 216 auto v1 = _mm256_set_m128i(
386 _mm_loadu_si128(reinterpret_cast<const __m128i*>(s + 48 + 0)),
387 _mm_loadu_si128(reinterpret_cast<const __m128i*>(s + 0)));
388 216 auto v2 = _mm256_set_m128i(
389 _mm_loadu_si128(reinterpret_cast<const __m128i*>(s + 48 + 16)),
390 _mm_loadu_si128(reinterpret_cast<const __m128i*>(s + 16)));
391 216 auto v3 = _mm256_set_m128i(
392 _mm_loadu_si128(reinterpret_cast<const __m128i*>(s + 48 + 32)),
393 _mm_loadu_si128(reinterpret_cast<const __m128i*>(s + 32)));
394 72 _mm256_storeu_si256(
395 72 reinterpret_cast<__m256i*>(dstp + x * (int)sizeof(pixel_t)),
396 kernel(v1, v2, v3));
397 }
398 72 srcp -= src_pitch;
399 72 dstp += dst_pitch;
400 }
401 12 }
402
403 // Explicit instantiations: RGB24 (uint8_t) channels B, G, R
404 template void extract_packed_rgb_noalpha_channel_avx2<uint8_t, 0>(const BYTE*, BYTE*, int, int, int, int);
405 template void extract_packed_rgb_noalpha_channel_avx2<uint8_t, 1>(const BYTE*, BYTE*, int, int, int, int);
406 template void extract_packed_rgb_noalpha_channel_avx2<uint8_t, 2>(const BYTE*, BYTE*, int, int, int, int);
407 // Explicit instantiations: RGB48 (uint16_t) channels B, G, R
408 template void extract_packed_rgb_noalpha_channel_avx2<uint16_t, 0>(const BYTE*, BYTE*, int, int, int, int);
409 template void extract_packed_rgb_noalpha_channel_avx2<uint16_t, 1>(const BYTE*, BYTE*, int, int, int, int);
410 template void extract_packed_rgb_noalpha_channel_avx2<uint16_t, 2>(const BYTE*, BYTE*, int, int, int, int);
411