GCC Code Coverage Report

Directory: avs_core/
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 100.0% 782 / 0 / 782
Functions: 100.0% 25 / 0 / 25
Branches: 93.7% 118 / 0 / 126
convert/intel/convert_rgb_sse.cpp
Line Branch Exec Source
1 // Avisynth v2.5. Copyright 2002 Ben Rudiak-Gould et al.
2 // http://avisynth.nl
3
4 // This program is free software; you can redistribute it and/or modify
5 // it under the terms of the GNU General Public License as published by
6 // the Free Software Foundation; either version 2 of the License, or
7 // (at your option) any later version.
8 //
9 // This program is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 // GNU General Public License for more details.
13 //
14 // You should have received a copy of the GNU General Public License
15 // along with this program; if not, write to the Free Software
16 // Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA, or visit
17 // http://www.gnu.org/copyleft/gpl.html .
18 //
19 // Linking Avisynth statically or dynamically with other modules is making a
20 // combined work based on Avisynth. Thus, the terms and conditions of the GNU
21 // General Public License cover the whole combination.
22 //
23 // As a special exception, the copyright holders of Avisynth give you
24 // permission to link Avisynth with independent modules that communicate with
25 // Avisynth solely through the interfaces defined in avisynth.h, regardless of the license
26 // terms of these independent modules, and to copy and distribute the
27 // resulting combined work under terms of your choice, provided that
28 // every copy of the combined work is accompanied by a complete copy of
29 // the source code of Avisynth (the version of Avisynth used to produce the
30 // combined work), being distributed under the terms of the GNU General
31 // Public License plus this exception. An independent module is a module
32 // which is not derived from or based on Avisynth, such as 3rd-party filters,
33 // import and export plugins, or graphical user interfaces.
34
35
36 #include "../convert_rgb.h"
37
38 // Intrinsics base header + really required extension headers
39 #if defined(_MSC_VER)
40 #include <intrin.h> // MSVC
41 #else
42 #include <x86intrin.h> // GCC/MinGW/Clang/LLVM
43 #endif
44 #include <tmmintrin.h> // SSSE3
45
46 #include <avs/alignment.h>
47
48
49
50
51 #if defined(GCC) || defined(CLANG)
52 __attribute__((__target__("ssse3")))
53 #endif
54 2 void convert_rgb48_to_rgb64_ssse3(const BYTE *srcp, BYTE *dstp, size_t src_pitch, size_t dst_pitch, size_t width, size_t height)
55 {
56 2 size_t mod16_width = sizeof(uint16_t)*(width & (~size_t(7)));
57 2 __m128i mask0 = _mm_set_epi8((char)0x80, (char)0x80, 11, 10, 9, 8, 7, 6, (char)0x80, (char)0x80, 5, 4, 3, 2, 1, 0);
58 2 __m128i mask1 = _mm_set_epi8((char)0x80, (char)0x80, 15, 14, 13, 12, 11, 10, (char)0x80, (char)0x80, 9, 8, 7, 6, 5, 4);
59 2 __m128i alpha = _mm_set_epi32(0xFFFF0000,0x00000000,0xFFFF0000,0x00000000);
60
61
2/2
✓ Branch 42 → 9 taken 8 times.
✓ Branch 42 → 43 taken 2 times.
 10 for (size_t y = 0; y < height; ++y) {
62
2/2
✓ Branch 37 → 10 taken 5 times.
✓ Branch 37 → 38 taken 8 times.
 13 for (size_t x = 0; x < mod16_width; x+= 16) {
63 10 __m128i src0 = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp+x*3));
64 // 7... ...0
65 // B G|R B G R B G #0 #1 (#2) x*3+0
66 // G R B G|R B G R (#2) #3 #4 (#5) x*3+16
67 // R B G R B G|R B (#5) #6 #7 x*3+32
68 5 __m128i dst = _mm_or_si128(alpha, _mm_shuffle_epi8(src0, mask0));
69 5 _mm_stream_si128(reinterpret_cast<__m128i*>(dstp+4*x), dst);
70
71 5 __m128i src1 = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp+x*3+16));
72 5 __m128i tmp = _mm_alignr_epi8(src1, src0, 12);
73 5 dst = _mm_or_si128(alpha, _mm_shuffle_epi8(tmp, mask0));
74 5 _mm_stream_si128(reinterpret_cast<__m128i*>(dstp+x*4+16), dst);
75
76 5 __m128i src2 = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp+x*3+32));
77 5 tmp = _mm_alignr_epi8(src2, src1, 8);
78 5 dst = _mm_or_si128(alpha, _mm_shuffle_epi8(tmp, mask0));
79 5 _mm_stream_si128(reinterpret_cast<__m128i*>(dstp+x*4+32), dst);
80
81 5 dst = _mm_or_si128(alpha, _mm_shuffle_epi8(src2, mask1));
82 5 _mm_stream_si128(reinterpret_cast<__m128i*>(dstp+x*4+48), dst);
83 }
84
85
2/2
✓ Branch 40 → 39 taken 30 times.
✓ Branch 40 → 41 taken 8 times.
 38 for (size_t x = mod16_width/sizeof(uint16_t); x < width; ++x) {
86 30 reinterpret_cast<uint16_t *>(dstp)[x*4+0] = reinterpret_cast<const uint16_t *>(srcp)[x*3+0];
87 30 reinterpret_cast<uint16_t *>(dstp)[x*4+1] = reinterpret_cast<const uint16_t *>(srcp)[x*3+1];
88 30 reinterpret_cast<uint16_t *>(dstp)[x*4+2] = reinterpret_cast<const uint16_t *>(srcp)[x*3+2];
89 30 reinterpret_cast<uint16_t *>(dstp)[x*4+3] = 65535;
90 }
91
92 8 srcp += src_pitch;
93 8 dstp += dst_pitch;
94 }
95 2 }
96
97 #if defined(GCC) || defined(CLANG)
98 __attribute__((__target__("ssse3")))
99 #endif
100 2 void convert_rgb24_to_rgb32_ssse3(const BYTE *srcp, BYTE *dstp, size_t src_pitch, size_t dst_pitch, size_t width, size_t height)
101 {
102 2 size_t mod16_width = (width + 3) & (~size_t(15)); //when the modulo is more than 13, a problem does not happen
103 2 __m128i mask0 = _mm_set_epi8((char)0x80, 11, 10, 9, (char)0x80, 8, 7, 6, (char)0x80, 5, 4, 3, (char)0x80, 2, 1, 0);
104 2 __m128i mask1 = _mm_set_epi8((char)0x80, 15, 14, 13, (char)0x80, 12, 11, 10, (char)0x80, 9, 8, 7, (char)0x80, 6, 5, 4);
105 2 __m128i alpha = _mm_set1_epi32(0xFF000000);
106
107
2/2
✓ Branch 44 → 11 taken 8 times.
✓ Branch 44 → 45 taken 2 times.
 10 for (size_t y = 0; y < height; ++y) {
108
2/2
✓ Branch 39 → 12 taken 10 times.
✓ Branch 39 → 40 taken 8 times.
 18 for (size_t x = 0; x < mod16_width; x+= 16) {
109 20 __m128i src0 = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp+x*3));
110 10 __m128i dst = _mm_or_si128(alpha, _mm_shuffle_epi8(src0, mask0));
111 10 _mm_stream_si128(reinterpret_cast<__m128i*>(dstp+4*x), dst);
112
113 10 __m128i src1 = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp+x*3+16));
114 10 __m128i tmp = _mm_alignr_epi8(src1, src0, 12);
115 10 dst = _mm_or_si128(alpha, _mm_shuffle_epi8(tmp, mask0));
116 10 _mm_stream_si128(reinterpret_cast<__m128i*>(dstp+x*4+16), dst);
117
118 10 __m128i src2 = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp+x*3+32));
119 10 tmp = _mm_alignr_epi8(src2, src1, 8);
120 10 dst = _mm_or_si128(alpha, _mm_shuffle_epi8(tmp, mask0));
121 10 _mm_stream_si128(reinterpret_cast<__m128i*>(dstp+x*4+32), dst);
122
123 10 dst = _mm_or_si128(alpha, _mm_shuffle_epi8(src2, mask1));
124 10 _mm_stream_si128(reinterpret_cast<__m128i*>(dstp+x*4+48), dst);
125 }
126
127
2/2
✓ Branch 42 → 41 taken 15 times.
✓ Branch 42 → 43 taken 8 times.
 23 for (size_t x = mod16_width; x < width; ++x) {
128 15 dstp[x*4+0] = srcp[x*3+0];
129 15 dstp[x*4+1] = srcp[x*3+1];
130 15 dstp[x*4+2] = srcp[x*3+2];
131 15 dstp[x*4+3] = 255;
132 }
133
134 8 srcp += src_pitch;
135 8 dstp += dst_pitch;
136 }
137 2 }
138
139
140 #if defined(GCC) || defined(CLANG)
141 __attribute__((__target__("ssse3")))
142 #endif
143 2 void convert_rgb64_to_rgb48_ssse3(const BYTE *srcp, BYTE *dstp, size_t src_pitch, size_t dst_pitch, size_t width, size_t height)
144 {
145 2 size_t mod16_width = sizeof(uint16_t) * ((width) & (~size_t(7))); // perhaps width+2 is still o.k
146 2 __m128i mask0 = _mm_set_epi8(13, 12, 11, 10, 9, 8, 5, 4, 3, 2, 1, 0, 15, 14, 7, 6); // BBGGRRBBGGRRAAAA
147 2 __m128i mask1 = _mm_set_epi8(15, 14, 7, 6, 13, 12, 11, 10, 9, 8, 5, 4, 3, 2, 1, 0); // AAAABBGGRRBBGGRR
148
149
2/2
✓ Branch 33 → 7 taken 8 times.
✓ Branch 33 → 34 taken 2 times.
 10 for (size_t y = 0; y < height; ++y) {
150
2/2
✓ Branch 28 → 8 taken 5 times.
✓ Branch 28 → 29 taken 8 times.
 13 for (size_t x = 0; x < mod16_width; x+= 16) {
151 5 __m128i src0 = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp+x*4)); //a1b1 g1r1 a0b0 g0r0
152 10 __m128i src1 = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp+x*4+16)); //a3b3 g3r3 a2b2 g2r2
153 5 src0 = _mm_shuffle_epi8(src0, mask0); //b1g1 r1b0 g0r0 a1a0
154 5 src1 = _mm_shuffle_epi8(src1, mask1); //a3a2 b3g3 r3b2 g2r2
155 5 __m128i dst = _mm_alignr_epi8(src1, src0, 4); //g2r2 b1g1 r1b0 g0r0
156 5 _mm_stream_si128(reinterpret_cast<__m128i*>(dstp+x*3), dst);
157
158 5 src0 = _mm_slli_si128(src1, 4); // b3g3 r3b2 g2r2 XXXX
159 10 src1 = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp + x * 4 + 32)); // a5b5 g5r5 a4b4 g4r4
160 5 src1 = _mm_shuffle_epi8(src1, mask1); // a5a4 b5g5 r5b4 g4r4
161 5 dst = _mm_alignr_epi8(src1, src0, 8); // r5b4 g4r4 b3g3 r3b2
162 5 _mm_stream_si128(reinterpret_cast<__m128i*>(dstp+x*3+16), dst);
163
164 5 src0 = _mm_slli_si128(src1, 4); // b5g5 r5b4 g4r4 XXXX
165 10 src1 = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp+x*4+48)); // a7b7 g7r7 a6b6 g6r6
166 5 src1 = _mm_shuffle_epi8(src1, mask1); // a7a6 b7g7 r7b6 g6r6
167 5 dst = _mm_alignr_epi8(src1, src0, 12); // b7g7 r7b6 g6r6 b5g5
168 5 _mm_stream_si128(reinterpret_cast<__m128i*>(dstp+x*3+32), dst);
169 }
170
171
2/2
✓ Branch 31 → 30 taken 30 times.
✓ Branch 31 → 32 taken 8 times.
 38 for (size_t x = mod16_width/sizeof(uint16_t); x < width; ++x) {
172 30 reinterpret_cast<uint16_t *>(dstp)[x*3+0] = reinterpret_cast<const uint16_t *>(srcp)[x*4+0];
173 30 reinterpret_cast<uint16_t *>(dstp)[x*3+1] = reinterpret_cast<const uint16_t *>(srcp)[x*4+1];
174 30 reinterpret_cast<uint16_t *>(dstp)[x*3+2] = reinterpret_cast<const uint16_t *>(srcp)[x*4+2];
175 }
176
177 8 srcp += src_pitch;
178 8 dstp += dst_pitch;
179 }
180 2 }
181
182 //todo: think how to port to sse2 without tons of shuffles or (un)packs
183 #if defined(GCC) || defined(CLANG)
184 __attribute__((__target__("ssse3")))
185 #endif
186 2 void convert_rgb32_to_rgb24_ssse3(const BYTE *srcp, BYTE *dstp, size_t src_pitch, size_t dst_pitch, size_t width, size_t height)
187 {
188 2 size_t mod16_width = (width + 3) & (~size_t(15)); //when the modulo is more than 13, a problem does not happen
189 2 __m128i mask0 = _mm_set_epi8(14, 13, 12, 10, 9, 8, 6, 5, 4, 2, 1, 0, 15, 11, 7, 3);
190 2 __m128i mask1 = _mm_set_epi8(15, 11, 7, 3, 14, 13, 12, 10, 9, 8, 6, 5, 4, 2, 1, 0);
191
192
2/2
✓ Branch 33 → 7 taken 8 times.
✓ Branch 33 → 34 taken 2 times.
 10 for (size_t y = 0; y < height; ++y) {
193
2/2
✓ Branch 28 → 8 taken 10 times.
✓ Branch 28 → 29 taken 8 times.
 18 for (size_t x = 0; x < mod16_width; x+= 16) {
194 10 __m128i src0 = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp+x*4)); //a3b3 g3r3 a2b2 g2r2 a1b1 g1r1 a0b0 g0r0
195 20 __m128i src1 = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp+x*4+16)); //a7b7 g7r7 a6b6 g6r6 a5b5 g5r5 a4b4 g4r4
196 10 src0 = _mm_shuffle_epi8(src0, mask0); //b3g3 r3b2 g2r2 b1g1 r1b0 g0r0 a3a2 a1a0
197 10 src1 = _mm_shuffle_epi8(src1, mask1); //a7a6 a5a4 b7g7 r7b6 g6r6 b5g5 r5b4 g4r4
198 10 __m128i dst = _mm_alignr_epi8(src1, src0, 4); //r5b4 g4r4 b3g3 r3b2 g2r2 b1g1 r1b0 g0r0
199 10 _mm_stream_si128(reinterpret_cast<__m128i*>(dstp+x*3), dst);
200
201 10 src0 = _mm_slli_si128(src1, 4); //b7g7 r7b6 g6r6 b5g5 r5b4 g4r4 XXXX XXXX
202 20 src1 = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp + x * 4 + 32)); //aBbB gBrB aAbA gArA a9b9 g9r9 a8b8 g8r8
203 10 src1 = _mm_shuffle_epi8(src1, mask1); //aBaA a9a8 bBgB rBbA gArA b9g9 r9b8 g8r8
204 10 dst = _mm_alignr_epi8(src1, src0, 8); //gArA b9g9 r9b8 g8r8 b7g7 r7b6 g6r6 b5g5
205 10 _mm_stream_si128(reinterpret_cast<__m128i*>(dstp+x*3+16), dst);
206
207 10 src0 = _mm_slli_si128(src1, 4); //bBgB rBbA gArA b9g9 r9b8 g8r8 XXXX XXXX
208 20 src1 = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp+x*4+48)); //aFbF gFrF aEbE gErE aDbD gDrD aCbC gCrC
209 10 src1 = _mm_shuffle_epi8(src1, mask1); //aFaE aDaC bFgF rFbE gErE bDgD rDbC gCrC
210 10 dst = _mm_alignr_epi8(src1, src0, 12); //bFgF rFbE gErE bDgD rDbC gCrC bBgB rBbA
211 10 _mm_stream_si128(reinterpret_cast<__m128i*>(dstp+x*3+32), dst);
212 }
213
214
2/2
✓ Branch 31 → 30 taken 15 times.
✓ Branch 31 → 32 taken 8 times.
 23 for (size_t x = mod16_width; x < width; ++x) {
215 15 dstp[x*3+0] = srcp[x*4+0];
216 15 dstp[x*3+1] = srcp[x*4+1];
217 15 dstp[x*3+2] = srcp[x*4+2];
218 }
219
220 8 srcp += src_pitch;
221 8 dstp += dst_pitch;
222 }
223 2 }
224
225 1 void convert_rgb32_to_rgb24_sse2(const BYTE* srcp, BYTE* dstp, size_t src_pitch, size_t dst_pitch, size_t width, size_t height) {
226 1 size_t mod4_width = width & (~size_t(3));
227
228
2/2
✓ Branch 37 → 3 taken 5 times.
✓ Branch 37 → 38 taken 1 time.
 6 for (size_t y = 0; y < height; ++y) {
229
2/2
✓ Branch 32 → 4 taken 35 times.
✓ Branch 32 → 33 taken 5 times.
 40 for (size_t x = 0; x < mod4_width; x += 4) {
230 // Load 4 pixels (16 bytes)
231 70 __m128i src = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + x * 4));
232
233 // Pixel 0 (bytes 0,1,2) and Pixel 1 (bytes 4,5,6)
234 // Goal: [ 00 00 r1 g1 b1 r0 g0 b0 ] in the low 64 bits
235 70 __m128i p0 = _mm_and_si128(src, _mm_set_epi32(0, 0, 0, 0x00FFFFFF));
236 35 __m128i p1 = _mm_and_si128(src, _mm_set_epi32(0, 0, 0x00FFFFFF, 0));
237 70 __m128i dst01 = _mm_or_si128(p0, _mm_srli_si128(p1, 1));
238
239 // Pixel 2 (bytes 8,9,10) and Pixel 3 (bytes 12,13,14)
240 70 __m128i p2 = _mm_and_si128(src, _mm_set_epi32(0, 0x00FFFFFF, 0, 0));
241 35 __m128i p3 = _mm_and_si128(src, _mm_set_epi32(0x00FFFFFF, 0, 0, 0));
242
243 // Shift p2 down to byte 6 and p3 down to byte 9
244 70 __m128i dst23 = _mm_or_si128(_mm_srli_si128(p2, 2), _mm_srli_si128(p3, 3));
245
246 // Combine them
247 35 __m128i final = _mm_or_si128(dst01, dst23);
248
249 // final now contains: [ B3 G3 R3 | B2 G2 R2 | B1 G1 R1 | B0 G0 R0 ]
250 // In the first 12 bytes.
251 // We store 8 bytes (64-bit) then 4 bytes (32-bit)
252 35 _mm_storel_epi64(reinterpret_cast<__m128i*>(dstp + x * 3), final);
253 70 *reinterpret_cast<int*>(dstp + x * 3 + 8) = _mm_cvtsi128_si32(_mm_srli_si128(final, 8));
254 }
255
256 // Standard remainder loop
257
2/2
✓ Branch 35 → 34 taken 5 times.
✓ Branch 35 → 36 taken 5 times.
 10 for (size_t x = mod4_width; x < width; ++x) {
258 5 dstp[x * 3 + 0] = srcp[x * 4 + 0];
259 5 dstp[x * 3 + 1] = srcp[x * 4 + 1];
260 5 dstp[x * 3 + 2] = srcp[x * 4 + 2];
261 }
262
263 5 srcp += src_pitch;
264 5 dstp += dst_pitch;
265 }
266 1 }
267
268
269 // ============================================================
270 // SSE2 emulation of _mm_alignr_epi8(hi, lo, N)
271 // Concatenates [hi:lo] as 32 bytes, returns bytes [N..N+15]
272 // ============================================================
273 template<int N>
274 static AVS_FORCEINLINE __m128i sse2_alignr(__m128i hi, __m128i lo) {
275 120 return _mm_or_si128(_mm_srli_si128(lo, N), _mm_slli_si128(hi, 16 - N));
276 }
277
278 // ============================================================
279 // 8-bit helper: gather B, G, R from 4 BGR8 pixels
280 //
281 // Input 'grp': low 12 bytes hold 4 BGR8 pixels
282 // byte: 0 1 2 3 4 5 6 7 8 9 10 11
283 // data: B0 G0 R0 B1 G1 R1 B2 G2 R2 B3 G3 R3
284 //
285 // Output bOut, gOut, rOut: valid bytes in positions 0..3 only
286 // bOut[0..3] = B0 B1 B2 B3
287 // gOut[0..3] = G0 G1 G2 G3
288 // rOut[0..3] = R0 R1 R2 R3
289 // ============================================================
290 static AVS_FORCEINLINE void gather_bgr8_4px(__m128i grp,
291 __m128i& bOut, __m128i& gOut, __m128i& rOut)
292 {
293 // B is at bytes 0, 3, 6, 9. We shift them to 0, 1, 2, 3.
294 240 __m128i b = _mm_and_si128(grp, _mm_set_epi32(0, 0, 0, 0x000000FF)); // B0
295 360 b = _mm_or_si128(b, _mm_and_si128(_mm_srli_si128(grp, 2), _mm_set_epi32(0, 0, 0, 0x0000FF00))); // B1
296 360 b = _mm_or_si128(b, _mm_and_si128(_mm_srli_si128(grp, 4), _mm_set_epi32(0, 0, 0, 0x00FF0000))); // B2
297 240 b = _mm_or_si128(b, _mm_and_si128(_mm_srli_si128(grp, 6), _mm_set_epi32(0, 0, 0, 0xFF000000))); // B3
298 120 bOut = b;
299
300 // G is at bytes 1, 4, 7, 10.
301 240 __m128i g = _mm_and_si128(_mm_srli_si128(grp, 1), _mm_set_epi32(0, 0, 0, 0x000000FF)); // G0
302 360 g = _mm_or_si128(g, _mm_and_si128(_mm_srli_si128(grp, 3), _mm_set_epi32(0, 0, 0, 0x0000FF00))); // G1
303 360 g = _mm_or_si128(g, _mm_and_si128(_mm_srli_si128(grp, 5), _mm_set_epi32(0, 0, 0, 0x00FF0000))); // G2
304 240 g = _mm_or_si128(g, _mm_and_si128(_mm_srli_si128(grp, 7), _mm_set_epi32(0, 0, 0, 0xFF000000))); // G3
305 120 gOut = g;
306
307 // R is at bytes 2, 5, 8, 11.
308 240 __m128i r = _mm_and_si128(_mm_srli_si128(grp, 2), _mm_set_epi32(0, 0, 0, 0x000000FF)); // R0
309 360 r = _mm_or_si128(r, _mm_and_si128(_mm_srli_si128(grp, 4), _mm_set_epi32(0, 0, 0, 0x0000FF00))); // R1
310 360 r = _mm_or_si128(r, _mm_and_si128(_mm_srli_si128(grp, 6), _mm_set_epi32(0, 0, 0, 0x00FF0000))); // R2
311 240 r = _mm_or_si128(r, _mm_and_si128(_mm_srli_si128(grp, 8), _mm_set_epi32(0, 0, 0, 0xFF000000))); // R3
312 120 rOut = r;
313 120 }
314
315 // ============================================================
316 // 8-bit: deinterleave 16 BGR8 pixels from 3 × 16-byte registers
317 // into full 16-byte B, G, R output registers
318 // ============================================================
319 static AVS_FORCEINLINE void deinterleave_bgr8_16px(
320 __m128i r0, __m128i r1, __m128i r2,
321 __m128i& B, __m128i& G, __m128i& R)
322 {
323 // Split 48-byte stream into 4 groups of 12 bytes (4 pixels each),
324 // each aligned to byte 0 of a 128-bit register:
325 // group0: r0[0..11]
326 // group1: r0[12..15] ++ r1[0..7] = alignr(r1, r0, 12)
327 // group2: r1[8..15] ++ r2[0..3] = alignr(r2, r1, 8)
328 // group3: r2[4..15] = srli(r2, 4)
329 30 __m128i g0 = r0;
330 30 __m128i g1 = sse2_alignr<12>(r1, r0);
331 30 __m128i g2 = sse2_alignr<8>(r2, r1);
332 30 __m128i g3 = _mm_srli_si128(r2, 4);
333
334 // Extract channels from each group (4 valid bytes each in positions 0..3)
335 __m128i B0, G0, R0, B1, G1, R1, B2, G2, R2, B3, G3, R3;
336 gather_bgr8_4px(g0, B0, G0, R0);
337 gather_bgr8_4px(g1, B1, G1, R1);
338 gather_bgr8_4px(g2, B2, G2, R2);
339 gather_bgr8_4px(g3, B3, G3, R3);
340
341 // Combine 4 groups × 4 bytes into one 16-byte register per channel.
342 // Each Bx/Gx/Rx has its 4 bytes in the low dword (bytes 0..3).
343 // unpacklo_epi32 interleaves low dwords of two registers:
344 // unpacklo_epi32(A, B) = [A.dw0, B.dw0, A.dw1, B.dw1]
345 // unpacklo_epi64 combines low qwords:
346 // unpacklo_epi64(A, B) = [A.qw0, B.qw0]
347 30 __m128i B01 = _mm_unpacklo_epi32(B0, B1); // [B_g0(4B), B_g1(4B), ...]
348 60 __m128i B23 = _mm_unpacklo_epi32(B2, B3);
349 30 B = _mm_unpacklo_epi64(B01, B23); // all 16 B bytes
350
351 30 __m128i G01 = _mm_unpacklo_epi32(G0, G1);
352 60 __m128i G23 = _mm_unpacklo_epi32(G2, G3);
353 30 G = _mm_unpacklo_epi64(G01, G23);
354
355 30 __m128i R01 = _mm_unpacklo_epi32(R0, R1);
356 60 __m128i R23 = _mm_unpacklo_epi32(R2, R3);
357 30 R = _mm_unpacklo_epi64(R01, R23);
358 30 }
359
360 // ============================================================
361 // 16-bit helper: gather B, G, R from 2 BGR16 pixels
362 //
363 // Input 'grp': low 12 bytes hold 2 BGR16 pixels
364 // word: 0 1 2 3 4 5
365 // data: B0 G0 R0 B1 G1 R1
366 //
367 // Output bOut, gOut, rOut: valid words in positions 0..1 only
368 // bOut[0..1] = B0 B1
369 // gOut[0..1] = G0 G1
370 // rOut[0..1] = R0 R1
371 // ============================================================
372 static AVS_FORCEINLINE void gather_bgr16_2px(__m128i grp,
373 __m128i& bOut, __m128i& gOut, __m128i& rOut)
374 {
375 // Words in grp: B0 G0 R0 B1 G1 R1 x x
376 // Blue: w0, w3. Use shuffle to bring them to low dword.
377 120 bOut = _mm_shufflelo_epi16(grp, _MM_SHUFFLE(3, 3, 3, 0));
378 // Green: w1, w4.
379 120 gOut = _mm_shufflelo_epi16(_mm_srli_si128(grp, 2), _MM_SHUFFLE(3, 3, 3, 0));
380 // Red: w2, w5.
381 90 rOut = _mm_shufflelo_epi16(_mm_srli_si128(grp, 4), _MM_SHUFFLE(3, 3, 3, 0));
382
383 // Mask out the upper 3 dwords if necessary (though unpacklo handles it)
384 120 __m128i mask = _mm_set_epi32(0, 0, 0, 0xFFFFFFFF);
385 120 bOut = _mm_and_si128(bOut, mask);
386 120 gOut = _mm_and_si128(gOut, mask);
387 120 rOut = _mm_and_si128(rOut, mask);
388 120 }
389
390 // ============================================================
391 // 16-bit: deinterleave 8 BGR16 pixels from 3 × 16-byte registers
392 // into full 16-byte B, G, R output registers
393 // ============================================================
394 static AVS_FORCEINLINE void deinterleave_bgr16_8px(
395 __m128i r0, __m128i r1, __m128i r2,
396 __m128i& B, __m128i& G, __m128i& R)
397 {
398 // Same group extraction as 8-bit case (48 bytes, same offsets)
399 30 __m128i g0 = r0;
400 30 __m128i g1 = sse2_alignr<12>(r1, r0);
401 30 __m128i g2 = sse2_alignr<8>(r2, r1);
402 30 __m128i g3 = _mm_srli_si128(r2, 4);
403
404 // Each group: 12 bytes = 6 words = 2 BGR16 pixels
405 // Each output xB/xG/xR has 2 valid words (4 bytes) in dword 0
406 __m128i B0, G0, R0, B1, G1, R1, B2, G2, R2, B3, G3, R3;
407 gather_bgr16_2px(g0, B0, G0, R0);
408 gather_bgr16_2px(g1, B1, G1, R1);
409 gather_bgr16_2px(g2, B2, G2, R2);
410 gather_bgr16_2px(g3, B3, G3, R3);
411
412 // Combine: 4 groups × 2 words = 8 words = 16 bytes per channel
413 30 __m128i B01 = _mm_unpacklo_epi32(B0, B1);
414 60 __m128i B23 = _mm_unpacklo_epi32(B2, B3);
415 30 B = _mm_unpacklo_epi64(B01, B23);
416
417 30 __m128i G01 = _mm_unpacklo_epi32(G0, G1);
418 60 __m128i G23 = _mm_unpacklo_epi32(G2, G3);
419 30 G = _mm_unpacklo_epi64(G01, G23);
420
421 30 __m128i R01 = _mm_unpacklo_epi32(R0, R1);
422 60 __m128i R23 = _mm_unpacklo_epi32(R2, R3);
423 30 R = _mm_unpacklo_epi64(R01, R23);
424 30 }
425
426 // ============================================================
427 // Main conversion function
428 // ============================================================
429 // RGB24/48 -> RGBP(A)8/16 SSE2 implementation. Ugly and not very quick.
430 template<typename pixel_t, bool targetHasAlpha>
431 4 void convert_rgb_to_rgbp_sse2(const BYTE* srcp, BYTE* (&dstp)[4],
432 int src_pitch, int(&dst_pitch)[4],
433 int width, int height)
434 {
435 4 const int pixels_at_a_time = (sizeof(pixel_t) == 1) ? 16 : 8;
436 4 const int wmod = (width / pixels_at_a_time) * pixels_at_a_time;
437
438 __m128i max_pixel_value;
439 if constexpr (sizeof(pixel_t) == 1)
440 2 max_pixel_value = _mm_set1_epi8((char)0xFF);
441 else
442 2 max_pixel_value = _mm_set1_epi16((short)0xFFFF);
443
444
8/8
void convert_rgb_to_rgbp_sse2<unsigned char, false>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 621 → 7 taken 5 times.
✓ Branch 621 → 622 taken 1 time.
void convert_rgb_to_rgbp_sse2<unsigned char, true>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 623 → 7 taken 5 times.
✓ Branch 623 → 624 taken 1 time.
void convert_rgb_to_rgbp_sse2<unsigned short, false>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 157 → 7 taken 5 times.
✓ Branch 157 → 158 taken 1 time.
void convert_rgb_to_rgbp_sse2<unsigned short, true>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 159 → 7 taken 5 times.
✓ Branch 159 → 160 taken 1 time.
 24 for (int y = height; y > 0; --y) {
445
446
8/8
void convert_rgb_to_rgbp_sse2<unsigned char, false>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 313 → 8 taken 10 times.
✓ Branch 313 → 314 taken 5 times.
void convert_rgb_to_rgbp_sse2<unsigned char, true>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 314 → 8 taken 10 times.
✓ Branch 314 → 315 taken 5 times.
void convert_rgb_to_rgbp_sse2<unsigned short, false>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 81 → 8 taken 10 times.
✓ Branch 81 → 82 taken 5 times.
void convert_rgb_to_rgbp_sse2<unsigned short, true>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 82 → 8 taken 10 times.
✓ Branch 82 → 83 taken 5 times.
 60 for (int x = 0; x < wmod; x += pixels_at_a_time) {
447 40 const BYTE* src = srcp + x * 3 * sizeof(pixel_t);
448 40 __m128i r0 = _mm_load_si128(reinterpret_cast<const __m128i*>(src));
449 40 __m128i r1 = _mm_load_si128(reinterpret_cast<const __m128i*>(src + 16));
450 80 __m128i r2 = _mm_load_si128(reinterpret_cast<const __m128i*>(src + 32));
451
452 __m128i B, G, R;
453 if constexpr (sizeof(pixel_t) == 1)
454 deinterleave_bgr8_16px(r0, r1, r2, B, G, R);
455 else
456 deinterleave_bgr16_8px(r0, r1, r2, B, G, R);
457
458 40 _mm_store_si128(reinterpret_cast<__m128i*>(dstp[1] + x * sizeof(pixel_t)), B);
459 40 _mm_store_si128(reinterpret_cast<__m128i*>(dstp[0] + x * sizeof(pixel_t)), G);
460 40 _mm_store_si128(reinterpret_cast<__m128i*>(dstp[2] + x * sizeof(pixel_t)), R);
461 if constexpr (targetHasAlpha)
462 20 _mm_store_si128(reinterpret_cast<__m128i*>(dstp[3] + x * sizeof(pixel_t)), max_pixel_value);
463 }
464
465
4/8
void convert_rgb_to_rgbp_sse2<unsigned char, false>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 314 → 315 taken 5 times.
✗ Branch 314 → 620 not taken.
void convert_rgb_to_rgbp_sse2<unsigned char, true>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 315 → 316 taken 5 times.
✗ Branch 315 → 622 not taken.
void convert_rgb_to_rgbp_sse2<unsigned short, false>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 82 → 83 taken 5 times.
✗ Branch 82 → 156 not taken.
void convert_rgb_to_rgbp_sse2<unsigned short, true>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 83 → 84 taken 5 times.
✗ Branch 83 → 158 not taken.
 20 if (wmod != width) {
466 20 size_t x = width - pixels_at_a_time;
467 20 const BYTE* src = srcp + x * 3 * sizeof(pixel_t);
468 20 __m128i r0 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src));
469 20 __m128i r1 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src + 16));
470 40 __m128i r2 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src + 32));
471
472 __m128i B, G, R;
473 if constexpr (sizeof(pixel_t) == 1)
474 deinterleave_bgr8_16px(r0, r1, r2, B, G, R);
475 else
476 deinterleave_bgr16_8px(r0, r1, r2, B, G, R);
477
478 20 _mm_storeu_si128(reinterpret_cast<__m128i*>(dstp[1] + x * sizeof(pixel_t)), B);
479 20 _mm_storeu_si128(reinterpret_cast<__m128i*>(dstp[0] + x * sizeof(pixel_t)), G);
480 20 _mm_storeu_si128(reinterpret_cast<__m128i*>(dstp[2] + x * sizeof(pixel_t)), R);
481 if constexpr (targetHasAlpha)
482 10 _mm_storeu_si128(reinterpret_cast<__m128i*>(dstp[3] + x * sizeof(pixel_t)), max_pixel_value);
483 }
484
485 20 srcp -= src_pitch;
486 20 dstp[0] += dst_pitch[0];
487 20 dstp[1] += dst_pitch[1];
488 20 dstp[2] += dst_pitch[2];
489 if constexpr (targetHasAlpha)
490 10 dstp[3] += dst_pitch[3];
491 }
492 4 }
493
494 // Instantiations
495 template void convert_rgb_to_rgbp_sse2<uint8_t, false>(const BYTE*, BYTE* (&)[4], int, int(&)[4], int, int);
496 template void convert_rgb_to_rgbp_sse2<uint8_t, true>(const BYTE*, BYTE* (&)[4], int, int(&)[4], int, int);
497 template void convert_rgb_to_rgbp_sse2<uint16_t, false>(const BYTE*, BYTE* (&)[4], int, int(&)[4], int, int);
498 template void convert_rgb_to_rgbp_sse2<uint16_t, true>(const BYTE*, BYTE* (&)[4], int, int(&)[4], int, int);
499
500 // minimum width: 48 bytes
501 template<typename pixel_t, bool targetHasAlpha>
502 #if defined(GCC) || defined(CLANG)
503 __attribute__((__target__("ssse3")))
504 #endif
505 4 void convert_rgb_to_rgbp_ssse3(const BYTE *srcp, BYTE * (&dstp)[4], int src_pitch, int(&dst_pitch)[4], int width, int height)
506 {
507 // RGB24: 3x16 bytes cycle, 16*(RGB) 8bit pixels
508 // RGB48: 3x16 bytes cycle, 8*(RGB) 16bit pixels
509 // 0123456789ABCDEF 0123456789ABCDEF 0123456789ABCDEF
510 // BGRBGRBGRBGRBGRB GRBGRBGRBGRBGRBG RBGRBGRBGRBGRBGR // 8 bit
511 // B G R B G R B G R B G R B G R B G R B G R B G R // 16 bit
512 // 1111111111112222 2222222233333333 3333444444444444
513
514 4 const int pixels_at_a_time = (sizeof(pixel_t) == 1) ? 16 : 8;
515 4 const int wmod = (width / pixels_at_a_time) * pixels_at_a_time; // 8 pixels for 8 bit, 4 pixels for 16 bit
516 __m128i mask;
517 if constexpr(sizeof(pixel_t) == 1)
518 2 mask = _mm_set_epi8(15, 14, 13, 12, 11, 8, 5, 2, 10, 7, 4, 1, 9, 6, 3, 0);
519 else
520 2 mask = _mm_set_epi8(15, 14, 13, 12, 11, 10, 5, 4, 9, 8, 3, 2, 7, 6, 1, 0);
521
522 __m128i max_pixel_value;
523 if constexpr(sizeof(pixel_t) == 1)
524 2 max_pixel_value = _mm_set1_epi8((char)0xFF);
525 else
526 2 max_pixel_value = _mm_set1_epi16((short)0xFFFF); // bits_per_pixel is 16
527
528
8/8
void convert_rgb_to_rgbp_ssse3<unsigned char, false>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 79 → 9 taken 5 times.
✓ Branch 79 → 80 taken 1 time.
void convert_rgb_to_rgbp_ssse3<unsigned char, true>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 79 → 9 taken 5 times.
✓ Branch 79 → 80 taken 1 time.
void convert_rgb_to_rgbp_ssse3<unsigned short, false>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 79 → 9 taken 5 times.
✓ Branch 79 → 80 taken 1 time.
void convert_rgb_to_rgbp_ssse3<unsigned short, true>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 79 → 9 taken 5 times.
✓ Branch 79 → 80 taken 1 time.
 24 for (int y = height; y > 0; --y) {
529 __m128i BGRA_1, BGRA_2, BGRA_3;
530
8/8
void convert_rgb_to_rgbp_ssse3<unsigned char, false>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 43 → 10 taken 10 times.
✓ Branch 43 → 44 taken 5 times.
void convert_rgb_to_rgbp_ssse3<unsigned char, true>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 43 → 10 taken 10 times.
✓ Branch 43 → 44 taken 5 times.
void convert_rgb_to_rgbp_ssse3<unsigned short, false>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 43 → 10 taken 10 times.
✓ Branch 43 → 44 taken 5 times.
void convert_rgb_to_rgbp_ssse3<unsigned short, true>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 43 → 10 taken 10 times.
✓ Branch 43 → 44 taken 5 times.
 60 for (int x = 0; x < wmod; x += pixels_at_a_time) {
531 40 BGRA_1 = _mm_load_si128(reinterpret_cast<const __m128i *>(srcp + x * 48 / pixels_at_a_time));
532 40 BGRA_2 = _mm_load_si128(reinterpret_cast<const __m128i *>(srcp + x * 48 / pixels_at_a_time + 16));
533 80 BGRA_3 = _mm_load_si128(reinterpret_cast<const __m128i *>(srcp + x * 48 / pixels_at_a_time + 32));
534
535 40 auto pack_lo = _mm_shuffle_epi8(BGRA_1, mask); // 111111111111: BBBBGGGGRRRR and rest: BGRB | BBGGRR and rest: BBGG
536 40 BGRA_1 = _mm_alignr_epi8(BGRA_2, BGRA_1, 12);
537 40 auto pack_hi = _mm_shuffle_epi8(BGRA_1, mask); // 222222222222: BBBBGGGGRRRR | BBGGRR
538 40 BGRA_2 = _mm_alignr_epi8(BGRA_3, BGRA_2, 8);
539 40 auto pack_lo2 = _mm_shuffle_epi8(BGRA_2, mask); // 333333333333: BBBBGGGGRRRR | BBGGRR
540 40 BGRA_3 = _mm_srli_si128(BGRA_3, 4); // to use the same mask
541 40 auto pack_hi2 = _mm_shuffle_epi8(BGRA_3, mask); // 444444444444: BBBBGGGGRRRR | BBGGRR
542
543 40 __m128i BG1 = _mm_unpacklo_epi32(pack_lo, pack_hi); // BBBB_lo BBBB_hi GGGG_lo GGGG_hi
544 40 __m128i BG2 = _mm_unpacklo_epi32(pack_lo2, pack_hi2); // BBBB_lo BBBB_hi GGGG_lo GGGG_hi
545 40 __m128i RA1 = _mm_unpackhi_epi32(pack_lo, pack_hi); // RRRR_lo RRRR_hi AAAA_lo AAAA_hi
546 40 __m128i RA2 = _mm_unpackhi_epi32(pack_lo2, pack_hi2); // RRRR_lo RRRR_hi AAAA_lo AAAA_hi
547 40 __m128i B = _mm_unpacklo_epi64(BG1, BG2);
548 40 _mm_store_si128(reinterpret_cast<__m128i *>(dstp[1] + x * sizeof(pixel_t)), B); // B
549 40 __m128i G = _mm_unpackhi_epi64(BG1, BG2);
550 40 _mm_store_si128(reinterpret_cast<__m128i *>(dstp[0] + x * sizeof(pixel_t)), G); // G
551 40 __m128i R = _mm_unpacklo_epi64(RA1, RA2);
552 40 _mm_store_si128(reinterpret_cast<__m128i *>(dstp[2] + x * sizeof(pixel_t)), R); // R
553 if (targetHasAlpha)
554 20 _mm_store_si128(reinterpret_cast<__m128i *>(dstp[3] + x * sizeof(pixel_t)), max_pixel_value); // A
555 }
556 // rest, unaligned but simd
557 // RGB24/48 source is 3 bytes/pixel: no power-of-2 stride, 64-byte alignment does not guarantee width is a multiple of 16/8
558 // So we cannot spare the last chunk
559
4/8
void convert_rgb_to_rgbp_ssse3<unsigned char, false>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 44 → 45 taken 5 times.
✗ Branch 44 → 77 not taken.
void convert_rgb_to_rgbp_ssse3<unsigned char, true>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 44 → 45 taken 5 times.
✗ Branch 44 → 78 not taken.
void convert_rgb_to_rgbp_ssse3<unsigned short, false>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 44 → 45 taken 5 times.
✗ Branch 44 → 77 not taken.
void convert_rgb_to_rgbp_ssse3<unsigned short, true>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 44 → 45 taken 5 times.
✗ Branch 44 → 78 not taken.
 20 if (wmod != width) {
560 // width = 17: 0..7 8..15, 16
561 // last_start = 1 (9..16 8 pixels) width - pixels_at_a_time
562 20 size_t x = (width - pixels_at_a_time);
563 20 BGRA_1 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(srcp + x * 48 / pixels_at_a_time));
564 20 BGRA_2 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(srcp + x * 48 / pixels_at_a_time + 16));
565 40 BGRA_3 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(srcp + x * 48 / pixels_at_a_time + 32));
566
567 20 auto pack_lo = _mm_shuffle_epi8(BGRA_1, mask); // 111111111111: BBBBGGGGRRRR and rest: BGRB | BBGGRR and rest: BBGG
568 20 BGRA_1 = _mm_alignr_epi8(BGRA_2, BGRA_1, 12);
569 20 auto pack_hi = _mm_shuffle_epi8(BGRA_1, mask); // 222222222222: BBBBGGGGRRRR | BBGGRR
570 20 BGRA_2 = _mm_alignr_epi8(BGRA_3, BGRA_2, 8);
571 20 auto pack_lo2 = _mm_shuffle_epi8(BGRA_2, mask); // 333333333333: BBBBGGGGRRRR | BBGGRR
572 20 BGRA_3 = _mm_srli_si128(BGRA_3, 4); // to use the same mask
573 20 auto pack_hi2 = _mm_shuffle_epi8(BGRA_3, mask); // 444444444444: BBBBGGGGRRRR | BBGGRR
574
575 20 __m128i BG1 = _mm_unpacklo_epi32(pack_lo, pack_hi); // BBBB_lo BBBB_hi GGGG_lo GGGG_hi
576 20 __m128i BG2 = _mm_unpacklo_epi32(pack_lo2, pack_hi2); // BBBB_lo BBBB_hi GGGG_lo GGGG_hi
577 20 __m128i RA1 = _mm_unpackhi_epi32(pack_lo, pack_hi); // RRRR_lo RRRR_hi AAAA_lo AAAA_hi
578 20 __m128i RA2 = _mm_unpackhi_epi32(pack_lo2, pack_hi2); // RRRR_lo RRRR_hi AAAA_lo AAAA_hi
579 20 __m128i B = _mm_unpacklo_epi64(BG1, BG2);
580 20 _mm_storeu_si128(reinterpret_cast<__m128i *>(dstp[1] + x * sizeof(pixel_t)), B); // B
581 20 __m128i G = _mm_unpackhi_epi64(BG1, BG2);
582 20 _mm_storeu_si128(reinterpret_cast<__m128i *>(dstp[0] + x * sizeof(pixel_t)), G); // G
583 20 __m128i R = _mm_unpacklo_epi64(RA1, RA2);
584 20 _mm_storeu_si128(reinterpret_cast<__m128i *>(dstp[2] + x * sizeof(pixel_t)), R); // R
585 if (targetHasAlpha)
586 10 _mm_storeu_si128(reinterpret_cast<__m128i *>(dstp[3] + x * sizeof(pixel_t)), max_pixel_value); // A
587 }
588 20 srcp -= src_pitch; // source packed RGB is upside down
589 20 dstp[0] += dst_pitch[0];
590 20 dstp[1] += dst_pitch[1];
591 20 dstp[2] += dst_pitch[2];
592 if (targetHasAlpha)
593 10 dstp[3] += dst_pitch[3];
594 }
595 4 }
596
597 //instantiate
598 //template<typename pixel_t, bool targetHasAlpha>
599 template void convert_rgb_to_rgbp_ssse3<uint8_t, false>(const BYTE* srcp, BYTE* (&dstp)[4], int src_pitch, int(&dst_pitch)[4], int width, int height);
600 template void convert_rgb_to_rgbp_ssse3<uint8_t, true>(const BYTE* srcp, BYTE* (&dstp)[4], int src_pitch, int(&dst_pitch)[4], int width, int height);
601 template void convert_rgb_to_rgbp_ssse3<uint16_t, false>(const BYTE* srcp, BYTE* (&dstp)[4], int src_pitch, int(&dst_pitch)[4], int width, int height);
602 template void convert_rgb_to_rgbp_ssse3<uint16_t, true>(const BYTE* srcp, BYTE* (&dstp)[4], int src_pitch, int(&dst_pitch)[4], int width, int height);
603
604
605 // minimum width: 32 bytes (8 RGBA pixels for 8 bits, 4 RGBA pixels for 16 bits)
606 // RGB32/64 -> RGBP(A)8/16 SSE2 implementation. Ugly and not very quick.
607 template<typename pixel_t, bool targetHasAlpha>
608 4 void convert_rgba_to_rgbp_sse2(const BYTE* srcp, BYTE* (&dstp)[4],
609 int src_pitch, int(&dst_pitch)[4], int width, int height)
610 {
611 4 const int pixels_at_a_time = (sizeof(pixel_t) == 1) ? 8 : 4;
612 4 __m128i zero = _mm_setzero_si128();
613
614
8/8
void convert_rgba_to_rgbp_sse2<unsigned char, false>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 46 → 5 taken 5 times.
✓ Branch 46 → 47 taken 1 time.
void convert_rgba_to_rgbp_sse2<unsigned char, true>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 49 → 5 taken 5 times.
✓ Branch 49 → 50 taken 1 time.
void convert_rgba_to_rgbp_sse2<unsigned short, false>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 26 → 5 taken 5 times.
✓ Branch 26 → 27 taken 1 time.
void convert_rgba_to_rgbp_sse2<unsigned short, true>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 29 → 5 taken 5 times.
✓ Branch 29 → 30 taken 1 time.
 24 for (int y = height; y > 0; --y) {
615 // Avisynth's scanline alignment is 64 bytes, so no remainder handling is needed for 8-bit (16 pixels) and 16-bit (8 pixels) formats.
616
8/8
void convert_rgba_to_rgbp_sse2<unsigned char, false>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 44 → 6 taken 20 times.
✓ Branch 44 → 45 taken 5 times.
void convert_rgba_to_rgbp_sse2<unsigned char, true>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 47 → 6 taken 20 times.
✓ Branch 47 → 48 taken 5 times.
void convert_rgba_to_rgbp_sse2<unsigned short, false>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 24 → 6 taken 20 times.
✓ Branch 24 → 25 taken 5 times.
void convert_rgba_to_rgbp_sse2<unsigned short, true>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 27 → 6 taken 20 times.
✓ Branch 27 → 28 taken 5 times.
 100 for (int x = 0; x < width; x += pixels_at_a_time) {
617 80 const BYTE* src = srcp + x * 4 * sizeof(pixel_t);
618 80 __m128i lo = _mm_load_si128(reinterpret_cast<const __m128i*>(src));
619 160 __m128i hi = _mm_load_si128(reinterpret_cast<const __m128i*>(src + 16));
620 __m128i ch0, ch1;
621
622 if constexpr (sizeof(pixel_t) == 1) {
623 40 __m128i w0 = _mm_unpacklo_epi8(lo, zero);
624 40 __m128i w1 = _mm_unpackhi_epi8(lo, zero);
625 40 __m128i c0 = _mm_unpacklo_epi16(w0, w1);
626 40 __m128i c1 = _mm_unpackhi_epi16(w0, w1);
627 40 __m128i lo_bg = _mm_unpacklo_epi16(c0, c1);
628 40 __m128i lo_ra = _mm_unpackhi_epi16(c0, c1);
629 40 __m128i w2 = _mm_unpacklo_epi8(hi, zero);
630 40 __m128i w3 = _mm_unpackhi_epi8(hi, zero);
631 40 __m128i c2 = _mm_unpacklo_epi16(w2, w3);
632 40 __m128i c3 = _mm_unpackhi_epi16(w2, w3);
633 40 __m128i hi_bg = _mm_unpacklo_epi16(c2, c3);
634 40 __m128i hi_ra = _mm_unpackhi_epi16(c2, c3);
635 40 ch0 = _mm_shuffle_epi32(_mm_packus_epi16(lo_bg, hi_bg), _MM_SHUFFLE(3, 1, 2, 0));
636 40 ch1 = _mm_shuffle_epi32(_mm_packus_epi16(lo_ra, hi_ra), _MM_SHUFFLE(3, 1, 2, 0));
637 }
638 else {
639 40 __m128i t0 = _mm_unpacklo_epi16(lo, hi);
640 40 __m128i t1 = _mm_unpackhi_epi16(lo, hi);
641 40 ch0 = _mm_unpacklo_epi16(t0, t1);
642 40 ch1 = _mm_unpackhi_epi16(t0, t1);
643 }
644 // Stores 8 byte per channel, so we can use storel_epi64. The high 64 bits of ch0/ch1 are ignored.
645 80 _mm_storel_epi64(reinterpret_cast<__m128i*>(dstp[1] + x * sizeof(pixel_t)), ch0); // B
646 80 _mm_storeh_pd(reinterpret_cast<double*>(dstp[0] + x * sizeof(pixel_t)),_mm_castsi128_pd(ch0)); // G
647 80 _mm_storel_epi64(reinterpret_cast<__m128i*>(dstp[2] + x * sizeof(pixel_t)), ch1); // R
648 if constexpr (targetHasAlpha)
649 40 _mm_storeh_pd(reinterpret_cast<double*>(dstp[3] + x * sizeof(pixel_t)), _mm_castsi128_pd(ch1)); // A
650 }
651
652 20 srcp -= src_pitch;
653 20 dstp[0] += dst_pitch[0];
654 20 dstp[1] += dst_pitch[1];
655 20 dstp[2] += dst_pitch[2];
656 if constexpr (targetHasAlpha)
657 10 dstp[3] += dst_pitch[3];
658 }
659 4 }
660
661 // Instantiations
662 template void convert_rgba_to_rgbp_sse2<uint8_t, false>(const BYTE*, BYTE* (&)[4], int, int(&)[4], int, int);
663 template void convert_rgba_to_rgbp_sse2<uint8_t, true>(const BYTE*, BYTE* (&)[4], int, int(&)[4], int, int);
664 template void convert_rgba_to_rgbp_sse2<uint16_t, false>(const BYTE*, BYTE* (&)[4], int, int(&)[4], int, int);
665 template void convert_rgba_to_rgbp_sse2<uint16_t, true>(const BYTE*, BYTE* (&)[4], int, int(&)[4], int, int);
666
667
668 template<typename pixel_t, bool targetHasAlpha>
669 #if defined(GCC) || defined(CLANG)
670 __attribute__((__target__("ssse3")))
671 #endif
672 4 void convert_rgba_to_rgbp_ssse3(const BYTE* srcp, BYTE* (&dstp)[4], int src_pitch, int(&dst_pitch)[4], int width, int height)
673 {
674 4 const int pixels_at_a_time = (sizeof(pixel_t) == 1) ? 8 : 4;
675
676 __m128i mask;
677 if constexpr (sizeof(pixel_t) == 1)
678 2 mask = _mm_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);
679 else
680 2 mask = _mm_set_epi8(15, 14, 7, 6, 13, 12, 5, 4, 11, 10, 3, 2, 9, 8, 1, 0);
681
682
8/8
void convert_rgba_to_rgbp_ssse3<unsigned char, false>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 26 → 5 taken 5 times.
✓ Branch 26 → 27 taken 1 time.
void convert_rgba_to_rgbp_ssse3<unsigned char, true>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 29 → 5 taken 5 times.
✓ Branch 29 → 30 taken 1 time.
void convert_rgba_to_rgbp_ssse3<unsigned short, false>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 26 → 5 taken 5 times.
✓ Branch 26 → 27 taken 1 time.
void convert_rgba_to_rgbp_ssse3<unsigned short, true>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 29 → 5 taken 5 times.
✓ Branch 29 → 30 taken 1 time.
 24 for (int y = height; y > 0; --y) {
683 // Avisynth's scanline alignment is 64 bytes, so no remainder handling is needed for 8 RGB32 or 4 RGB64 pixels
684
8/8
void convert_rgba_to_rgbp_ssse3<unsigned char, false>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 24 → 6 taken 20 times.
✓ Branch 24 → 25 taken 5 times.
void convert_rgba_to_rgbp_ssse3<unsigned char, true>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 27 → 6 taken 20 times.
✓ Branch 27 → 28 taken 5 times.
void convert_rgba_to_rgbp_ssse3<unsigned short, false>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 24 → 6 taken 20 times.
✓ Branch 24 → 25 taken 5 times.
void convert_rgba_to_rgbp_ssse3<unsigned short, true>(unsigned char const*, unsigned char* (&) [4], int, int (&) [4], int, int):
✓ Branch 27 → 6 taken 20 times.
✓ Branch 27 → 28 taken 5 times.
 100 for (int x = 0; x < width; x += pixels_at_a_time) {
685 80 __m128i BGRA_lo = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp + x * 4 * sizeof(pixel_t)));
686 160 __m128i BGRA_hi = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp + x * 4 * sizeof(pixel_t) + 16));
687 80 __m128i pack_lo = _mm_shuffle_epi8(BGRA_lo, mask);
688 80 __m128i pack_hi = _mm_shuffle_epi8(BGRA_hi, mask);
689 80 __m128i eightbytes_of_pixels = _mm_unpacklo_epi32(pack_lo, pack_hi);
690 80 _mm_storel_epi64(reinterpret_cast<__m128i*>(dstp[1] + x * sizeof(pixel_t)), eightbytes_of_pixels); // B
691 80 _mm_storeh_pd(reinterpret_cast<double*>(dstp[0] + x * sizeof(pixel_t)), _mm_castsi128_pd(eightbytes_of_pixels)); // G
692 80 eightbytes_of_pixels = _mm_unpackhi_epi32(pack_lo, pack_hi);
693 80 _mm_storel_epi64(reinterpret_cast<__m128i*>(dstp[2] + x * sizeof(pixel_t)), eightbytes_of_pixels); // R
694 if constexpr (targetHasAlpha)
695 40 _mm_storeh_pd(reinterpret_cast<double*>(dstp[3] + x * sizeof(pixel_t)), _mm_castsi128_pd(eightbytes_of_pixels)); // A
696 }
697 20 srcp -= src_pitch;
698 20 dstp[0] += dst_pitch[0];
699 20 dstp[1] += dst_pitch[1];
700 20 dstp[2] += dst_pitch[2];
701 if constexpr (targetHasAlpha)
702 10 dstp[3] += dst_pitch[3];
703 }
704 4 }
705
706 //instantiate
707 //template<typename pixel_t, bool targetHasAlpha>
708 template void convert_rgba_to_rgbp_ssse3<uint8_t, false>(const BYTE* srcp, BYTE* (&dstp)[4], int src_pitch, int(&dst_pitch)[4], int width, int height);
709 template void convert_rgba_to_rgbp_ssse3<uint8_t, true>(const BYTE* srcp, BYTE* (&dstp)[4], int src_pitch, int(&dst_pitch)[4], int width, int height);
710 template void convert_rgba_to_rgbp_ssse3<uint16_t, false>(const BYTE* srcp, BYTE* (&dstp)[4], int src_pitch, int(&dst_pitch)[4], int width, int height);
711 template void convert_rgba_to_rgbp_ssse3<uint16_t, true>(const BYTE* srcp, BYTE* (&dstp)[4], int src_pitch, int(&dst_pitch)[4], int width, int height);
712
713 template<typename pixel_t, bool hasSrcAlpha>
714 4 void convert_rgbp_to_rgba_sse2(const BYTE* (&srcp)[4], BYTE* dstp, int(&src_pitch)[4], int dst_pitch, int width, int height)
715 {
716 4 const int pixels_at_a_time = 8 / sizeof(pixel_t); // 8x uint8_t, 4x uint16_t
717 4 const __m128i transparent = _mm_set1_epi8((char)0xFF);
718 // Avisynth's scanline alignment is 64 bytes, so no remainder handling is needed for 8 RGB32 or 4 RGB64 pixels
719
8/8
void convert_rgbp_to_rgba_sse2<unsigned char, false>(unsigned char const* (&) [4], unsigned char*, int (&) [4], int, int, int):
✓ Branch 39 → 7 taken 5 times.
✓ Branch 39 → 40 taken 1 time.
void convert_rgbp_to_rgba_sse2<unsigned char, true>(unsigned char const* (&) [4], unsigned char*, int (&) [4], int, int, int):
✓ Branch 45 → 7 taken 5 times.
✓ Branch 45 → 46 taken 1 time.
void convert_rgbp_to_rgba_sse2<unsigned short, false>(unsigned char const* (&) [4], unsigned char*, int (&) [4], int, int, int):
✓ Branch 39 → 7 taken 5 times.
✓ Branch 39 → 40 taken 1 time.
void convert_rgbp_to_rgba_sse2<unsigned short, true>(unsigned char const* (&) [4], unsigned char*, int (&) [4], int, int, int):
✓ Branch 45 → 7 taken 5 times.
✓ Branch 45 → 46 taken 1 time.
 24 for (int y = 0; y < height; y++) {
720
8/8
void convert_rgbp_to_rgba_sse2<unsigned char, false>(unsigned char const* (&) [4], unsigned char*, int (&) [4], int, int, int):
✓ Branch 37 → 8 taken 30 times.
✓ Branch 37 → 38 taken 5 times.
void convert_rgbp_to_rgba_sse2<unsigned char, true>(unsigned char const* (&) [4], unsigned char*, int (&) [4], int, int, int):
✓ Branch 43 → 8 taken 30 times.
✓ Branch 43 → 44 taken 5 times.
void convert_rgbp_to_rgba_sse2<unsigned short, false>(unsigned char const* (&) [4], unsigned char*, int (&) [4], int, int, int):
✓ Branch 37 → 8 taken 30 times.
✓ Branch 37 → 38 taken 5 times.
void convert_rgbp_to_rgba_sse2<unsigned short, true>(unsigned char const* (&) [4], unsigned char*, int (&) [4], int, int, int):
✓ Branch 43 → 8 taken 30 times.
✓ Branch 43 → 44 taken 5 times.
 140 for (int x = 0; x < width; x += pixels_at_a_time) {
721 __m128i R, G, B, A;
722 120 G = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(srcp[0] + x * sizeof(pixel_t)));
723 120 B = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(srcp[1] + x * sizeof(pixel_t)));
724 120 R = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(srcp[2] + x * sizeof(pixel_t)));
725 if constexpr (hasSrcAlpha)
726 120 A = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(srcp[3] + x * sizeof(pixel_t)));
727 else
728 60 A = transparent;
729
730 if constexpr (sizeof(pixel_t) == 1) {
731 60 __m128i BG = _mm_unpacklo_epi8(B, G);
732 60 __m128i RA = _mm_unpacklo_epi8(R, A);
733 60 _mm_store_si128(reinterpret_cast<__m128i*>(dstp + x * 4), _mm_unpacklo_epi16(BG, RA)); // pixels 0..3
734 60 _mm_store_si128(reinterpret_cast<__m128i*>(dstp + x * 4 + 16), _mm_unpackhi_epi16(BG, RA)); // pixels 4..7
735 }
736 else {
737 60 __m128i BG = _mm_unpacklo_epi16(B, G);
738 60 __m128i RA = _mm_unpacklo_epi16(R, A);
739 60 _mm_store_si128(reinterpret_cast<__m128i*>(dstp + x * 8), _mm_unpacklo_epi32(BG, RA)); // pixels 0..1
740 60 _mm_store_si128(reinterpret_cast<__m128i*>(dstp + x * 8 + 16), _mm_unpackhi_epi32(BG, RA)); // pixels 2..3
741 }
742 }
743 20 dstp -= dst_pitch;
744 20 srcp[0] += src_pitch[0];
745 20 srcp[1] += src_pitch[1];
746 20 srcp[2] += src_pitch[2];
747 if constexpr (hasSrcAlpha)
748 10 srcp[3] += src_pitch[3];
749 }
750 4 }
751
752 //instantiate
753 //template<typename pixel_t, bool hasSrcAlpha>
754 template void convert_rgbp_to_rgba_sse2<uint8_t, false>(const BYTE* (&srcp)[4], BYTE* dstp, int(&src_pitch)[4], int dst_pitch, int width, int height);
755 template void convert_rgbp_to_rgba_sse2<uint8_t, true>(const BYTE* (&srcp)[4], BYTE* dstp, int(&src_pitch)[4], int dst_pitch, int width, int height);
756 template void convert_rgbp_to_rgba_sse2<uint16_t, false>(const BYTE* (&srcp)[4], BYTE* dstp, int(&src_pitch)[4], int dst_pitch, int width, int height);
757 template void convert_rgbp_to_rgba_sse2<uint16_t, true>(const BYTE* (&srcp)[4], BYTE* dstp, int(&src_pitch)[4], int dst_pitch, int width, int height);
758
759
760