GCC Code Coverage Report

Directory: avs_core/
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 80.7% 293 / 0 / 363
Functions: 94.1% 16 / 0 / 17
Branches: 79.7% 51 / 0 / 64
filters/intel/turn_avx2.cpp
Line Branch Exec Source
1 // AviSynth+. Copyright 2026- AviSynth+ Project
2 // https://avs-plus.net
3 // http://avisynth.nl
4 // This program is free software; you can redistribute it and/or modify
5 // it under the terms of the GNU General Public License as published by
6 // the Free Software Foundation; either version 2 of the License, or
7 // (at your option) any later version.
8 //
9 // This program is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 // GNU General Public License for more details.
13 //
14 // You should have received a copy of the GNU General Public License
15 // along with this program; if not, write to the Free Software
16 // Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA, or visit
17 // http://www.gnu.org/copyleft/gpl.html .
18 //
19 // Linking Avisynth statically or dynamically with other modules is making a
20 // combined work based on Avisynth. Thus, the terms and conditions of the GNU
21 // General Public License cover the whole combination.
22 //
23 // As a special exception, the copyright holders of Avisynth give you
24 // permission to link Avisynth with independent modules that communicate with
25 // Avisynth solely through the interfaces defined in avisynth.h, regardless of the license
26 // terms of these independent modules, and to copy and distribute the
27 // resulting combined work under terms of your choice, provided that
28 // every copy of the combined work is accompanied by a complete copy of
29 // the source code of Avisynth (the version of Avisynth used to produce the
30 // combined work), being distributed under the terms of the GNU General
31 // Public License plus this exception. An independent module is a module
32 // which is not derived from or based on Avisynth, such as 3rd-party filters,
33 // import and export plugins, or graphical user interfaces.
34
35 #include "../turn.h"
36 #include "turn_avx2.h"
37
38 #if defined(_MSC_VER)
39 #include <intrin.h> // MSVC
40 #else
41 #include <x86intrin.h> // GCC/MinGW/Clang/LLVM
42 #endif
43 #include <immintrin.h>
44
45 #include <cstdint>
46
47 //-------------------------------------------------------------------------------------------------
48 // 8-bit Transpose Kernels
49 //-------------------------------------------------------------------------------------------------
50
51 // The final winner over 8x8, 16x8 and 8x32.
52 // Transpose 32x8 (input) -> 8x32 (output)
53 // Input: 32 rows of 8 pixels (uint8_t)
54 // Output: 8 rows of 32 pixels
55 static AVS_FORCEINLINE void transpose_32x8x8_avx2(const uint8_t* AVS_RESTRICT srcp, uint8_t* AVS_RESTRICT dstp, const int src_pitch, const int dst_pitch)
56 {
57 // Load 32 rows, 8 bytes each
58 // One 256 bit register contains 4x8 bytes, which are 4 rows apart, so A,E,I,M in reg0, B,F,J,N in reg1, etc.
59 // after reaching the 16th row, it continues with Q,U,Y,c in reg4, R,V,Z,d in reg5, etc.
60 ✗ auto load_quad_row = [](const uint8_t* base, int pitch, int offset) -> __m256i {
61 ✗ __m128i r0 = _mm_loadu_si64((const __m128i*)(base + (offset + 0 * 4) * pitch));
62 ✗ __m128i r1 = _mm_loadu_si64((const __m128i*)(base + (offset + 1 * 4) * pitch));
63 ✗ __m128i r2 = _mm_loadu_si64((const __m128i*)(base + (offset + 2 * 4) * pitch));
64 ✗ __m128i r3 = _mm_loadu_si64((const __m128i*)(base + (offset + 3 * 4) * pitch));
65
66 ✗ __m128i r01 = _mm_unpacklo_epi64(r0, r1);
67 ✗ __m128i r23 = _mm_unpacklo_epi64(r2, r3);
68 ✗ return _mm256_inserti128_si256(_mm256_castsi128_si256(r01), r23, 1);
69 };
70
71 // Load 32 rows into 8 registers
72 ✗ __m256i r0 = load_quad_row(srcp, src_pitch, 0); // Rows 0,4,8,12
73 ✗ __m256i r1 = load_quad_row(srcp, src_pitch, 1); // Rows 1,5,9,13
74 ✗ __m256i r2 = load_quad_row(srcp, src_pitch, 2); // Rows 2,6,10,14
75 ✗ __m256i r3 = load_quad_row(srcp, src_pitch, 3); // Rows 3,7,11,15
76 ✗ __m256i r4 = load_quad_row(srcp, src_pitch, 16); // Rows 16,20,24,28
77 ✗ __m256i r5 = load_quad_row(srcp, src_pitch, 17); // Rows 17,21,25,29
78 ✗ __m256i r6 = load_quad_row(srcp, src_pitch, 18); // Rows 18,22,26,30
79 ✗ __m256i r7 = load_quad_row(srcp, src_pitch, 19); // Rows 19,23,27,31
80
81 ✗ __m256i t0 = _mm256_unpacklo_epi8(r0, r1);
82 ✗ __m256i t1 = _mm256_unpackhi_epi8(r0, r1);
83 ✗ __m256i t2 = _mm256_unpacklo_epi8(r2, r3);
84 ✗ __m256i t3 = _mm256_unpackhi_epi8(r2, r3);
85 ✗ __m256i t4 = _mm256_unpacklo_epi8(r4, r5);
86 ✗ __m256i t5 = _mm256_unpackhi_epi8(r4, r5);
87 ✗ __m256i t6 = _mm256_unpacklo_epi8(r6, r7);
88 ✗ __m256i t7 = _mm256_unpackhi_epi8(r6, r7);
89
90 ✗ __m256i u0 = _mm256_unpacklo_epi16(t0, t2);
91 ✗ __m256i u1 = _mm256_unpacklo_epi16(t1, t3);
92 ✗ __m256i u2 = _mm256_unpackhi_epi16(t0, t2);
93 ✗ __m256i u3 = _mm256_unpackhi_epi16(t1, t3);
94 ✗ __m256i u4 = _mm256_unpacklo_epi16(t4, t6);
95 ✗ __m256i u5 = _mm256_unpacklo_epi16(t5, t7);
96 ✗ __m256i u6 = _mm256_unpackhi_epi16(t4, t6);
97 ✗ __m256i u7 = _mm256_unpackhi_epi16(t5, t7);
98
99 ✗ __m256i v0 = _mm256_unpacklo_epi32(u0, u1);
100 ✗ __m256i v1 = _mm256_unpackhi_epi32(u0, u1);
101 ✗ __m256i v2 = _mm256_unpacklo_epi32(u2, u3);
102 ✗ __m256i v3 = _mm256_unpackhi_epi32(u2, u3);
103 ✗ __m256i v4 = _mm256_unpacklo_epi32(u4, u5);
104 ✗ __m256i v5 = _mm256_unpackhi_epi32(u4, u5);
105 ✗ __m256i v6 = _mm256_unpacklo_epi32(u6, u7);
106 ✗ __m256i v7 = _mm256_unpackhi_epi32(u6, u7);
107
108 ✗ __m256i w0 = _mm256_unpacklo_epi64(v0, v4); // AH0, QX0, IP0, Yf0
109 ✗ __m256i w1 = _mm256_unpackhi_epi64(v0, v4); // AH1, QX1, IP1, Yf1
110 ✗ __m256i w2 = _mm256_unpacklo_epi64(v1, v5); // AH2, QX2, IP2, Yf2
111 ✗ __m256i w3 = _mm256_unpackhi_epi64(v1, v5); // AH3, QX3, IP3, Yf3
112 ✗ __m256i w4 = _mm256_unpacklo_epi64(v2, v6); // AH4, QX4, IP4, Yf4
113 ✗ __m256i w5 = _mm256_unpackhi_epi64(v2, v6); // AH5, QX5, IP5, Yf5
114 ✗ __m256i w6 = _mm256_unpacklo_epi64(v3, v7); // AH6, QX6, IP6, Yf6
115 ✗ __m256i w7 = _mm256_unpackhi_epi64(v3, v7); // AH7, QX7, IP7, Yf7
116
117 // Cross-lane permute to fix AVX2 dual-lane unpack.
118 // Arrange 64-bit blocks into the correct linear order
119 ✗ const int PERM_CTRL = 0b11'01'10'00; // Standard 0,2,1,3 permute
120 ✗ __m256i out0 = _mm256_permute4x64_epi64(w0, PERM_CTRL); // AH0 | IP0 | QX0 | Yf0: ABCEFGHIJKLMNOPQRTSUVWXYZabcdef_0 ready
121 ✗ __m256i out1 = _mm256_permute4x64_epi64(w1, PERM_CTRL); // AH1 | IP1 | QX1 | Yf1
122 ✗ __m256i out2 = _mm256_permute4x64_epi64(w2, PERM_CTRL); // AH2 | IP2 | QX2 | Yf2
123 ✗ __m256i out3 = _mm256_permute4x64_epi64(w3, PERM_CTRL); // AH3 | IP3 | QX3 | Yf3
124 ✗ __m256i out4 = _mm256_permute4x64_epi64(w4, PERM_CTRL); // AH4 | IP4 | QX4 | Yf4
125 ✗ __m256i out5 = _mm256_permute4x64_epi64(w5, PERM_CTRL); // AH5 | IP5 | QX5 | Yf5
126 ✗ __m256i out6 = _mm256_permute4x64_epi64(w6, PERM_CTRL); // AH6 | IP6 | QX6 | Yf6
127 ✗ __m256i out7 = _mm256_permute4x64_epi64(w7, PERM_CTRL); // AH7 | IP7 | QX7 | Yf7
128
129 _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + 0 * dst_pitch), out0);
130 ✗ _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + 1 * dst_pitch), out1);
131 ✗ _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + 2 * dst_pitch), out2);
132 ✗ _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + 3 * dst_pitch), out3);
133 ✗ _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + 4 * dst_pitch), out4);
134 ✗ _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + 5 * dst_pitch), out5);
135 ✗ _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + 6 * dst_pitch), out6);
136 ✗ _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + 7 * dst_pitch), out7);
137 ✗ }
138
139 28 void turn_right_plane_8_avx2(const BYTE* srcp, BYTE* dstp, int src_rowsize, int src_height, int src_pitch, int dst_pitch)
140 {
141 // Start at the BOTTOM left of the source
142 28 const uint8_t* s0 = srcp + src_pitch * (src_height - 1);
143
144 28 constexpr int PIXELS_W = 8; // width block size
145 28 constexpr int PIXELS_H = 32; // height block size
146
147 28 const int w = src_rowsize & ~(PIXELS_W * sizeof(uint8_t) - 1);
148 28 const int h = src_height & ~(PIXELS_H - 1);
149 28 const int simd_width_in_pixels = w / sizeof(uint8_t);
150
151
1/2
✗ Branch 88 → 3 not taken.
✓ Branch 88 → 89 taken 28 times.
 28 for (int y = 0; y < h; y += PIXELS_H)
152 {
153 // Destination starts at y offset
154 ✗ uint8_t* d0 = dstp + y;
155
156 ✗ for (int x = 0; x < w; x += PIXELS_W * sizeof(uint8_t))
157 {
158 // Use negative pitch to load rows bottom-to-top to avoid row reversing after transpose.
159 ✗ transpose_32x8x8_avx2(s0 + x, d0, -src_pitch, dst_pitch);
160
161 ✗ d0 += PIXELS_W * dst_pitch;
162 }
163 ✗ s0 -= PIXELS_H * src_pitch;
164 }
165
166 // Boundary handling fallback
167
1/2
✓ Branch 89 → 90 taken 28 times.
✗ Branch 89 → 91 not taken.
 28 if (src_rowsize != w)
168 {
169 // consider calling the sse2 version when difference is at least 8, since sse2 does 8x8 blocks
170 28 turn_right_plane_8_c(srcp + w, dstp + dst_pitch * simd_width_in_pixels, src_rowsize - w, src_height, src_pitch, dst_pitch);
171 }
172
173
1/2
✓ Branch 91 → 92 taken 28 times.
✗ Branch 91 → 93 not taken.
 28 if (src_height != h)
174 {
175 28 turn_right_plane_8_c(srcp, dstp + h * sizeof(uint8_t), src_rowsize, src_height - h, src_pitch, dst_pitch);
176 }
177 28 }
178
179 14 void turn_left_plane_8_avx2(const BYTE* srcp, BYTE* dstp, int src_rowsize, int src_height, int src_pitch, int dst_pitch)
180 {
181 14 turn_right_plane_8_avx2(srcp + src_pitch * (src_height - 1), dstp + dst_pitch * (src_rowsize / sizeof(uint8_t) - 1), src_rowsize, src_height, -src_pitch, -dst_pitch);
182 14 }
183
184 //-------------------------------------------------------------------------------------------------
185 // 16-bit Transpose Kernels
186 //-------------------------------------------------------------------------------------------------
187
188 // Transpose 8x16 (input) -> 16x8 (output)
189 // Input: 16 rows of 8 pixels (uint16_t)
190 // Output: 8 rows of 16 pixels
191 static AVS_FORCEINLINE void transpose_16x8x16_avx2(const BYTE* AVS_RESTRICT srcp, BYTE* AVS_RESTRICT dstp, const int src_pitch, const int dst_pitch)
192 {
193 // Helper to load 16 bytes from row i (low lane) and row i+8 (high lane)
194 32 auto load_dual_row_avx2 = [](const BYTE* base, int pitch, int row_idx) -> __m256i
195 {
196 32 __m128i lo = _mm_loadu_si128(reinterpret_cast<const __m128i*>(base + pitch * row_idx));
197 64 __m128i hi = _mm_loadu_si128(reinterpret_cast<const __m128i*>(base + pitch * (row_idx + 8)));
198 32 return _mm256_inserti128_si256(_mm256_castsi128_si256(lo), hi, 1);
199 };
200
201 // Load 16 rows into 8 registers.
202 // Row x in low lane and row x+8 in high lane.
203 4 __m256i v0 = load_dual_row_avx2(srcp, src_pitch, 0); // r0 | r8 // A0..A7 | I0..I7
204 4 __m256i v1 = load_dual_row_avx2(srcp, src_pitch, 1); // r1 | r9 // B0..B7 | J0..J7
205 4 __m256i v2 = load_dual_row_avx2(srcp, src_pitch, 2); // r2 | r10 // C0..C7 | K0..K7
206 4 __m256i v3 = load_dual_row_avx2(srcp, src_pitch, 3); // r3 | r11 // D0..D7 | L0..L7
207 4 __m256i v4 = load_dual_row_avx2(srcp, src_pitch, 4); // r4 | r12 // E0..E7 | M0..M7
208 4 __m256i v5 = load_dual_row_avx2(srcp, src_pitch, 5); // r5 | r13 // F0..F7 | N0..N7
209 4 __m256i v6 = load_dual_row_avx2(srcp, src_pitch, 6); // r6 | r14 // G0..G7 | O0..O7
210 4 __m256i v7 = load_dual_row_avx2(srcp, src_pitch, 7); // r7 | r15 // H0..H7 | P0..P7
211
212 // Standard 8x8 Transpose Logic (applied to both lanes simultaneously)
213 4 __m256i t0 = _mm256_unpacklo_epi16(v0, v1); // A0 B0 A1 B1 A2 B2 A3 B3 | I0 J0 I1 J1 I2 J2 I3 J3
214 4 __m256i t1 = _mm256_unpackhi_epi16(v0, v1); // A4 B4 A5 B5 A6 B6 A7 B7 | I4 J4 I5 J5 I6 J6 I7 J7
215 4 __m256i t2 = _mm256_unpacklo_epi16(v2, v3); // C0 D0 C1 D1 C2 D2 C3 D3 | K0 L0 K1 L1 K2 L2 K3 L3
216 4 __m256i t3 = _mm256_unpackhi_epi16(v2, v3); // C4 D4 C5 D5 C6 D6 C7 D7 | K4 L4 K5 L5 K6 L6 K7 L7
217 4 __m256i t4 = _mm256_unpacklo_epi16(v4, v5); // E0 F0 E1 F1 E2 F2 E3 F3 | M0 N0 M1 N1 M2 N2 M3 N3
218 4 __m256i t5 = _mm256_unpackhi_epi16(v4, v5); // E4 F4 E5 F5 E6 F6 E7 F7 | M4 N4 M5 N5 M6 N6 M7 N7
219 4 __m256i t6 = _mm256_unpacklo_epi16(v6, v7); // G0 H0 G1 H1 G2 H2 G3 H3 | O0 P0 O1 P1 O2 P2 O3 P3
220 4 __m256i t7 = _mm256_unpackhi_epi16(v6, v7); // G4 H4 G5 H5 G6 H6 G7 H7 | O4 P4 O5 P5 O6 P6 O7 P7
221
222 4 __m256i m0 = _mm256_unpacklo_epi32(t0, t2); // AD0,AD1 | IL0,IL1
223 4 __m256i m1 = _mm256_unpackhi_epi32(t0, t2); // AD2,AD3 | IL2,IL3
224 4 __m256i m2 = _mm256_unpacklo_epi32(t1, t3); // AD4,AD5 | IL4,IL5
225 4 __m256i m3 = _mm256_unpackhi_epi32(t1, t3); // AD6,AD7 | IL6,IL7
226 4 __m256i m4 = _mm256_unpacklo_epi32(t4, t6); // EH0,EH1 | MP0,MP1
227 4 __m256i m5 = _mm256_unpackhi_epi32(t4, t6); // EH2,EH3 | MP2,MP3
228 4 __m256i m6 = _mm256_unpacklo_epi32(t5, t7); // EH4,EH5 | MP4,MP5
229 4 __m256i m7 = _mm256_unpackhi_epi32(t5, t7); // EH6,EH7 | MP6,MP7
230
231 4 __m256i out0 = _mm256_unpacklo_epi64(m0, m4); // AD0,EH0 | IL0,MP0
232 4 __m256i out1 = _mm256_unpackhi_epi64(m0, m4); // AD1,EH1 | IL1,MP1
233 4 __m256i out2 = _mm256_unpacklo_epi64(m1, m5); // AD2,EH2 | IL2,MP2
234 4 __m256i out3 = _mm256_unpackhi_epi64(m1, m5); // AD3,EH3 | IL3,MP3
235 4 __m256i out4 = _mm256_unpacklo_epi64(m2, m6); // AD4,EH4 | IL4,MP4
236 4 __m256i out5 = _mm256_unpackhi_epi64(m2, m6); // AD5,EH5 | IL5,MP5
237 4 __m256i out6 = _mm256_unpacklo_epi64(m3, m7); // AD6,EH6 | IL6,MP6
238 4 __m256i out7 = _mm256_unpackhi_epi64(m3, m7); // AD7,EH7 | IL7,MP7
239
240 _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + dst_pitch * 0), out0);
241 4 _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + dst_pitch * 1), out1);
242 4 _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + dst_pitch * 2), out2);
243 4 _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + dst_pitch * 3), out3);
244 4 _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + dst_pitch * 4), out4);
245 4 _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + dst_pitch * 5), out5);
246 4 _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + dst_pitch * 6), out6);
247 4 _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + dst_pitch * 7), out7);
248 4 }
249
250 8 void turn_right_plane_16_avx2(const BYTE* srcp, BYTE* dstp, int src_rowsize, int src_height, int src_pitch, int dst_pitch)
251 {
252 // source order: bottom to top
253 8 const BYTE* s0 = srcp + src_pitch * (src_height - 1);
254
255 8 constexpr int PIXELS_W = 8; // width block size
256 8 constexpr int PIXELS_H = 16; // height block size
257
258 8 const int w = src_rowsize & ~(PIXELS_W * sizeof(uint16_t) - 1);
259 8 const int h = src_height & ~(PIXELS_H - 1);
260 8 const int simd_width_in_pixels = w / sizeof(uint16_t);
261
262
2/2
✓ Branch 72 → 3 taken 2 times.
✓ Branch 72 → 73 taken 8 times.
 10 for (int y = 0; y < h; y += PIXELS_H)
263 {
264 2 BYTE* d0 = dstp + y * sizeof(uint16_t);
265
266
2/2
✓ Branch 70 → 4 taken 4 times.
✓ Branch 70 → 71 taken 2 times.
 6 for (int x = 0; x < w; x += PIXELS_W * sizeof(uint16_t))
267 {
268 // Use negative pitch to load rows bottom-to-top to avoid row reversing after transpose.
269 4 transpose_16x8x16_avx2(s0 + x, d0, -src_pitch, dst_pitch);
270 4 d0 += PIXELS_W * dst_pitch;
271 }
272 2 s0 -= PIXELS_H * src_pitch;
273 }
274
275 // Boundary handling
276
1/2
✓ Branch 73 → 74 taken 8 times.
✗ Branch 73 → 75 not taken.
 8 if (src_rowsize != w)
277 {
278 8 turn_right_plane_16_c(srcp + w, dstp + dst_pitch * simd_width_in_pixels, src_rowsize - w, src_height, src_pitch, dst_pitch);
279 }
280
281
1/2
✓ Branch 75 → 76 taken 8 times.
✗ Branch 75 → 77 not taken.
 8 if (src_height != h)
282 {
283 8 turn_right_plane_16_c(srcp, dstp + h * sizeof(uint16_t), src_rowsize, src_height - h, src_pitch, dst_pitch);
284 }
285 8 }
286
287 4 void turn_left_plane_16_avx2(const BYTE* srcp, BYTE* dstp, int src_rowsize, int src_height, int src_pitch, int dst_pitch)
288 {
289 4 turn_right_plane_16_avx2(srcp + src_pitch * (src_height - 1), dstp + dst_pitch * (src_rowsize / sizeof(uint16_t) - 1), src_rowsize, src_height, -src_pitch, -dst_pitch);
290 4 }
291
292 //-------------------------------------------------------------------------------------------------
293 // 32-bit Transpose Kernels (Float/RGB32)
294 //-------------------------------------------------------------------------------------------------
295
296 static AVS_FORCEINLINE void transpose_8x8_32bit_avx2(const BYTE* AVS_RESTRICT srcp, BYTE* AVS_RESTRICT dstp, int src_pitch, int dst_pitch)
297 {
298 // load 8 rows of 8 pixels (32-bit each)
299 4 __m256i r0 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(srcp + src_pitch * 0));
300 4 __m256i r1 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(srcp + src_pitch * 1));
301 4 __m256i r2 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(srcp + src_pitch * 2));
302 4 __m256i r3 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(srcp + src_pitch * 3));
303 4 __m256i r4 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(srcp + src_pitch * 4));
304 4 __m256i r5 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(srcp + src_pitch * 5));
305 4 __m256i r6 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(srcp + src_pitch * 6));
306 8 __m256i r7 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(srcp + src_pitch * 7));
307
308 4 __m256i t0 = _mm256_unpacklo_epi32(r0, r1);
309 4 __m256i t1 = _mm256_unpackhi_epi32(r0, r1);
310 4 __m256i t2 = _mm256_unpacklo_epi32(r2, r3);
311 4 __m256i t3 = _mm256_unpackhi_epi32(r2, r3);
312 4 __m256i t4 = _mm256_unpacklo_epi32(r4, r5);
313 4 __m256i t5 = _mm256_unpackhi_epi32(r4, r5);
314 4 __m256i t6 = _mm256_unpacklo_epi32(r6, r7);
315 4 __m256i t7 = _mm256_unpackhi_epi32(r6, r7);
316
317 4 __m256i q0 = _mm256_unpacklo_epi64(t0, t2);
318 4 __m256i q1 = _mm256_unpackhi_epi64(t0, t2);
319 4 __m256i q2 = _mm256_unpacklo_epi64(t1, t3);
320 4 __m256i q3 = _mm256_unpackhi_epi64(t1, t3);
321 4 __m256i q4 = _mm256_unpacklo_epi64(t4, t6);
322 4 __m256i q5 = _mm256_unpackhi_epi64(t4, t6);
323 4 __m256i q6 = _mm256_unpacklo_epi64(t5, t7);
324 4 __m256i q7 = _mm256_unpackhi_epi64(t5, t7);
325
326 // cross-lane permute
327 // out0..3 gets low lane of (rows 0-3) and low lane of (rows 4-7)
328 // out4..7 gets high lane of (rows 0-3) and high lane of (rows 4-7)
329 4 __m256i out0 = _mm256_permute2x128_si256(q0, q4, 0x20);
330 4 __m256i out1 = _mm256_permute2x128_si256(q1, q5, 0x20);
331 4 __m256i out2 = _mm256_permute2x128_si256(q2, q6, 0x20);
332 4 __m256i out3 = _mm256_permute2x128_si256(q3, q7, 0x20);
333 4 __m256i out4 = _mm256_permute2x128_si256(q0, q4, 0x31);
334 4 __m256i out5 = _mm256_permute2x128_si256(q1, q5, 0x31);
335 4 __m256i out6 = _mm256_permute2x128_si256(q2, q6, 0x31);
336 4 __m256i out7 = _mm256_permute2x128_si256(q3, q7, 0x31);
337
338 _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + dst_pitch * 0), out0);
339 4 _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + dst_pitch * 1), out1);
340 4 _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + dst_pitch * 2), out2);
341 4 _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + dst_pitch * 3), out3);
342 4 _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + dst_pitch * 4), out4);
343 4 _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + dst_pitch * 5), out5);
344 4 _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + dst_pitch * 6), out6);
345 4 _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + dst_pitch * 7), out7);
346 4 }
347
348 // float, rgb32
349 4 void turn_right_plane_32_avx2(const BYTE* srcp, BYTE* dstp, int src_rowsize, int src_height, int src_pitch, int dst_pitch)
350 {
351 // source order: bottom to top
352 4 const BYTE* s0 = srcp + src_pitch * (src_height - 1);
353
354 4 constexpr int PIXELS_W = 8; // width block size
355 4 constexpr int PIXELS_H = 8; // height block size
356
357 4 const int w = src_rowsize & ~(PIXELS_W * sizeof(uint32_t) - 1);
358 4 const int h = src_height & ~(PIXELS_H - 1);
359 4 const int simd_width_in_pixels = w / sizeof(uint32_t);
360
361
2/2
✓ Branch 64 → 3 taken 4 times.
✓ Branch 64 → 65 taken 4 times.
 8 for (int y = 0; y < h; y += PIXELS_H)
362 {
363 // Destination grows forward: y index becomes the x-offset in the rotated image
364 4 BYTE* d0 = dstp + y * sizeof(uint32_t);
365
366
2/2
✓ Branch 62 → 4 taken 4 times.
✓ Branch 62 → 63 taken 4 times.
 8 for (int x = 0; x < w; x += PIXELS_W * sizeof(uint32_t))
367 {
368 // Use negative pitch to load rows bottom-to-top to avoid row reversing after transpose.
369 4 transpose_8x8_32bit_avx2(s0 + x, d0, -src_pitch, dst_pitch);
370 4 d0 += PIXELS_W * dst_pitch;
371 }
372 4 s0 -= PIXELS_H * src_pitch;
373 }
374
375 // Boundary handling fallbacks
376
1/2
✓ Branch 65 → 66 taken 4 times.
✗ Branch 65 → 67 not taken.
 4 if (src_rowsize != w)
377 {
378 4 turn_right_plane_32_c(srcp + w, dstp + dst_pitch * simd_width_in_pixels, src_rowsize - w, src_height, src_pitch, dst_pitch);
379 }
380
1/2
✓ Branch 67 → 68 taken 4 times.
✗ Branch 67 → 69 not taken.
 4 if (src_height != h)
381 {
382 4 turn_right_plane_32_c(srcp, dstp + h * sizeof(uint32_t), src_rowsize, src_height - h, src_pitch, dst_pitch);
383 }
384 4 }
385
386 2 void turn_left_plane_32_avx2(const BYTE* srcp, BYTE* dstp, int src_rowsize, int src_height, int src_pitch, int dst_pitch)
387 {
388 2 turn_right_plane_32_avx2(srcp + src_pitch * (src_height - 1), dstp + dst_pitch * (src_rowsize / sizeof(float) - 1), src_rowsize, src_height, -src_pitch, -dst_pitch);
389 2 }
390
391 1 void turn_left_rgb32_avx2(const BYTE* srcp, BYTE* dstp, int src_rowsize, int src_height, int src_pitch, int dst_pitch)
392 {
393 // packed rgb is upside down
394 1 turn_right_plane_32_avx2(srcp, dstp, src_rowsize, src_height, src_pitch, dst_pitch);
395 1 }
396
397 1 void turn_right_rgb32_avx2(const BYTE* srcp, BYTE* dstp, int src_rowsize, int src_height, int src_pitch, int dst_pitch)
398 {
399 // packed rgb is upside down
400 1 turn_left_plane_32_avx2(srcp, dstp, src_rowsize, src_height, src_pitch, dst_pitch);
401 1 }
402
403 //-------------------------------------------------------------------------------------------------
404 // 64-bit Transpose Kernels (RGB64)
405 //-------------------------------------------------------------------------------------------------
406
407 static AVS_FORCEINLINE void transpose_4x4_64bit_avx2(const BYTE* AVS_RESTRICT srcp, BYTE* AVS_RESTRICT dstp, int src_pitch, int dst_pitch)
408 {
409 // Load 4 rows (4 pixels each reg = 32 bytes = 1 YMM register)
410 4 __m256i r0 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(srcp + src_pitch * 0));
411 4 __m256i r1 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(srcp + src_pitch * 1));
412 4 __m256i r2 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(srcp + src_pitch * 2));
413 8 __m256i r3 = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(srcp + src_pitch * 3));
414
415 4 __m256i t0 = _mm256_unpacklo_epi64(r0, r1); // [R0_P0, R1_P0 | R0_P2, R1_P2]
416 4 __m256i t1 = _mm256_unpackhi_epi64(r0, r1); // [R0_P1, R1_P1 | R0_P3, R1_P3]
417 4 __m256i t2 = _mm256_unpacklo_epi64(r2, r3); // [R2_P0, R3_P0 | R2_P2, R3_P2]
418 4 __m256i t3 = _mm256_unpackhi_epi64(r2, r3); // [R2_P1, R3_P1 | R2_P3, R3_P3]
419
420 // Cross lane permute
421 // Combine low lanes of t0/t2, t1/t3 and high lanes of t0/t2, t1/t3
422 4 __m256i d0 = _mm256_permute2x128_si256(t0, t2, 0x20); // Column 0: [R0-R3]_P0
423 4 __m256i d1 = _mm256_permute2x128_si256(t1, t3, 0x20); // Column 1: [R0-R3]_P1
424 4 __m256i d2 = _mm256_permute2x128_si256(t0, t2, 0x31); // Column 2: [R0-R3]_P2
425 4 __m256i d3 = _mm256_permute2x128_si256(t1, t3, 0x31); // Column 3: [R0-R3]_P3
426
427 _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + dst_pitch * 0), d0);
428 4 _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + dst_pitch * 1), d1);
429 4 _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + dst_pitch * 2), d2);
430 4 _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstp + dst_pitch * 3), d3);
431 4 }
432
433 4 void turn_right_plane_64_avx2(const BYTE* srcp, BYTE* dstp, int src_rowsize, int src_height, int src_pitch, int dst_pitch)
434 {
435 // Start at bottom row
436 4 const BYTE* s0 = srcp + src_pitch * (src_height - 1);
437
438 4 constexpr int PIXELS_W = 4;
439 4 constexpr int PIXELS_H = 4;
440
441 4 const int w = src_rowsize & ~(PIXELS_W * sizeof(uint64_t) - 1);
442 4 const int h = src_height & ~(PIXELS_H - 1);
443 4 const int simd_width_in_pixels = w / sizeof(uint64_t);
444
445
2/2
✓ Branch 28 → 3 taken 4 times.
✓ Branch 28 → 29 taken 4 times.
 8 for (int y = 0; y < h; y += PIXELS_H)
446 {
447 // Destination starts at y offset (which is the x-coordinate in turn-right)
448 4 BYTE* d0 = dstp + y * sizeof(uint64_t);
449
450
2/2
✓ Branch 26 → 4 taken 4 times.
✓ Branch 26 → 27 taken 4 times.
 8 for (int x = 0; x < w; x += PIXELS_W * sizeof(uint64_t))
451 {
452 // Standard transpose + Negative Pitch = 90 CW Turn
453 4 transpose_4x4_64bit_avx2(s0 + x, d0, -src_pitch, dst_pitch);
454 4 d0 += PIXELS_W * dst_pitch;
455 }
456 4 s0 -= PIXELS_H * src_pitch;
457 }
458
459
2/2
✓ Branch 29 → 30 taken 3 times.
✓ Branch 29 → 31 taken 1 time.
 4 if (src_rowsize != w)
460 {
461 3 turn_right_plane_c<uint64_t>(srcp + w, dstp + dst_pitch * simd_width_in_pixels, src_rowsize - w, src_height, src_pitch, dst_pitch);
462 }
463
2/2
✓ Branch 31 → 32 taken 2 times.
✓ Branch 31 → 33 taken 2 times.
 4 if (src_height != h)
464 {
465 2 turn_right_plane_c<uint64_t>(srcp, dstp + h * sizeof(uint64_t), src_rowsize, src_height - h, src_pitch, dst_pitch);
466 }
467
468 4 }
469
470 2 void turn_left_rgb64_avx2(const BYTE* srcp, BYTE* dstp, int src_rowsize, int src_height, int src_pitch, int dst_pitch)
471 {
472 // packed rgb is upside down
473 2 turn_right_plane_64_avx2(srcp, dstp, src_rowsize, src_height, src_pitch, dst_pitch);
474 2 }
475
476 2 void turn_right_rgb64_avx2(const BYTE* srcp, BYTE* dstp, int src_rowsize, int src_height, int src_pitch, int dst_pitch)
477 {
478 // packed rgb is upside down
479 2 turn_right_plane_64_avx2(srcp + src_pitch * (src_height - 1), dstp + dst_pitch * (src_rowsize / sizeof(uint64_t) - 1), src_rowsize, src_height, -src_pitch, -dst_pitch);
480 2 }
481
482 //-------------------------------------------------------------------------------------------------
483 // Turn 180 (Flip)
484 //-------------------------------------------------------------------------------------------------
485
486 template <typename T>
487 14 void turn_180_plane_avx2(const BYTE* srcp, BYTE* dstp, int src_rowsize, int src_height, int src_pitch, int dst_pitch)
488 {
489 14 const BYTE* AVS_RESTRICT s0 = srcp;
490 14 const int row_bytes = src_rowsize; // src_rowsize is the byte width of the row.
491
492 // d0 points to the byte after the last pixel (byte) of the last row of dst.
493 // dst_pitch * (src_height - 1) is the start of the last row.
494 // + row_bytes is the address after the last byte of that row.
495 14 BYTE* AVS_RESTRICT d0 = dstp + dst_pitch * (src_height - 1) + row_bytes;
496
497 14 constexpr int vector_bytes = 32;
498 14 const int aligned_row_bytes = row_bytes & ~(vector_bytes - 1);
499
500 // Prepare Shuffle Masks (These masks are correct for byte reversal within lanes)
501 __m256i shuf_mask; // only for 8/16-bit types
502 if constexpr (sizeof(T) == 1) {
503 // Reverse bytes in 16-byte chunks (31..16 and 15..0)
504 8 shuf_mask = _mm256_set_epi8(
505 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
506 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
507 );
508 }
509 else if constexpr (sizeof(T) == 2) {
510 // Reverse words (16-bit elements) in 16-byte chunks
511 4 shuf_mask = _mm256_set_epi8(
512 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14,
513 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14
514 );
515 }
516 // For 32/64 we use permute instructions, no shuffle mask needed.
517
518
8/8
void turn_180_plane_avx2<unsigned char>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 18 → 5 taken 30 times.
✓ Branch 18 → 19 taken 8 times.
void turn_180_plane_avx2<unsigned int>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 18 → 3 taken 7 times.
✓ Branch 18 → 19 taken 1 time.
void turn_180_plane_avx2<unsigned long>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 14 → 3 taken 7 times.
✓ Branch 14 → 15 taken 1 time.
void turn_180_plane_avx2<unsigned short>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 18 → 5 taken 15 times.
✓ Branch 18 → 19 taken 4 times.
 73 for (int y = 0; y < src_height; ++y)
519 {
520 // Main AVX2 Loop
521
8/8
void turn_180_plane_avx2<unsigned char>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 12 → 6 taken 9 times.
✓ Branch 12 → 13 taken 30 times.
void turn_180_plane_avx2<unsigned int>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 12 → 4 taken 7 times.
✓ Branch 12 → 13 taken 7 times.
void turn_180_plane_avx2<unsigned long>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 8 → 4 taken 7 times.
✓ Branch 8 → 9 taken 7 times.
void turn_180_plane_avx2<unsigned short>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 12 → 6 taken 7 times.
✓ Branch 12 → 13 taken 15 times.
 89 for (int x = 0; x < aligned_row_bytes; x += vector_bytes)
522 {
523 53 __m256i src = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(s0 + x));
524
525 if constexpr (sizeof(T) == 8) { // uint64_t (4 pixels)
526 // Swap 64-bit elements: 3, 2, 1, 0
527 7 src = _mm256_permute4x64_epi64(src, _MM_SHUFFLE(0, 1, 2, 3));
528 }
529 else if constexpr (sizeof(T) == 4) { // uint32_t (8 pixels)
530 // Reverse 32-bit elements: 7, 6, 5, 4, 3, 2, 1, 0
531 7 const __m256i idx = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7);
532 7 src = _mm256_permutevar8x32_epi32(src, idx);
533 }
534 else { // uint8_t or uint16_t
535 // reverse within 128-bit lanes (using precomputed mask)
536 16 src = _mm256_shuffle_epi8(src, shuf_mask);
537 // Swap the 128-bit lanes
538 16 src = _mm256_permute2x128_si256(src, src, 0x01);
539 }
540
541 // Store backwards: d0 points to end, x is byte offset from start.
542 30 _mm256_storeu_si256(reinterpret_cast<__m256i*>(d0 - x - vector_bytes), src);
543 }
544
545 // fallback for leftovers
546 59 int x = aligned_row_bytes;
547 59 int rem_bytes = row_bytes - x;
548
549
4/8
void turn_180_plane_avx2<unsigned char>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 13 → 14 taken 30 times.
✗ Branch 13 → 17 not taken.
void turn_180_plane_avx2<unsigned int>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 13 → 14 taken 7 times.
✗ Branch 13 → 17 not taken.
void turn_180_plane_avx2<unsigned long>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 9 → 10 taken 7 times.
✗ Branch 9 → 13 not taken.
void turn_180_plane_avx2<unsigned short>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 13 → 14 taken 15 times.
✗ Branch 13 → 17 not taken.
 59 if (rem_bytes > 0) {
550 59 const BYTE* src_tail = s0 + x;
551
552 // Destination pointer: The first pixel of the remainder (src_tail)
553 // must go to the address *just before* where the AVX loop finished writing.
554 // The AVX loop finished writing at: d0 - aligned_row_bytes.
555 // The remaining bytes (rem_bytes) are written to fill the gap.
556 // Start of destination remainder region: d0 - row_bytes
557 59 BYTE* dst_rem_start = d0 - row_bytes;
558
559 59 const int n_pixels = rem_bytes / sizeof(T);
560
561 // Pointers for pixel-wise copy and reverse.
562 59 const T* s_ptr = reinterpret_cast<const T*>(src_tail);
563
564 // d_ptr starts at the last pixel position of the remainder region,
565 // and moves backward to reverse the order.
566 59 T* d_ptr = reinterpret_cast<T*>(dst_rem_start + rem_bytes) - 1;
567
568
8/8
void turn_180_plane_avx2<unsigned char>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 16 → 15 taken 114 times.
✓ Branch 16 → 17 taken 30 times.
void turn_180_plane_avx2<unsigned int>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 16 → 15 taken 7 times.
✓ Branch 16 → 17 taken 7 times.
void turn_180_plane_avx2<unsigned long>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 12 → 11 taken 7 times.
✓ Branch 12 → 13 taken 7 times.
void turn_180_plane_avx2<unsigned short>(unsigned char const*, unsigned char*, int, int, int, int):
✓ Branch 16 → 15 taken 43 times.
✓ Branch 16 → 17 taken 15 times.
 230 for (int k = 0; k < n_pixels; ++k) {
569 171 *d_ptr = s_ptr[k];
570 171 d_ptr--; // Move backward
571 }
572 }
573 59 s0 += src_pitch;
574 59 d0 -= dst_pitch;
575 }
576 14 }
577
578 // Instantiate templates
579 template void turn_180_plane_avx2<uint8_t>(const BYTE* srcp, BYTE* dstp, int src_rowsize, int src_height, int src_pitch, int dst_pitch);
580 template void turn_180_plane_avx2<uint16_t>(const BYTE* srcp, BYTE* dstp, int src_rowsize, int src_height, int src_pitch, int dst_pitch);
581 template void turn_180_plane_avx2<uint32_t>(const BYTE* srcp, BYTE* dstp, int src_rowsize, int src_height, int src_pitch, int dst_pitch);
582 template void turn_180_plane_avx2<uint64_t>(const BYTE* srcp, BYTE* dstp, int src_rowsize, int src_height, int src_pitch, int dst_pitch);
583