GCC Code Coverage Report

Directory: avs_core/
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 100.0% 360 / 0 / 360
Functions: 100.0% 25 / 0 / 25
Branches: 90.0% 81 / 0 / 90
filters/overlay/intel/444convert_sse.cpp
Line Branch Exec Source
1 // Avisynth v2.5. Copyright 2002 Ben Rudiak-Gould et al.
2 // http://avisynth.nl
3
4 // This program is free software; you can redistribute it and/or modify
5 // it under the terms of the GNU General Public License as published by
6 // the Free Software Foundation; either version 2 of the License, or
7 // (at your option) any later version.
8 //
9 // This program is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 // GNU General Public License for more details.
13 //
14 // You should have received a copy of the GNU General Public License
15 // along with this program; if not, write to the Free Software
16 // Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA, or visit
17 // http://www.gnu.org/copyleft/gpl.html .
18 //
19 // Linking Avisynth statically or dynamically with other modules is making a
20 // combined work based on Avisynth. Thus, the terms and conditions of the GNU
21 // General Public License cover the whole combination.
22 //
23 // As a special exception, the copyright holders of Avisynth give you
24 // permission to link Avisynth with independent modules that communicate with
25 // Avisynth solely through the interfaces defined in avisynth.h, regardless of the license
26 // terms of these independent modules, and to copy and distribute the
27 // resulting combined work under terms of your choice, provided that
28 // every copy of the combined work is accompanied by a complete copy of
29 // the source code of Avisynth (the version of Avisynth used to produce the
30 // combined work), being distributed under the terms of the GNU General
31 // Public License plus this exception. An independent module is a module
32 // which is not derived from or based on Avisynth, such as 3rd-party filters,
33 // import and export plugins, or graphical user interfaces.
34
35 // Overlay (c) 2003, 2004 by Klaus Post
36
37 #include "444convert_sse.h"
38 #include "../../../core/internal.h"
39
40 // Intrinsics base header + required extension headers
41 #if defined(_MSC_VER)
42 #include <intrin.h>
43 #else
44 #include <x86intrin.h>
45 #endif
46 #include <smmintrin.h> // SSE4.1
47
48 // fast in-place conversions from and to 4:4:4
49
50 /***** YV12 -> YUV 4:4:4 ******/
51
52 template<typename pixel_t>
53 18 static void convert_yv12_chroma_to_yv24_sse2(BYTE *dstp, const BYTE *srcp, int dst_pitch, int src_pitch, int src_width, int src_height) {
54 18 src_width *= sizeof(pixel_t);
55 18 int mod8_width = src_width / 8 * 8;
56
4/4
void convert_yv12_chroma_to_yv24_sse2<unsigned char>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 29 → 3 taken 37 times.
✓ Branch 29 → 30 taken 17 times.
void convert_yv12_chroma_to_yv24_sse2<unsigned short>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 29 → 3 taken 5 times.
✓ Branch 29 → 30 taken 1 time.
 60 for (int y = 0; y < src_height; ++y) {
57
4/4
void convert_yv12_chroma_to_yv24_sse2<unsigned char>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 15 → 4 taken 10 times.
✓ Branch 15 → 16 taken 37 times.
void convert_yv12_chroma_to_yv24_sse2<unsigned short>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 15 → 4 taken 10 times.
✓ Branch 15 → 16 taken 5 times.
 62 for (int x = 0; x < mod8_width; x+=8) {
58 // 0 0 0 0 0 0 0 0 U7 U6 U5 U4 U3 U2 U1 U0 for 8 bits
59 // 0 0 0 0 U3 U2 U1 U0 for 16 bits
60 40 __m128i src = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(srcp+x));
61 if constexpr(sizeof(pixel_t) == 1)
62 10 src = _mm_unpacklo_epi8(src, src); //U7 U7 U6 U6 U5 U5 U4 U4 U3 U3 U2 U2 U1 U1 U0 U0
63 else
64 10 src = _mm_unpacklo_epi16(src, src); //U3 U3 U2 U2 U1 U1 U0 U0
65
66 20 _mm_store_si128(reinterpret_cast<__m128i*>(dstp+x*2), src);
67 20 _mm_store_si128(reinterpret_cast<__m128i*>(dstp+x*2 + dst_pitch), src);
68 }
69
70
2/4
void convert_yv12_chroma_to_yv24_sse2<unsigned char>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 16 → 17 taken 37 times.
✗ Branch 16 → 28 not taken.
void convert_yv12_chroma_to_yv24_sse2<unsigned short>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 16 → 17 taken 5 times.
✗ Branch 16 → 28 not taken.
 42 if (mod8_width != src_width) {
71 84 __m128i src = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(srcp+src_width - 8));
72 if constexpr(sizeof(pixel_t) == 1)
73 37 src = _mm_unpacklo_epi8(src, src);
74 else
75 5 src = _mm_unpacklo_epi16(src, src);
76
77 42 _mm_storeu_si128(reinterpret_cast<__m128i*>(dstp + (src_width * 2) - 16), src);
78 42 _mm_storeu_si128(reinterpret_cast<__m128i*>(dstp + (src_width * 2) - 16 + dst_pitch), src);
79 }
80
81 42 dstp += dst_pitch*2;
82 42 srcp += src_pitch;
83 }
84 18 }
85
86
87
88 /***** YV16 -> YUV 4:4:4 ******/
89
90 template<typename pixel_t>
91 10 static void convert_yv16_chroma_to_yv24_sse2(BYTE *dstp, const BYTE *srcp, int dst_pitch, int src_pitch, int src_width, int src_height) {
92 10 src_width *= sizeof(pixel_t);
93 10 int mod8_width = src_width / 8 * 8;
94
4/4
void convert_yv16_chroma_to_yv24_sse2<unsigned char>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 27 → 3 taken 45 times.
✓ Branch 27 → 28 taken 9 times.
void convert_yv16_chroma_to_yv24_sse2<unsigned short>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 27 → 3 taken 5 times.
✓ Branch 27 → 28 taken 1 time.
 60 for (int y = 0; y < src_height; ++y) {
95
4/4
void convert_yv16_chroma_to_yv24_sse2<unsigned char>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 14 → 4 taken 10 times.
✓ Branch 14 → 15 taken 45 times.
void convert_yv16_chroma_to_yv24_sse2<unsigned short>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 14 → 4 taken 10 times.
✓ Branch 14 → 15 taken 5 times.
 70 for (int x = 0; x < mod8_width; x+=8) {
96 40 __m128i src = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(srcp+x));
97 if constexpr(sizeof(pixel_t) == 1)
98 10 src = _mm_unpacklo_epi8(src, src);
99 else
100 10 src = _mm_unpacklo_epi16(src, src);
101
102 20 _mm_store_si128(reinterpret_cast<__m128i*>(dstp+x*2), src);
103 }
104
105
2/4
void convert_yv16_chroma_to_yv24_sse2<unsigned char>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 15 → 16 taken 45 times.
✗ Branch 15 → 26 not taken.
void convert_yv16_chroma_to_yv24_sse2<unsigned short>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 15 → 16 taken 5 times.
✗ Branch 15 → 26 not taken.
 50 if (mod8_width != src_width) {
106 100 __m128i src = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(srcp+src_width - 8));
107 if constexpr(sizeof(pixel_t)==1)
108 45 src = _mm_unpacklo_epi8(src, src);
109 else
110 5 src = _mm_unpacklo_epi16(src, src);
111
112 50 _mm_storeu_si128(reinterpret_cast<__m128i*>(dstp + (src_width * 2) - 16), src);
113 }
114
115 50 dstp += dst_pitch;
116 50 srcp += src_pitch;
117 }
118 10 }
119
120
121 /***** YV24 -> YV12 chroma downsamplers ******/
122
123 // YV24->YV12 float types: simple 2x2 averaging
124 static AVS_FORCEINLINE __m128 convert_yv24_chroma_block_to_yv12_float_sse2(const __m128 &src_line0_p0, const __m128 &src_line1_p0, const __m128 &src_line0_p1, const __m128 &src_line1_p1, const __m128 &onefourth) {
125 46 __m128 avg1f = _mm_add_ps(src_line0_p0, src_line1_p0);
126 46 __m128 avg2f = _mm_add_ps(src_line0_p1, src_line1_p1);
127 92 avg1f = _mm_add_ps(avg1f, _mm_castsi128_ps(_mm_srli_epi64(_mm_castps_si128(avg1f), 32)));
128 69 avg2f = _mm_add_ps(avg2f, _mm_castsi128_ps(_mm_srli_epi64(_mm_castps_si128(avg2f), 32)));
129 46 return _mm_mul_ps(_mm_shuffle_ps(avg1f, avg2f, _MM_SHUFFLE(2, 0, 2, 0)), onefourth);
130 }
131
132 2 void convert_yv24_chroma_to_yv12_float_sse2(BYTE *dstp, const BYTE *srcp, int dst_pitch, int src_pitch, int dst_width, const int dst_height) {
133 2 const __m128 onefourth = _mm_set1_ps(0.25f);
134
135
2/2
✓ Branch 45 → 5 taken 11 times.
✓ Branch 45 → 46 taken 2 times.
 13 for (int y = 0; y < dst_height; ++y) {
136 11 int x = 0;
137
2/2
✓ Branch 40 → 6 taken 23 times.
✓ Branch 40 → 41 taken 11 times.
 34 for (; x + 16 <= dst_width; x += 16) {
138 23 __m128 p0 = _mm_loadu_ps(reinterpret_cast<const float*>(srcp + x * 2));
139 23 __m128 p1 = _mm_loadu_ps(reinterpret_cast<const float*>(srcp + x * 2 + 16));
140 23 __m128 q0 = _mm_loadu_ps(reinterpret_cast<const float*>(srcp + x * 2 + src_pitch));
141 46 __m128 q1 = _mm_loadu_ps(reinterpret_cast<const float*>(srcp + x * 2 + src_pitch + 16));
142 23 _mm_storeu_ps(reinterpret_cast<float*>(dstp + x),
143 convert_yv24_chroma_block_to_yv12_float_sse2(p0, q0, p1, q1, onefourth));
144 }
145 // scalar tail (0-3 remaining float pixels)
146 11 const float *s0 = reinterpret_cast<const float*>(srcp);
147 11 const float *s1 = reinterpret_cast<const float*>(srcp + src_pitch);
148 11 float *d = reinterpret_cast<float*>(dstp);
149
2/2
✓ Branch 43 → 42 taken 23 times.
✓ Branch 43 → 44 taken 11 times.
 34 for (int px = x / 4; px < dst_width / 4; ++px)
150 23 d[px] = (s0[px * 2] + s0[px * 2 + 1] + s1[px * 2] + s1[px * 2 + 1]) * 0.25f;
151
152 11 dstp += dst_pitch;
153 11 srcp += src_pitch * 2;
154 }
155 2 }
156
157 // YV24->YV12 uint8_t: exact (a+b+c+d+2)>>2 using SSSE3 pmaddubsw.
158 #if defined(GCC) || defined(CLANG)
159 __attribute__((__target__("ssse3")))
160 #endif
161 2 void convert_yv24_chroma_to_yv12_u8_ssse3(BYTE *dstp, const BYTE *srcp, int dst_pitch, int src_pitch, int dst_width, const int dst_height)
162 {
163 2 const __m128i ones = _mm_set1_epi8(1);
164 2 const __m128i two = _mm_set1_epi16(2);
165
166
2/2
✓ Branch 49 → 11 taken 8 times.
✓ Branch 49 → 50 taken 2 times.
 10 for (int y = 0; y < dst_height; ++y) {
167 8 int x = 0;
168
2/2
✓ Branch 44 → 12 taken 18 times.
✓ Branch 44 → 45 taken 8 times.
 26 for (; x + 16 <= dst_width; x += 16) {
169 18 __m128i row0_lo = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + x * 2));
170 18 __m128i row0_hi = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + x * 2 + 16));
171 18 __m128i row1_lo = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + x * 2 + src_pitch));
172 36 __m128i row1_hi = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + x * 2 + src_pitch + 16));
173
174 18 __m128i hsum0_lo = _mm_maddubs_epi16(row0_lo, ones);
175 18 __m128i hsum0_hi = _mm_maddubs_epi16(row0_hi, ones);
176 18 __m128i hsum1_lo = _mm_maddubs_epi16(row1_lo, ones);
177 18 __m128i hsum1_hi = _mm_maddubs_epi16(row1_hi, ones);
178
179 54 __m128i sum_lo = _mm_srli_epi16(_mm_add_epi16(_mm_add_epi16(hsum0_lo, hsum1_lo), two), 2);
180 54 __m128i sum_hi = _mm_srli_epi16(_mm_add_epi16(_mm_add_epi16(hsum0_hi, hsum1_hi), two), 2);
181
182 18 _mm_storeu_si128(reinterpret_cast<__m128i*>(dstp + x), _mm_packus_epi16(sum_lo, sum_hi));
183 }
184 // scalar tail (0-15 remaining u8 pixels)
185
2/2
✓ Branch 47 → 46 taken 34 times.
✓ Branch 47 → 48 taken 8 times.
 42 for (; x < dst_width; ++x)
186 34 dstp[x] = (srcp[x*2] + srcp[x*2+1] + srcp[x*2+src_pitch] + srcp[x*2+src_pitch+1] + 2) >> 2;
187
188 8 dstp += dst_pitch;
189 8 srcp += src_pitch * 2;
190 }
191 2 }
192
193 // YV24->YV12 uint16_t: exact (a+b+c+d+2)>>2.
194 // lessthan16bit=true: values <= 16383; madd_epi16 signed interpretation is correct.
195 // lessthan16bit=false: XOR 0x8000 pivot before madd maps u16->i16 [-32768,32767];
196 // sum becomes (v0+v1+v2+v3 - 4*32768 + 2); srai gives result-32768; XOR 0x8000 restores u16.
197 template<bool lessthan16bit>
198 3 static void convert_yv24_chroma_to_yv12_u16_sse2(BYTE *dstp, const BYTE *srcp, int dst_pitch, int src_pitch, int dst_width, const int dst_height)
199 {
200 3 const __m128i ones = _mm_set1_epi16(1);
201 3 const __m128i two = _mm_set1_epi32(2);
202 3 const __m128i xor_sign = _mm_set1_epi16((short)0x8000);
203
204
4/4
void convert_yv24_chroma_to_yv12_u16_sse2<false>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 63 → 15 taken 8 times.
✓ Branch 63 → 64 taken 2 times.
void convert_yv24_chroma_to_yv12_u16_sse2<true>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 53 → 15 taken 5 times.
✓ Branch 53 → 54 taken 1 time.
 16 for (int y = 0; y < dst_height; ++y) {
205 13 int x = 0;
206
4/4
void convert_yv24_chroma_to_yv12_u16_sse2<false>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 58 → 16 taken 21 times.
✓ Branch 58 → 59 taken 8 times.
void convert_yv24_chroma_to_yv12_u16_sse2<true>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 48 → 16 taken 15 times.
✓ Branch 48 → 49 taken 5 times.
 49 for (; x + 16 <= dst_width; x += 16) {
207 36 __m128i row0_lo = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + x * 2));
208 36 __m128i row0_hi = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + x * 2 + 16));
209 36 __m128i row1_lo = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + x * 2 + src_pitch));
210 72 __m128i row1_hi = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + x * 2 + src_pitch + 16));
211
212 if constexpr (!lessthan16bit) {
213 21 row0_lo = _mm_xor_si128(row0_lo, xor_sign);
214 21 row0_hi = _mm_xor_si128(row0_hi, xor_sign);
215 21 row1_lo = _mm_xor_si128(row1_lo, xor_sign);
216 21 row1_hi = _mm_xor_si128(row1_hi, xor_sign);
217 }
218
219 36 __m128i hsum0_lo = _mm_madd_epi16(row0_lo, ones);
220 36 __m128i hsum0_hi = _mm_madd_epi16(row0_hi, ones);
221 36 __m128i hsum1_lo = _mm_madd_epi16(row1_lo, ones);
222 36 __m128i hsum1_hi = _mm_madd_epi16(row1_hi, ones);
223
224 108 __m128i sum_lo = _mm_srai_epi32(_mm_add_epi32(_mm_add_epi32(hsum0_lo, hsum1_lo), two), 2);
225 108 __m128i sum_hi = _mm_srai_epi32(_mm_add_epi32(_mm_add_epi32(hsum0_hi, hsum1_hi), two), 2);
226
227 36 __m128i result = _mm_packs_epi32(sum_lo, sum_hi);
228 if constexpr (!lessthan16bit)
229 21 result = _mm_xor_si128(result, xor_sign);
230 36 _mm_storeu_si128(reinterpret_cast<__m128i*>(dstp + x), result);
231 }
232 // scalar tail (0-7 remaining u16 pixels)
233 13 const uint16_t *s0 = reinterpret_cast<const uint16_t*>(srcp);
234 13 const uint16_t *s1 = reinterpret_cast<const uint16_t*>(srcp + src_pitch);
235 13 uint16_t *d = reinterpret_cast<uint16_t*>(dstp);
236
4/4
void convert_yv24_chroma_to_yv12_u16_sse2<false>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 61 → 60 taken 24 times.
✓ Branch 61 → 62 taken 8 times.
void convert_yv24_chroma_to_yv12_u16_sse2<true>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 51 → 50 taken 15 times.
✓ Branch 51 → 52 taken 5 times.
 52 for (int px = x / 2; px < dst_width / 2; ++px)
237 39 d[px] = (uint16_t)((s0[px*2] + s0[px*2+1] + s1[px*2] + s1[px*2+1] + 2) >> 2);
238
239 13 dstp += dst_pitch;
240 13 srcp += src_pitch * 2;
241 }
242 3 }
243
244 #if defined(GCC) || defined(CLANG)
245 __attribute__((__target__("sse4.1")))
246 #endif
247 2 void convert_yv24_chroma_to_yv12_u16_sse41(BYTE *dstp, const BYTE *srcp, int dst_pitch, int src_pitch, int dst_width, const int dst_height)
248 {
249 // XOR 0x8000 pivot: u16→i16 before madd_epi16 (signed); srai+packs+XOR restores u16
250 2 const __m128i xor_sign = _mm_set1_epi16((short)0x8000);
251 2 const __m128i ones = _mm_set1_epi16(1);
252 2 const __m128i two = _mm_set1_epi32(2);
253
254
2/2
✓ Branch 63 → 15 taken 8 times.
✓ Branch 63 → 64 taken 2 times.
 10 for (int y = 0; y < dst_height; ++y) {
255 8 int x = 0;
256
2/2
✓ Branch 58 → 16 taken 21 times.
✓ Branch 58 → 59 taken 8 times.
 29 for (; x + 16 <= dst_width; x += 16) {
257 42 __m128i row0_lo = _mm_xor_si128(_mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + x * 2)), xor_sign);
258 42 __m128i row0_hi = _mm_xor_si128(_mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + x * 2 + 16)), xor_sign);
259 42 __m128i row1_lo = _mm_xor_si128(_mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + x * 2 + src_pitch)), xor_sign);
260 63 __m128i row1_hi = _mm_xor_si128(_mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + x * 2 + src_pitch + 16)), xor_sign);
261
262 21 __m128i hsum0_lo = _mm_madd_epi16(row0_lo, ones);
263 21 __m128i hsum0_hi = _mm_madd_epi16(row0_hi, ones);
264 21 __m128i hsum1_lo = _mm_madd_epi16(row1_lo, ones);
265 21 __m128i hsum1_hi = _mm_madd_epi16(row1_hi, ones);
266
267 63 __m128i sum_lo = _mm_srai_epi32(_mm_add_epi32(_mm_add_epi32(hsum0_lo, hsum1_lo), two), 2);
268 63 __m128i sum_hi = _mm_srai_epi32(_mm_add_epi32(_mm_add_epi32(hsum0_hi, hsum1_hi), two), 2);
269
270 42 _mm_storeu_si128(reinterpret_cast<__m128i*>(dstp + x),
271 _mm_xor_si128(_mm_packs_epi32(sum_lo, sum_hi), xor_sign));
272 }
273 // scalar tail (0-7 remaining u16 pixels)
274 8 const uint16_t *s0 = reinterpret_cast<const uint16_t*>(srcp);
275 8 const uint16_t *s1 = reinterpret_cast<const uint16_t*>(srcp + src_pitch);
276 8 uint16_t *d = reinterpret_cast<uint16_t*>(dstp);
277
2/2
✓ Branch 61 → 60 taken 24 times.
✓ Branch 61 → 62 taken 8 times.
 32 for (int px = x / 2; px < dst_width / 2; ++px)
278 24 d[px] = (uint16_t)((s0[px*2] + s0[px*2+1] + s1[px*2] + s1[px*2+1] + 2) >> 2);
279
280 8 dstp += dst_pitch;
281 8 srcp += src_pitch * 2;
282 }
283 2 }
284
285 // YV24->YV12 uint16_t lessthan16bit fast path: pure u16 arithmetic via hadd_epi16 (SSSE3).
286 #if defined(GCC) || defined(CLANG)
287 __attribute__((__target__("ssse3")))
288 #endif
289 1 void convert_yv24_chroma_to_yv12_u16_lessthan16bit_ssse3(BYTE *dstp, const BYTE *srcp, int dst_pitch, int src_pitch, int dst_width, const int dst_height)
290 {
291 1 const __m128i two = _mm_set1_epi16(2);
292
293
2/2
✓ Branch 33 → 7 taken 5 times.
✓ Branch 33 → 34 taken 1 time.
 6 for (int y = 0; y < dst_height; ++y) {
294 5 int x = 0;
295
2/2
✓ Branch 28 → 8 taken 15 times.
✓ Branch 28 → 29 taken 5 times.
 20 for (; x + 16 <= dst_width; x += 16) {
296 15 __m128i row0_lo = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + x * 2));
297 15 __m128i row0_hi = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + x * 2 + 16));
298 15 __m128i row1_lo = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + x * 2 + src_pitch));
299 30 __m128i row1_hi = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + x * 2 + src_pitch + 16));
300
301 15 __m128i vert_lo = _mm_add_epi16(row0_lo, row1_lo);
302 15 __m128i vert_hi = _mm_add_epi16(row0_hi, row1_hi);
303
304 45 _mm_storeu_si128(reinterpret_cast<__m128i*>(dstp + x),
305 _mm_srli_epi16(_mm_add_epi16(_mm_hadd_epi16(vert_lo, vert_hi), two), 2));
306 }
307 // scalar tail (0-7 remaining u16 pixels)
308 5 const uint16_t *s0 = reinterpret_cast<const uint16_t*>(srcp);
309 5 const uint16_t *s1 = reinterpret_cast<const uint16_t*>(srcp + src_pitch);
310 5 uint16_t *d = reinterpret_cast<uint16_t*>(dstp);
311
2/2
✓ Branch 31 → 30 taken 15 times.
✓ Branch 31 → 32 taken 5 times.
 20 for (int px = x / 2; px < dst_width / 2; ++px)
312 15 d[px] = (uint16_t)((s0[px*2] + s0[px*2+1] + s1[px*2] + s1[px*2+1] + 2) >> 2);
313
314 5 dstp += dst_pitch;
315 5 srcp += src_pitch * 2;
316 }
317 1 }
318
319
320 /***** YV24 -> YV16 chroma downsamplers ******/
321
322 static AVS_FORCEINLINE __m128 convert_yv24_chroma_block_to_yv16_float_sse2(const __m128 &src_line0_p0, const __m128 &src_line0_p1, const __m128 &half) {
323 38 __m128 avg1 = src_line0_p0;
324 38 __m128 avg2 = src_line0_p1;
325 152 avg1 = _mm_add_ps(avg1, _mm_castsi128_ps(_mm_srli_epi64(_mm_castps_si128(avg1), 32)));
326 114 avg2 = _mm_add_ps(avg2, _mm_castsi128_ps(_mm_srli_epi64(_mm_castps_si128(avg2), 32)));
327 76 return _mm_mul_ps(_mm_shuffle_ps(avg1, avg2, _MM_SHUFFLE(2, 0, 2, 0)), half);
328 }
329
330 // Requires dst_width >= 16 and 16-byte aligned src/dst (checked by caller)
331 2 void convert_yv24_chroma_to_yv16_float_sse2(BYTE *dstp, const BYTE *srcp, int dst_pitch, int src_pitch, int dst_width, const int dst_height) {
332 2 int mod16_width = dst_width / 16 * 16;
333 2 const __m128 half = _mm_set1_ps(0.5f);
334
335
2/2
✓ Branch 61 → 5 taken 12 times.
✓ Branch 61 → 62 taken 2 times.
 14 for (int y = 0; y < dst_height; ++y) {
336
2/2
✓ Branch 32 → 6 taken 26 times.
✓ Branch 32 → 33 taken 12 times.
 38 for (int x = 0; x < mod16_width; x += 16) {
337 26 __m128 src_line0_p0 = _mm_load_ps(reinterpret_cast<const float*>(srcp + x * 2));
338 52 __m128 src_line0_p1 = _mm_load_ps(reinterpret_cast<const float*>(srcp + x * 2 + 16));
339
340 26 __m128 avg = convert_yv24_chroma_block_to_yv16_float_sse2(src_line0_p0, src_line0_p1, half);
341
342 26 _mm_store_ps(reinterpret_cast<float*>(dstp + x), avg);
343 }
344
345
1/2
✓ Branch 33 → 34 taken 12 times.
✗ Branch 33 → 60 not taken.
 12 if (mod16_width != dst_width) {
346 12 __m128 src_line0_p0 = _mm_loadu_ps(reinterpret_cast<const float*>(srcp + dst_width * 2 - 32));
347 24 __m128 src_line0_p1 = _mm_loadu_ps(reinterpret_cast<const float*>(srcp + dst_width * 2 - 16));
348
349 12 __m128 avg = convert_yv24_chroma_block_to_yv16_float_sse2(src_line0_p0, src_line0_p1, half);
350
351 12 _mm_storeu_ps(reinterpret_cast<float*>(dstp + dst_width - 16), avg);
352 }
353
354 12 dstp += dst_pitch;
355 12 srcp += src_pitch;
356 }
357 2 }
358
359 // uint8_t, uint16_t
360 template<typename pixel_t>
361 static AVS_FORCEINLINE __m128i convert_yv24_chroma_block_to_yv16_sse2(const __m128i &src_line0_p0, const __m128i &src_line0_p1, const __m128i &mask) {
362 __m128i avg1, avg2;
363
364 if constexpr(sizeof(pixel_t) == 1) {
365 84 __m128i avg1_sh = _mm_srli_epi16(src_line0_p0, 8);
366 42 __m128i avg2_sh = _mm_srli_epi16(src_line0_p1, 8);
367
368 42 avg1 = _mm_avg_epu8(src_line0_p0, avg1_sh);
369 42 avg2 = _mm_avg_epu8(src_line0_p1, avg2_sh);
370 }
371 else {
372 56 __m128i avg1_sh = _mm_srli_epi32(src_line0_p0, 16);
373 28 __m128i avg2_sh = _mm_srli_epi32(src_line0_p1, 16);
374
375 28 avg1 = _mm_avg_epu16(src_line0_p0, avg1_sh);
376 28 avg2 = _mm_avg_epu16(src_line0_p1, avg2_sh);
377 }
378
379 70 avg1 = _mm_and_si128(avg1, mask);
380 140 avg2 = _mm_and_si128(avg2, mask);
381
382 __m128i packed;
383 if constexpr(sizeof(pixel_t) == 1)
384 42 packed = _mm_packus_epi16(avg1, avg2);
385 else
386 28 packed = _MM_PACKUS_EPI32(avg1, avg2); // SSE4.1 simul for SSE2
387 70 return packed;
388 }
389
390 template<typename pixel_t>
391 #if defined(GCC) || defined(CLANG)
392 __attribute__((__target__("sse4.1")))
393 #endif
394 static AVS_FORCEINLINE __m128i convert_yv24_chroma_block_to_yv16_sse41(const __m128i &src_line0_p0, const __m128i &src_line0_p1, const __m128i &mask)
395 {
396 __m128i avg1, avg2;
397
398 if constexpr (sizeof(pixel_t) == 1) {
399 44 __m128i avg1_sh = _mm_srli_epi16(src_line0_p0, 8);
400 22 __m128i avg2_sh = _mm_srli_epi16(src_line0_p1, 8);
401
402 22 avg1 = _mm_avg_epu8(src_line0_p0, avg1_sh);
403 22 avg2 = _mm_avg_epu8(src_line0_p1, avg2_sh);
404 }
405 else {
406 56 __m128i avg1_sh = _mm_srli_epi32(src_line0_p0, 16);
407 28 __m128i avg2_sh = _mm_srli_epi32(src_line0_p1, 16);
408
409 28 avg1 = _mm_avg_epu16(src_line0_p0, avg1_sh);
410 28 avg2 = _mm_avg_epu16(src_line0_p1, avg2_sh);
411 }
412
413 50 avg1 = _mm_and_si128(avg1, mask);
414 100 avg2 = _mm_and_si128(avg2, mask);
415
416 __m128i packed;
417 if constexpr (sizeof(pixel_t) == 1)
418 22 packed = _mm_packus_epi16(avg1, avg2);
419 else
420 28 packed = _mm_packus_epi32(avg1, avg2); // SSE4.1
421 50 return packed;
422 }
423
424 // Requires 16-byte aligned src/dst (checked by caller); dst_width in bytes
425 template<typename pixel_t>
426 8 static void convert_yv24_chroma_to_yv16_sse2(BYTE *dstp, const BYTE *srcp, int dst_pitch, int src_pitch, int dst_width, const int dst_height) {
427 8 int mod16_width = dst_width / 16 * 16;
428
429 __m128i mask;
430 if constexpr(sizeof(pixel_t) == 1)
431 6 mask = _mm_set1_epi16(0x00FF);
432 else
433 2 mask = _mm_set1_epi32(0x0000FFFF);
434
435
4/4
void convert_yv24_chroma_to_yv16_sse2<unsigned char>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 55 → 7 taken 31 times.
✓ Branch 55 → 56 taken 6 times.
void convert_yv24_chroma_to_yv16_sse2<unsigned short>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 87 → 7 taken 11 times.
✓ Branch 87 → 88 taken 2 times.
 50 for (int y = 0; y < dst_height; ++y) {
436
4/4
void convert_yv24_chroma_to_yv16_sse2<unsigned char>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 30 → 8 taken 11 times.
✓ Branch 30 → 31 taken 31 times.
void convert_yv24_chroma_to_yv16_sse2<unsigned short>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 46 → 8 taken 17 times.
✓ Branch 46 → 47 taken 11 times.
 70 for (int x = 0; x < mod16_width; x+=16) {
437 28 __m128i src_line0_p0 = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp+x*2));
438 56 __m128i src_line0_p1 = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp+x*2+16));
439
440 28 __m128i avg = convert_yv24_chroma_block_to_yv16_sse2<pixel_t>(src_line0_p0, src_line0_p1, mask);
441
442 28 _mm_store_si128(reinterpret_cast<__m128i*>(dstp+x), avg);
443 }
444
445
2/4
void convert_yv24_chroma_to_yv16_sse2<unsigned char>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 31 → 32 taken 31 times.
✗ Branch 31 → 54 not taken.
void convert_yv24_chroma_to_yv16_sse2<unsigned short>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 47 → 48 taken 11 times.
✗ Branch 47 → 86 not taken.
 42 if (mod16_width != dst_width) {
446 42 __m128i src_line0_p0 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp+dst_width*2-32));
447 84 __m128i src_line0_p1 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp+dst_width*2-16));
448
449 42 __m128i avg = convert_yv24_chroma_block_to_yv16_sse2<pixel_t>(src_line0_p0, src_line0_p1, mask);
450
451 42 _mm_storeu_si128(reinterpret_cast<__m128i*>(dstp+dst_width-16), avg);
452 }
453
454 42 dstp += dst_pitch;
455 42 srcp += src_pitch;
456 }
457 8 }
458
459 template<typename pixel_t>
460 #if defined(GCC) || defined(CLANG)
461 __attribute__((__target__("sse4.1")))
462 #endif
463 4 static void convert_yv24_chroma_to_yv16_sse41(BYTE *dstp, const BYTE *srcp, int dst_pitch, int src_pitch, int dst_width, const int dst_height)
464 {
465 4 int mod16_width = dst_width / 16 * 16;
466
467 __m128i mask;
468 if constexpr (sizeof(pixel_t) == 1)
469 2 mask = _mm_set1_epi16(0x00FF);
470 else
471 2 mask = _mm_set1_epi32(0x0000FFFF);
472
473
4/4
void convert_yv24_chroma_to_yv16_sse41<unsigned char>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 55 → 7 taken 11 times.
✓ Branch 55 → 56 taken 2 times.
void convert_yv24_chroma_to_yv16_sse41<unsigned short>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 55 → 7 taken 11 times.
✓ Branch 55 → 56 taken 2 times.
 26 for (int y = 0; y < dst_height; ++y) {
474
4/4
void convert_yv24_chroma_to_yv16_sse41<unsigned char>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 30 → 8 taken 11 times.
✓ Branch 30 → 31 taken 11 times.
void convert_yv24_chroma_to_yv16_sse41<unsigned short>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 30 → 8 taken 17 times.
✓ Branch 30 → 31 taken 11 times.
 50 for (int x = 0; x < mod16_width; x += 16) {
475 28 __m128i src_line0_p0 = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp + x * 2));
476 56 __m128i src_line0_p1 = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp + x * 2 + 16));
477
478 28 __m128i avg = convert_yv24_chroma_block_to_yv16_sse41<pixel_t>(src_line0_p0, src_line0_p1, mask);
479
480 28 _mm_store_si128(reinterpret_cast<__m128i*>(dstp + x), avg);
481 }
482
483
2/4
void convert_yv24_chroma_to_yv16_sse41<unsigned char>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 31 → 32 taken 11 times.
✗ Branch 31 → 54 not taken.
void convert_yv24_chroma_to_yv16_sse41<unsigned short>(unsigned char*, unsigned char const*, int, int, int, int):
✓ Branch 31 → 32 taken 11 times.
✗ Branch 31 → 54 not taken.
 22 if (mod16_width != dst_width) {
484 22 __m128i src_line0_p0 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + dst_width * 2 - 32));
485 44 __m128i src_line0_p1 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + dst_width * 2 - 16));
486
487 22 __m128i avg = convert_yv24_chroma_block_to_yv16_sse41<pixel_t>(src_line0_p0, src_line0_p1, mask);
488
489 22 _mm_storeu_si128(reinterpret_cast<__m128i*>(dstp + dst_width - 16), avg);
490 }
491
492 22 dstp += dst_pitch;
493 22 srcp += src_pitch;
494 }
495 4 }
496
497
498 /***** Typed non-template wrapper functions (called from 444convert.cpp dispatch) *****/
499
500 17 void conv_yv12_to_yv24_chroma_u8_sse2(BYTE *dstp, const BYTE *srcp, int dst_pitch, int src_pitch, int src_width, int src_height) {
501 17 convert_yv12_chroma_to_yv24_sse2<uint8_t>(dstp, srcp, dst_pitch, src_pitch, src_width, src_height);
502 17 }
503 1 void conv_yv12_to_yv24_chroma_u16_sse2(BYTE *dstp, const BYTE *srcp, int dst_pitch, int src_pitch, int src_width, int src_height) {
504 1 convert_yv12_chroma_to_yv24_sse2<uint16_t>(dstp, srcp, dst_pitch, src_pitch, src_width, src_height);
505 1 }
506
507 9 void conv_yv16_to_yv24_chroma_u8_sse2(BYTE *dstp, const BYTE *srcp, int dst_pitch, int src_pitch, int src_width, int src_height) {
508 9 convert_yv16_chroma_to_yv24_sse2<uint8_t>(dstp, srcp, dst_pitch, src_pitch, src_width, src_height);
509 9 }
510 1 void conv_yv16_to_yv24_chroma_u16_sse2(BYTE *dstp, const BYTE *srcp, int dst_pitch, int src_pitch, int src_width, int src_height) {
511 1 convert_yv16_chroma_to_yv24_sse2<uint16_t>(dstp, srcp, dst_pitch, src_pitch, src_width, src_height);
512 1 }
513
514 1 void conv_yv24_to_yv12_chroma_u16_lessthan16bit_sse2(BYTE *dstp, const BYTE *srcp, int dst_pitch, int src_pitch, int dst_width, int dst_height) {
515 1 convert_yv24_chroma_to_yv12_u16_sse2<true>(dstp, srcp, dst_pitch, src_pitch, dst_width, dst_height);
516 1 }
517 2 void conv_yv24_to_yv12_chroma_u16_true16bit_sse2(BYTE *dstp, const BYTE *srcp, int dst_pitch, int src_pitch, int dst_width, int dst_height) {
518 2 convert_yv24_chroma_to_yv12_u16_sse2<false>(dstp, srcp, dst_pitch, src_pitch, dst_width, dst_height);
519 2 }
520
521 6 void conv_yv24_to_yv16_chroma_u8_sse2(BYTE *dstp, const BYTE *srcp, int dst_pitch, int src_pitch, int dst_width, int dst_height) {
522 6 convert_yv24_chroma_to_yv16_sse2<uint8_t>(dstp, srcp, dst_pitch, src_pitch, dst_width, dst_height);
523 6 }
524 2 void conv_yv24_to_yv16_chroma_u16_sse2(BYTE *dstp, const BYTE *srcp, int dst_pitch, int src_pitch, int dst_width, int dst_height) {
525 2 convert_yv24_chroma_to_yv16_sse2<uint16_t>(dstp, srcp, dst_pitch, src_pitch, dst_width, dst_height);
526 2 }
527 2 void conv_yv24_to_yv16_chroma_u8_sse41(BYTE *dstp, const BYTE *srcp, int dst_pitch, int src_pitch, int dst_width, int dst_height) {
528 2 convert_yv24_chroma_to_yv16_sse41<uint8_t>(dstp, srcp, dst_pitch, src_pitch, dst_width, dst_height);
529 2 }
530 2 void conv_yv24_to_yv16_chroma_u16_sse41(BYTE *dstp, const BYTE *srcp, int dst_pitch, int src_pitch, int dst_width, int dst_height) {
531 2 convert_yv24_chroma_to_yv16_sse41<uint16_t>(dstp, srcp, dst_pitch, src_pitch, dst_width, dst_height);
532 2 }
533