GCC Code Coverage Report

Directory: avs_core/
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 100.0% 69 / 0 / 69
Functions: 100.0% 3 / 0 / 3
Branches: 100.0% 12 / 0 / 12
filters/intel/greyscale_sse.cpp
Line Branch Exec Source
1 // Avisynth v2.5. Copyright 2002 Ben Rudiak-Gould et al.
2 // http://avisynth.nl
3
4 // This program is free software; you can redistribute it and/or modify
5 // it under the terms of the GNU General Public License as published by
6 // the Free Software Foundation; either version 2 of the License, or
7 // (at your option) any later version.
8 //
9 // This program is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 // GNU General Public License for more details.
13 //
14 // You should have received a copy of the GNU General Public License
15 // along with this program; if not, write to the Free Software
16 // Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA, or visit
17 // http://www.gnu.org/copyleft/gpl.html .
18 //
19 // Linking Avisynth statically or dynamically with other modules is making a
20 // combined work based on Avisynth. Thus, the terms and conditions of the GNU
21 // General Public License cover the whole combination.
22 //
23 // As a special exception, the copyright holders of Avisynth give you
24 // permission to link Avisynth with independent modules that communicate with
25 // Avisynth solely through the interfaces defined in avisynth.h, regardless of the license
26 // terms of these independent modules, and to copy and distribute the
27 // resulting combined work under terms of your choice, provided that
28 // every copy of the combined work is accompanied by a complete copy of
29 // the source code of Avisynth (the version of Avisynth used to produce the
30 // combined work), being distributed under the terms of the GNU General
31 // Public License plus this exception. An independent module is a module
32 // which is not derived from or based on Avisynth, such as 3rd-party filters,
33 // import and export plugins, or graphical user interfaces.
34
35
36
37 #include "greyscale_sse.h"
38 #include "../convert/convert_matrix.h"
39
40 // Intrinsics base header + really required extension headers
41 #if defined(_MSC_VER)
42 #include <intrin.h> // MSVC
43 #else
44 #include <x86intrin.h> // GCC/MinGW/Clang/LLVM
45 #endif
46 #include <smmintrin.h> // SSE4.1
47
48 #include <avs/config.h>
49 #include <avs/types.h>
50 #include "avs/minmax.h"
51
52 3 void greyscale_yuy2_sse2(BYTE *srcp, size_t /*width*/, size_t height, size_t pitch) {
53 3 __m128i luma_mask = _mm_set1_epi16(0x00FF);
54 3 __m128i chroma_value = _mm_set1_epi16((short)0x8000);
55 3 BYTE* end_point = srcp + pitch * height;
56
57
2/2
✓ Branch 19 → 11 taken 63 times.
✓ Branch 19 → 20 taken 3 times.
 66 while(srcp < end_point) {
58 63 __m128i src = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp));
59 63 src = _mm_and_si128(src, luma_mask);
60 63 src = _mm_or_si128(src, chroma_value);
61 _mm_store_si128(reinterpret_cast<__m128i*>(srcp), src);
62
63 63 srcp += 16;
64 }
65 3 }
66
67 4 void greyscale_rgb32_sse2(BYTE *srcp, size_t /*width*/, size_t height, size_t pitch, ConversionMatrix &m) {
68 4 const bool has_offset_rgb = 0 != m.offset_rgb;
69 // greyscale RGB is putting pack the calculated pixels to rgb
70 // Limited range input remains limited range output (-offset_rgb is the same as offset_y)
71
72 8 __m128i matrix = _mm_set_epi16(0, m.y_r, m.y_g, m.y_b, 0, m.y_r, m.y_g, m.y_b);
73 4 __m128i zero = _mm_setzero_si128();
74 // .15 frac bit integer arithmetic
75 4 int round_mask_and_luma_offset_i = (1 << 14) + (m.offset_y << 15);
76 4 __m128i round_mask_and_luma_offset = _mm_set1_epi32(round_mask_and_luma_offset_i); // four pixels at a time
77 4 __m128i alpha_mask = _mm_set1_epi32(0xFF000000);
78
79 4 __m128i offset_rgb = _mm_set_epi16(0, m.offset_rgb, m.offset_rgb, m.offset_rgb, 0, m.offset_rgb, m.offset_rgb, m.offset_rgb);
80
81 4 BYTE* end_point = srcp + pitch * height;
82
83
2/2
✓ Branch 61 → 17 taken 100 times.
✓ Branch 61 → 62 taken 4 times.
 104 while(srcp < end_point) {
84 100 __m128i src = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp));
85 100 __m128i alpha = _mm_and_si128(src, alpha_mask);
86 100 __m128i pixel01 = _mm_unpacklo_epi8(src, zero);
87 100 __m128i pixel23 = _mm_unpackhi_epi8(src, zero);
88
89
2/2
✓ Branch 25 → 26 taken 50 times.
✓ Branch 25 → 31 taken 50 times.
 100 if (has_offset_rgb) {
90 50 pixel01 = _mm_add_epi16(pixel01, offset_rgb);
91 50 pixel23 = _mm_add_epi16(pixel23, offset_rgb);
92 }
93
94 100 pixel01 = _mm_madd_epi16(pixel01, matrix);
95 100 pixel23 = _mm_madd_epi16(pixel23, matrix);
96
97 300 __m128i tmp = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(pixel01), _mm_castsi128_ps(pixel23), _MM_SHUFFLE(3, 1, 3, 1))); // r3*cyr | r2*cyr | r1*cyr | r0*cyr
98 300 __m128i tmp2 = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(pixel01), _mm_castsi128_ps(pixel23), _MM_SHUFFLE(2, 0, 2, 0)));
99
100 100 tmp = _mm_add_epi32(tmp, tmp2);
101 100 tmp = _mm_add_epi32(tmp, round_mask_and_luma_offset);
102 100 tmp = _mm_srli_epi32(tmp, 15); // 0 0 0 p3 | 0 0 0 p2 | 0 0 0 p1 | 0 0 0 p0 the grey scale byte itself
103
104 // make r=g=b
105
106 100 __m128i result = _mm_or_si128(tmp, _mm_slli_si128(tmp, 1)); // 0 0 p3 p3 | 0 0 p2 p2 | 0 0 p1 p1 | 0 0 p0 p0
107 200 result = _mm_or_si128(result, _mm_slli_si128(tmp, 2)); // 0 p3 p3 p3 | 0 p2 p2 p2 | 0 p1 p1 p1 | 0 p0 p0 p0
108 100 result = _mm_or_si128(alpha, result);
109
110 _mm_store_si128(reinterpret_cast<__m128i*>(srcp), result);
111
112 100 srcp += 16;
113 }
114 4 }
115
116 #if defined(GCC) || defined(CLANG)
117 __attribute__((__target__("sse4.1")))
118 #endif
119 4 void greyscale_rgb64_sse41(BYTE *srcp, size_t /*width*/, size_t height, size_t pitch, ConversionMatrix &m)
120 {
121 4 const bool has_offset_rgb = 0 != m.offset_rgb;
122 // greyscale RGB is putting pack the calculated pixels to rgb
123 // Limited range input remains limited range output (-offset_rgb is the same as offset_y)
124
125 8 __m128i matrix = _mm_set_epi32(0, m.y_r, m.y_g, m.y_b);
126 4 __m128i zero = _mm_setzero_si128();
127 // .15 frac bit integer arithmetic
128 4 int round_mask_and_luma_offset_i = (1 << 14) + (m.offset_y << 15);
129 4 __m128i round_mask_and_luma_offset = _mm_set_epi32(0, round_mask_and_luma_offset_i, round_mask_and_luma_offset_i, round_mask_and_luma_offset_i);
130 4 uint64_t mask64 = 0xFFFF000000000000ull;
131 4 __m128i alpha_mask = _mm_set_epi32((uint32_t)(mask64 >> 32),(uint32_t)mask64,(uint32_t)(mask64 >> 32),(uint32_t)mask64);
132
133 4 __m128i offset_rgb = _mm_set_epi32(0, m.offset_rgb, m.offset_rgb, m.offset_rgb); // signed (e.g. -16) if exists, for addition
134
135 4 BYTE* end_point = srcp + pitch * height;
136
137
2/2
✓ Branch 49 → 13 taken 110 times.
✓ Branch 49 → 50 taken 4 times.
 114 while(srcp < end_point) {
138 110 __m128i src = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp)); // 2x64bit pixels
139
140 110 __m128i srclo = _mm_unpacklo_epi16(src, zero); // pixel1
141
2/2
✓ Branch 17 → 18 taken 55 times.
✓ Branch 17 → 21 taken 55 times.
 110 if(has_offset_rgb)
142 55 srclo = _mm_add_epi32(srclo, offset_rgb);
143 110 __m128i mullo = _mm_mullo_epi32(srclo, matrix); // 0, mul_r1, mul_g1, mul_b1 // sse41
144
145 110 __m128i srchi = _mm_unpackhi_epi16(src, zero); // pixel2
146
2/2
✓ Branch 25 → 26 taken 55 times.
✓ Branch 25 → 29 taken 55 times.
 110 if(has_offset_rgb)
147 55 srchi = _mm_add_epi32(srchi, offset_rgb);
148 110 __m128i mulhi = _mm_mullo_epi32(srchi, matrix); // 0, mul_r2, mul_g2, mul_b2 // sse41
149
150 110 __m128i alpha = _mm_and_si128(src, alpha_mask); // put back later
151
152 // ssse3
153 110 __m128i result = _mm_hadd_epi32(mullo, mulhi); // 0+mul_r1 | mul_g1+mul_b1 | 0+mul_r2 | mul_g2+mul_b2
154 110 result = _mm_hadd_epi32(result, zero); // 0+mul_r1+mul_g1+mul_b1 | 0+mul_r2+mul_g2+mul_b2 | 0 | 0
155
156 110 result = _mm_add_epi32(result, round_mask_and_luma_offset);
157 110 result = _mm_srli_epi32(result, 15);
158 // we have the greyscale value of two pixels as int32 0 0 | 0 0 | 0 p1 | 0 p0
159 // we need 0 p1 p1 p1 0 p0 p0 p0
160
161 220 __m128i result1 = _mm_or_si128(_mm_slli_si128(result, 2), result);
162 // 0 0 | 0 0 | p1 p1 | p0 p0
163 110 result = _mm_unpacklo_epi32(result1, result);
164 // 0 p1 | p1 p1 | 0 p0 | p0 p0
165
166 110 result = _mm_or_si128(alpha, result); // put back initial alpha
167
168 _mm_store_si128(reinterpret_cast<__m128i*>(srcp), result);
169
170 110 srcp += 16;
171 }
172 4 }
173
174 #ifdef X86_32
175 void greyscale_yuy2_mmx(BYTE *srcp, size_t width, size_t height, size_t pitch) {
176 bool not_mod8 = false;
177 size_t loop_limit = min((pitch / 8) * 8, ((width*4 + 7) / 8) * 8);
178
179 __m64 luma_mask = _mm_set1_pi16(0x00FF);
180 __m64 chroma_value = _mm_set1_pi16((short)0x8000);
181
182 for (size_t y = 0; y < height; ++y) {
183 for (size_t x = 0; x < loop_limit; x+=8) {
184 __m64 src = *reinterpret_cast<const __m64*>(srcp+x);
185 src = _mm_and_si64(src, luma_mask);
186 src = _mm_or_si64(src, chroma_value);
187 *reinterpret_cast<__m64*>(srcp+x) = src;
188 }
189
190 if (loop_limit < width) {
191 __m64 src = *reinterpret_cast<const __m64*>(srcp+width-8);
192 src = _mm_and_si64(src, luma_mask);
193 src = _mm_or_si64(src, chroma_value);
194 *reinterpret_cast<__m64*>(srcp+width-8) = src;
195 }
196
197 srcp += pitch;
198 }
199 _mm_empty();
200 }
201
202 static AVS_FORCEINLINE __m64 greyscale_rgb32_core_mmx(__m64& src, __m64& alpha_mask, __m64& zero, __m64& matrix, __m64& rgb_offset, __m64& round_mask_and_luma_offset, bool has_offset_rgb) {
203 __m64 alpha = _mm_and_si64(src, alpha_mask);
204 __m64 pixel0 = _mm_unpacklo_pi8(src, zero);
205 __m64 pixel1 = _mm_unpackhi_pi8(src, zero);
206
207 if (has_offset_rgb) {
208 pixel0 = _mm_add_pi16(pixel0, rgb_offset); // single pixel 4x16 bit
209 pixel1 = _mm_add_pi16(pixel1, rgb_offset); // single pixel 4x16 bit
210 }
211
212 pixel0 = _mm_madd_pi16(pixel0, matrix); //a0*0 + r0*cyr | g0*cyg + b0*cyb
213 pixel1 = _mm_madd_pi16(pixel1, matrix); //a1*0 + r1*cyr | g1*cyg + b1*cyb
214
215 __m64 tmp = _mm_unpackhi_pi32(pixel0, pixel1); // r1*cyr | r0*cyr
216 __m64 tmp2 = _mm_unpacklo_pi32(pixel0, pixel1); // g1*cyg + b1*cyb | g0*cyg + b0*cyb
217
218 tmp = _mm_add_pi32(tmp, tmp2); // r1*cyr + g1*cyg + b1*cyb | r0*cyr + g0*cyg + b0*cyb
219 tmp = _mm_add_pi32(tmp, round_mask_and_luma_offset); // r1*cyr + g1*cyg + b1*cyb + 32768 | r0*cyr + g0*cyg + b0*cyb + 32768
220 tmp = _mm_srli_pi32(tmp, 15); // 0 0 0 p2 | 0 0 0 p1
221
222 __m64 shifted = _mm_slli_si64(tmp, 8);
223 tmp = _mm_or_si64(tmp, shifted); // 0 0 p2 p2 | 0 0 p1 p1
224 tmp = _mm_or_si64(tmp, _mm_slli_si64(shifted, 8)); // 0 p2 p2 p2 | 0 p1 p1 p1
225 return _mm_or_si64(tmp, alpha);
226 }
227
228 void greyscale_rgb32_mmx(BYTE *srcp, size_t width, size_t height, size_t pitch, ConversionMatrix &m) {
229 const bool has_offset_rgb = 0 != m.offset_rgb;
230
231 __m64 matrix = _mm_set_pi16(0, m.y_r, m.y_g, m.y_b);
232 __m64 zero = _mm_setzero_si64();
233 // .15 frac bit integer arithmetic
234 __m64 round_mask_and_luma_offset = _mm_set1_pi32((1 << 14) + (m.offset_y << 15)); // two pixels at a time
235 __m64 alpha_mask = _mm_set1_pi32(0xFF000000);
236
237 __m64 offset_rgb = _mm_set_pi16(0, m.offset_rgb, m.offset_rgb, m.offset_rgb); // omit alpha. signed (e.g. -16) if exists, for addition
238
239 size_t loop_limit = min((pitch / 8) * 8, ((width*4 + 7) / 8) * 8);
240
241 for (size_t y = 0; y < height; ++y) {
242 for (size_t x = 0; x < loop_limit; x+=8) {
243 __m64 src = *reinterpret_cast<const __m64*>(srcp+x); //pixels 0 and 1
244 __m64 result = greyscale_rgb32_core_mmx(src, alpha_mask, zero, matrix, offset_rgb, round_mask_and_luma_offset, has_offset_rgb);
245
246 *reinterpret_cast<__m64*>(srcp+x) = result;
247 }
248
249 if (loop_limit < width) {
250 __m64 src = *reinterpret_cast<const __m64*>(srcp+width-8); //pixels 0 and 1
251 __m64 result = greyscale_rgb32_core_mmx(src, alpha_mask, zero, matrix, offset_rgb, round_mask_and_luma_offset, has_offset_rgb);
252
253 *reinterpret_cast<__m64*>(srcp+width-8) = result;
254 }
255
256 srcp += pitch;
257 }
258 _mm_empty();
259 }
260 #endif
261
262
263