filters/intel/focus_sse.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // Avisynth v2.5. Copyright 2002 Ben Rudiak-Gould et al. | ||
| 2 | // http://avisynth.nl | ||
| 3 | |||
| 4 | // This program is free software; you can redistribute it and/or modify | ||
| 5 | // it under the terms of the GNU General Public License as published by | ||
| 6 | // the Free Software Foundation; either version 2 of the License, or | ||
| 7 | // (at your option) any later version. | ||
| 8 | // | ||
| 9 | // This program is distributed in the hope that it will be useful, | ||
| 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| 12 | // GNU General Public License for more details. | ||
| 13 | // | ||
| 14 | // You should have received a copy of the GNU General Public License | ||
| 15 | // along with this program; if not, write to the Free Software | ||
| 16 | // Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA, or visit | ||
| 17 | // http://www.gnu.org/copyleft/gpl.html . | ||
| 18 | // | ||
| 19 | // Linking Avisynth statically or dynamically with other modules is making a | ||
| 20 | // combined work based on Avisynth. Thus, the terms and conditions of the GNU | ||
| 21 | // General Public License cover the whole combination. | ||
| 22 | // | ||
| 23 | // As a special exception, the copyright holders of Avisynth give you | ||
| 24 | // permission to link Avisynth with independent modules that communicate with | ||
| 25 | // Avisynth solely through the interfaces defined in avisynth.h, regardless of the license | ||
| 26 | // terms of these independent modules, and to copy and distribute the | ||
| 27 | // resulting combined work under terms of your choice, provided that | ||
| 28 | // every copy of the combined work is accompanied by a complete copy of | ||
| 29 | // the source code of Avisynth (the version of Avisynth used to produce the | ||
| 30 | // combined work), being distributed under the terms of the GNU General | ||
| 31 | // Public License plus this exception. An independent module is a module | ||
| 32 | // which is not derived from or based on Avisynth, such as 3rd-party filters, | ||
| 33 | // import and export plugins, or graphical user interfaces. | ||
| 34 | |||
| 35 | #include <avisynth.h> | ||
| 36 | #include "../focus.h" | ||
| 37 | |||
| 38 | // Intrinsics base header + really required extension headers | ||
| 39 | #if defined(_MSC_VER) | ||
| 40 | #include <intrin.h> // MSVC | ||
| 41 | #else | ||
| 42 | #include <x86intrin.h> // GCC/MinGW/Clang/LLVM | ||
| 43 | #endif | ||
| 44 | #include <smmintrin.h> // SSE4.1 | ||
| 45 | |||
| 46 | #include "../core/internal.h" | ||
| 47 | #include <stdint.h> | ||
| 48 | |||
| 49 | /**************************************** | ||
| 50 | *** AdjustFocus helper classes *** | ||
| 51 | *** Originally by Ben R.G. *** | ||
| 52 | *** MMX code by Marc FD *** | ||
| 53 | *** Adaptation and bugfixes sh0dan *** | ||
| 54 | *** Code actually requires ISSE! *** | ||
| 55 | *** Not anymore - pure MMX IanB *** | ||
| 56 | *** Implement boundary proc. IanB *** | ||
| 57 | *** Impl. full 8bit MMX proc. IanB *** | ||
| 58 | ***************************************/ | ||
| 59 | |||
| 60 | // for linker reasons these forceinlined functions appear in C, intel sse2 and avx2 source as well | ||
| 61 | template<typename pixel_t> | ||
| 62 | static AVS_FORCEINLINE void af_horizontal_planar_process_line_c(pixel_t left, BYTE* dstp8, size_t row_size, int center_weight, int outer_weight) { | ||
| 63 | size_t x; | ||
| 64 | 17 | pixel_t* dstp = reinterpret_cast<pixel_t*>(dstp8); | |
| 65 | typedef typename std::conditional < sizeof(pixel_t) == 1, int, int64_t>::type weight_t; // for calling the right ScaledPixelClip() | ||
| 66 | 17 | size_t width = row_size / sizeof(pixel_t); | |
| 67 |
2/2✓ Branch 208 → 204 taken 68 times.
✓ Branch 208 → 209 taken 17 times.
|
85 | for (x = 0; x < width - 1; ++x) { |
| 68 | 68 | pixel_t temp = ScaledPixelClip((weight_t)(dstp[x] * (weight_t)center_weight + (left + dstp[x + 1]) * (weight_t)outer_weight)); | |
| 69 | 68 | left = dstp[x]; | |
| 70 | 68 | dstp[x] = temp; | |
| 71 | } | ||
| 72 | // ScaledPixelClip has 2 overloads: BYTE/uint16_t (int/int64 i) | ||
| 73 | 17 | dstp[x] = ScaledPixelClip((weight_t)(dstp[x] * (weight_t)center_weight + (left + dstp[x]) * (weight_t)outer_weight)); | |
| 74 | 17 | } | |
| 75 | |||
| 76 | static AVS_FORCEINLINE void af_horizontal_planar_process_line_uint16_c(uint16_t left, BYTE* dstp8, size_t row_size, int center_weight, int outer_weight, int bits_per_pixel) { | ||
| 77 | size_t x; | ||
| 78 | typedef uint16_t pixel_t; | ||
| 79 | 34 | pixel_t* dstp = reinterpret_cast<pixel_t*>(dstp8); | |
| 80 | 34 | const int max_pixel_value = (1 << bits_per_pixel) - 1; // clamping on 10-12-14-16 bitdepth | |
| 81 | typedef std::conditional < sizeof(pixel_t) == 1, int, int64_t>::type weight_t; // for calling the right ScaledPixelClip() | ||
| 82 | 34 | size_t width = row_size / sizeof(pixel_t); | |
| 83 |
4/4✓ Branch 208 → 204 taken 34 times.
✓ Branch 208 → 209 taken 17 times.
✓ Branch 334 → 330 taken 34 times.
✓ Branch 334 → 335 taken 17 times.
|
102 | for (x = 0; x < width - 1; ++x) { |
| 84 | 68 | pixel_t temp = (pixel_t)ScaledPixelClipEx((weight_t)(dstp[x] * (weight_t)center_weight + (left + dstp[x + 1]) * (weight_t)outer_weight), max_pixel_value); | |
| 85 | 68 | left = dstp[x]; | |
| 86 | 68 | dstp[x] = temp; | |
| 87 | } | ||
| 88 | // ScaledPixelClip has 2 overloads: BYTE/uint16_t (int/int64 i) | ||
| 89 | 34 | dstp[x] = ScaledPixelClipEx((weight_t)(dstp[x] * (weight_t)center_weight + (left + dstp[x]) * (weight_t)outer_weight), max_pixel_value); | |
| 90 | 34 | } | |
| 91 | |||
| 92 | static AVS_FORCEINLINE void af_horizontal_planar_process_line_float_c(float left, float* dstp, size_t row_size, float center_weight, float outer_weight) { | ||
| 93 | size_t x; | ||
| 94 | 10 | size_t width = row_size / sizeof(float); | |
| 95 |
2/2✓ Branch 90 → 89 taken 20 times.
✓ Branch 90 → 91 taken 10 times.
|
30 | for (x = 0; x < width - 1; ++x) { |
| 96 | 20 | float temp = dstp[x] * center_weight + (left + dstp[x + 1]) * outer_weight; | |
| 97 | 20 | left = dstp[x]; | |
| 98 | 20 | dstp[x] = temp; | |
| 99 | } | ||
| 100 | 10 | dstp[x] = dstp[x] * center_weight + (left + dstp[x]) * outer_weight; | |
| 101 | 10 | } | |
| 102 | |||
| 103 | 2 | void af_vertical_sse2_float(BYTE* line_buf, BYTE* dstp, const int height, const int pitch, const int row_size, const float amount) { | |
| 104 | |||
| 105 | 2 | const float center_weight = amount; | |
| 106 | 2 | const float outer_weight = (1.0f - amount) / 2.0f; | |
| 107 | |||
| 108 | 2 | __m128 center_weight_simd = _mm_set1_ps(center_weight); | |
| 109 | 2 | __m128 outer_weight_simd = _mm_set1_ps(outer_weight); | |
| 110 | |||
| 111 |
2/2✓ Branch 27 → 7 taken 8 times.
✓ Branch 27 → 28 taken 2 times.
|
10 | for (int y = 0; y < height - 1; ++y) { |
| 112 |
2/2✓ Branch 25 → 8 taken 16 times.
✓ Branch 25 → 26 taken 8 times.
|
24 | for (int x = 0; x < row_size; x += 16) { |
| 113 | 16 | __m128 upper = _mm_load_ps(reinterpret_cast<const float*>(line_buf + x)); | |
| 114 | 16 | __m128 center = _mm_load_ps(reinterpret_cast<const float*>(dstp + x)); | |
| 115 | 16 | __m128 lower = _mm_load_ps(reinterpret_cast<const float*>(dstp + pitch + x)); | |
| 116 | 16 | _mm_store_ps(reinterpret_cast<float*>(line_buf + x), center); | |
| 117 | |||
| 118 | 16 | __m128 tmp1 = _mm_mul_ps(center, center_weight_simd); | |
| 119 | 32 | __m128 tmp2 = _mm_mul_ps(_mm_add_ps(upper, lower), outer_weight_simd); | |
| 120 | 16 | __m128 result = _mm_add_ps(tmp1, tmp2); | |
| 121 | |||
| 122 | 16 | _mm_store_ps(reinterpret_cast<float*>(dstp + x), result); | |
| 123 | } | ||
| 124 | 8 | dstp += pitch; | |
| 125 | } | ||
| 126 | |||
| 127 | //last line | ||
| 128 |
2/2✓ Branch 43 → 29 taken 4 times.
✓ Branch 43 → 44 taken 2 times.
|
6 | for (int x = 0; x < row_size; x += 16) { |
| 129 | 4 | __m128 upper = _mm_load_ps(reinterpret_cast<const float*>(line_buf + x)); | |
| 130 | 8 | __m128 center = _mm_load_ps(reinterpret_cast<const float*>(dstp + x)); | |
| 131 | |||
| 132 | 4 | __m128 tmp1 = _mm_mul_ps(center, center_weight_simd); | |
| 133 | 8 | __m128 tmp2 = _mm_mul_ps(_mm_add_ps(upper, center), outer_weight_simd); // last line: center instead of lower | |
| 134 | 4 | __m128 result = _mm_add_ps(tmp1, tmp2); | |
| 135 | |||
| 136 | 4 | _mm_store_ps(reinterpret_cast<float*>(dstp + x), result); | |
| 137 | } | ||
| 138 | 2 | } | |
| 139 | |||
| 140 | |||
| 141 | static AVS_FORCEINLINE __m128i af_blend_sse2(__m128i &upper, __m128i ¢er, __m128i &lower, __m128i ¢er_weight, __m128i &outer_weight, __m128i &round_mask) { | ||
| 142 | 596 | __m128i outer_tmp = _mm_add_epi16(upper, lower); | |
| 143 | 298 | __m128i center_tmp = _mm_mullo_epi16(center, center_weight); | |
| 144 | |||
| 145 | 596 | outer_tmp = _mm_mullo_epi16(outer_tmp, outer_weight); | |
| 146 | |||
| 147 | 298 | __m128i result = _mm_adds_epi16(center_tmp, outer_tmp); | |
| 148 | 298 | result = _mm_adds_epi16(result, center_tmp); | |
| 149 | 596 | result = _mm_adds_epi16(result, round_mask); | |
| 150 | 298 | return _mm_srai_epi16(result, 7); | |
| 151 | } | ||
| 152 | |||
| 153 | static AVS_FORCEINLINE __m128i af_blend_uint16_t_sse2(__m128i &upper, __m128i ¢er, __m128i &lower, __m128i ¢er_weight, __m128i &outer_weight, __m128i &round_mask) { | ||
| 154 | 278 | __m128i outer_tmp = _mm_add_epi32(upper, lower); | |
| 155 | 278 | __m128i center_tmp = _MM_MULLO_EPI32(center, center_weight); // sse2: mullo simulation | |
| 156 | 278 | outer_tmp = _MM_MULLO_EPI32(outer_tmp, outer_weight); | |
| 157 | |||
| 158 | 556 | __m128i result = _mm_add_epi32(center_tmp, outer_tmp); | |
| 159 | 278 | result = _mm_add_epi32(result, center_tmp); | |
| 160 | 556 | result = _mm_add_epi32(result, round_mask); | |
| 161 | 556 | return _mm_srai_epi32(result, 7); | |
| 162 | } | ||
| 163 | |||
| 164 | #if defined(GCC) || defined(CLANG) | ||
| 165 | __attribute__((__target__("sse4.1"))) | ||
| 166 | #endif | ||
| 167 | static AVS_FORCEINLINE __m128i af_blend_uint16_t_sse41(__m128i &upper, __m128i ¢er, __m128i &lower, __m128i ¢er_weight, __m128i &outer_weight, __m128i &round_mask) | ||
| 168 | { | ||
| 169 | 556 | __m128i outer_tmp = _mm_add_epi32(upper, lower); | |
| 170 | 278 | __m128i center_tmp = _mm_mullo_epi32(center, center_weight); | |
| 171 | 556 | outer_tmp = _mm_mullo_epi32(outer_tmp, outer_weight); | |
| 172 | |||
| 173 | 278 | __m128i result = _mm_add_epi32(center_tmp, outer_tmp); | |
| 174 | 278 | result = _mm_add_epi32(result, center_tmp); | |
| 175 | 556 | result = _mm_add_epi32(result, round_mask); | |
| 176 | 278 | return _mm_srai_epi32(result, 7); | |
| 177 | } | ||
| 178 | |||
| 179 | static AVS_FORCEINLINE __m128 af_blend_float_sse2(__m128 &upper, __m128 ¢er, __m128 &lower, __m128 ¢er_weight, __m128 &outer_weight) { | ||
| 180 | 20 | __m128 tmp1 = _mm_mul_ps(center, center_weight); | |
| 181 | 30 | __m128 tmp2 = _mm_mul_ps(_mm_add_ps(upper, lower), outer_weight); | |
| 182 | 10 | return _mm_add_ps(tmp1, tmp2); | |
| 183 | } | ||
| 184 | |||
| 185 | |||
| 186 | static AVS_FORCEINLINE __m128i af_unpack_blend_sse2(__m128i &left, __m128i ¢er, __m128i &right, __m128i ¢er_weight, __m128i &outer_weight, __m128i &round_mask, __m128i &zero) { | ||
| 187 | 122 | __m128i left_lo = _mm_unpacklo_epi8(left, zero); | |
| 188 | 61 | __m128i left_hi = _mm_unpackhi_epi8(left, zero); | |
| 189 | 61 | __m128i center_lo = _mm_unpacklo_epi8(center, zero); | |
| 190 | 61 | __m128i center_hi = _mm_unpackhi_epi8(center, zero); | |
| 191 | 61 | __m128i right_lo = _mm_unpacklo_epi8(right, zero); | |
| 192 | 122 | __m128i right_hi = _mm_unpackhi_epi8(right, zero); | |
| 193 | |||
| 194 | 61 | __m128i result_lo = af_blend_sse2(left_lo, center_lo, right_lo, center_weight, outer_weight, round_mask); | |
| 195 | 61 | __m128i result_hi = af_blend_sse2(left_hi, center_hi, right_hi, center_weight, outer_weight, round_mask); | |
| 196 | |||
| 197 | 122 | return _mm_packus_epi16(result_lo, result_hi); | |
| 198 | } | ||
| 199 | |||
| 200 | static AVS_FORCEINLINE __m128i af_unpack_blend_uint16_t_sse2(__m128i &left, __m128i ¢er, __m128i &right, __m128i ¢er_weight, __m128i &outer_weight, __m128i &round_mask, __m128i &zero) { | ||
| 201 | 142 | __m128i left_lo = _mm_unpacklo_epi16(left, zero); | |
| 202 | 71 | __m128i left_hi = _mm_unpackhi_epi16(left, zero); | |
| 203 | 71 | __m128i center_lo = _mm_unpacklo_epi16(center, zero); | |
| 204 | 71 | __m128i center_hi = _mm_unpackhi_epi16(center, zero); | |
| 205 | 71 | __m128i right_lo = _mm_unpacklo_epi16(right, zero); | |
| 206 | 142 | __m128i right_hi = _mm_unpackhi_epi16(right, zero); | |
| 207 | |||
| 208 | 71 | __m128i result_lo = af_blend_uint16_t_sse2(left_lo, center_lo, right_lo, center_weight, outer_weight, round_mask); | |
| 209 | 71 | __m128i result_hi = af_blend_uint16_t_sse2(left_hi, center_hi, right_hi, center_weight, outer_weight, round_mask); | |
| 210 | 142 | return _MM_PACKUS_EPI32(result_lo, result_hi); // sse4.1 simul | |
| 211 | } | ||
| 212 | |||
| 213 | #if defined(GCC) || defined(CLANG) | ||
| 214 | __attribute__((__target__("sse4.1"))) | ||
| 215 | #endif | ||
| 216 | static AVS_FORCEINLINE __m128i af_unpack_blend_uint16_t_sse41(__m128i &left, __m128i ¢er, __m128i &right, __m128i ¢er_weight, __m128i &outer_weight, __m128i &round_mask, __m128i &zero) | ||
| 217 | { | ||
| 218 | 142 | __m128i left_lo = _mm_unpacklo_epi16(left, zero); | |
| 219 | 71 | __m128i left_hi = _mm_unpackhi_epi16(left, zero); | |
| 220 | 71 | __m128i center_lo = _mm_unpacklo_epi16(center, zero); | |
| 221 | 71 | __m128i center_hi = _mm_unpackhi_epi16(center, zero); | |
| 222 | 71 | __m128i right_lo = _mm_unpacklo_epi16(right, zero); | |
| 223 | 142 | __m128i right_hi = _mm_unpackhi_epi16(right, zero); | |
| 224 | |||
| 225 | 71 | __m128i result_lo = af_blend_uint16_t_sse41(left_lo, center_lo, right_lo, center_weight, outer_weight, round_mask); | |
| 226 | 71 | __m128i result_hi = af_blend_uint16_t_sse41(left_hi, center_hi, right_hi, center_weight, outer_weight, round_mask); | |
| 227 | 142 | return _mm_packus_epi32(result_lo, result_hi); | |
| 228 | } | ||
| 229 | |||
| 230 | |||
| 231 | 3 | void af_vertical_uint16_t_sse2(BYTE* line_buf, BYTE* dstp, int height, int pitch, int row_size, int amount) { | |
| 232 | // amount was: half_amount (32768). Full: 65536 (2**16) | ||
| 233 | // now it becomes 2**(16-9)=2**7 scale | ||
| 234 | 3 | int t = (amount + 256) >> 9; // 16-9 = 7 -> shift in | |
| 235 | 3 | __m128i center_weight = _mm_set1_epi32(t); | |
| 236 | 6 | __m128i outer_weight = _mm_set1_epi32(64 - t); | |
| 237 | 3 | __m128i round_mask = _mm_set1_epi32(0x40); | |
| 238 | 3 | __m128i zero = _mm_setzero_si128(); | |
| 239 | |||
| 240 |
2/2✓ Branch 117 → 17 taken 14 times.
✓ Branch 117 → 118 taken 3 times.
|
17 | for (int y = 0; y < height - 1; ++y) { |
| 241 |
2/2✓ Branch 115 → 18 taken 56 times.
✓ Branch 115 → 116 taken 14 times.
|
70 | for (int x = 0; x < row_size; x += 16) { |
| 242 | 56 | __m128i upper = _mm_load_si128(reinterpret_cast<const __m128i*>(line_buf + x)); | |
| 243 | 56 | __m128i center = _mm_load_si128(reinterpret_cast<const __m128i*>(dstp + x)); | |
| 244 | 56 | __m128i lower = _mm_load_si128(reinterpret_cast<const __m128i*>(dstp + pitch + x)); | |
| 245 | 56 | _mm_store_si128(reinterpret_cast<__m128i*>(line_buf + x), center); | |
| 246 | |||
| 247 | 56 | __m128i upper_lo = _mm_unpacklo_epi16(upper, zero); | |
| 248 | 56 | __m128i upper_hi = _mm_unpackhi_epi16(upper, zero); | |
| 249 | 56 | __m128i center_lo = _mm_unpacklo_epi16(center, zero); | |
| 250 | 56 | __m128i center_hi = _mm_unpackhi_epi16(center, zero); | |
| 251 | 56 | __m128i lower_lo = _mm_unpacklo_epi16(lower, zero); | |
| 252 | 56 | __m128i lower_hi = _mm_unpackhi_epi16(lower, zero); | |
| 253 | |||
| 254 | 56 | __m128i result_lo = af_blend_uint16_t_sse2(upper_lo, center_lo, lower_lo, center_weight, outer_weight, round_mask); | |
| 255 | 56 | __m128i result_hi = af_blend_uint16_t_sse2(upper_hi, center_hi, lower_hi, center_weight, outer_weight, round_mask); | |
| 256 | |||
| 257 | 56 | __m128i result = _MM_PACKUS_EPI32(result_lo, result_hi); // sse4.1 simul | |
| 258 | |||
| 259 | 56 | _mm_store_si128(reinterpret_cast<__m128i*>(dstp + x), result); | |
| 260 | } | ||
| 261 | 14 | dstp += pitch; | |
| 262 | } | ||
| 263 | |||
| 264 | //last line | ||
| 265 |
2/2✓ Branch 209 → 119 taken 12 times.
✓ Branch 209 → 210 taken 3 times.
|
15 | for (int x = 0; x < row_size; x += 16) { |
| 266 | 12 | __m128i upper = _mm_load_si128(reinterpret_cast<const __m128i*>(line_buf + x)); | |
| 267 | 24 | __m128i center = _mm_load_si128(reinterpret_cast<const __m128i*>(dstp + x)); | |
| 268 | |||
| 269 | 12 | __m128i upper_lo = _mm_unpacklo_epi16(upper, zero); | |
| 270 | 12 | __m128i upper_hi = _mm_unpackhi_epi16(upper, zero); | |
| 271 | 12 | __m128i center_lo = _mm_unpacklo_epi16(center, zero); | |
| 272 | 12 | __m128i center_hi = _mm_unpackhi_epi16(center, zero); | |
| 273 | |||
| 274 | 12 | __m128i result_lo = af_blend_uint16_t_sse2(upper_lo, center_lo, center_lo, center_weight, outer_weight, round_mask); | |
| 275 | 12 | __m128i result_hi = af_blend_uint16_t_sse2(upper_hi, center_hi, center_hi, center_weight, outer_weight, round_mask); | |
| 276 | |||
| 277 | 12 | __m128i result = _MM_PACKUS_EPI32(result_lo, result_hi); // sse4.1 simul | |
| 278 | |||
| 279 | 12 | _mm_store_si128(reinterpret_cast<__m128i*>(dstp + x), result); | |
| 280 | } | ||
| 281 | 3 | } | |
| 282 | |||
| 283 | #if defined(GCC) || defined(CLANG) | ||
| 284 | __attribute__((__target__("sse4.1"))) | ||
| 285 | #endif | ||
| 286 | 3 | void af_vertical_uint16_t_sse41(BYTE* line_buf, BYTE* dstp, int height, int pitch, int row_size, int amount) | |
| 287 | { | ||
| 288 | // amount was: half_amount (32768). Full: 65536 (2**16) | ||
| 289 | // now it becomes 2**(16-9)=2**7 scale | ||
| 290 | 3 | int t = (amount + 256) >> 9; // 16-9 = 7 -> shift in | |
| 291 | 3 | __m128i center_weight = _mm_set1_epi32(t); | |
| 292 | 6 | __m128i outer_weight = _mm_set1_epi32(64 - t); | |
| 293 | 3 | __m128i round_mask = _mm_set1_epi32(0x40); | |
| 294 | 3 | __m128i zero = _mm_setzero_si128(); | |
| 295 | |||
| 296 |
2/2✓ Branch 75 → 17 taken 14 times.
✓ Branch 75 → 76 taken 3 times.
|
17 | for (int y = 0; y < height - 1; ++y) { |
| 297 |
2/2✓ Branch 73 → 18 taken 56 times.
✓ Branch 73 → 74 taken 14 times.
|
70 | for (int x = 0; x < row_size; x += 16) { |
| 298 | 56 | __m128i upper = _mm_load_si128(reinterpret_cast<const __m128i*>(line_buf + x)); | |
| 299 | 56 | __m128i center = _mm_load_si128(reinterpret_cast<const __m128i*>(dstp + x)); | |
| 300 | 56 | __m128i lower = _mm_load_si128(reinterpret_cast<const __m128i*>(dstp + pitch + x)); | |
| 301 | 56 | _mm_store_si128(reinterpret_cast<__m128i*>(line_buf + x), center); | |
| 302 | |||
| 303 | 56 | __m128i upper_lo = _mm_unpacklo_epi16(upper, zero); | |
| 304 | 56 | __m128i upper_hi = _mm_unpackhi_epi16(upper, zero); | |
| 305 | 56 | __m128i center_lo = _mm_unpacklo_epi16(center, zero); | |
| 306 | 56 | __m128i center_hi = _mm_unpackhi_epi16(center, zero); | |
| 307 | 56 | __m128i lower_lo = _mm_unpacklo_epi16(lower, zero); | |
| 308 | 56 | __m128i lower_hi = _mm_unpackhi_epi16(lower, zero); | |
| 309 | |||
| 310 | 56 | __m128i result_lo = af_blend_uint16_t_sse41(upper_lo, center_lo, lower_lo, center_weight, outer_weight, round_mask); | |
| 311 | 56 | __m128i result_hi = af_blend_uint16_t_sse41(upper_hi, center_hi, lower_hi, center_weight, outer_weight, round_mask); | |
| 312 | |||
| 313 | 56 | __m128i result = _mm_packus_epi32(result_lo, result_hi); | |
| 314 | |||
| 315 | 56 | _mm_store_si128(reinterpret_cast<__m128i*>(dstp + x), result); | |
| 316 | } | ||
| 317 | 14 | dstp += pitch; | |
| 318 | } | ||
| 319 | |||
| 320 | //last line | ||
| 321 |
2/2✓ Branch 125 → 77 taken 12 times.
✓ Branch 125 → 126 taken 3 times.
|
15 | for (int x = 0; x < row_size; x += 16) { |
| 322 | 12 | __m128i upper = _mm_load_si128(reinterpret_cast<const __m128i*>(line_buf + x)); | |
| 323 | 24 | __m128i center = _mm_load_si128(reinterpret_cast<const __m128i*>(dstp + x)); | |
| 324 | |||
| 325 | 12 | __m128i upper_lo = _mm_unpacklo_epi16(upper, zero); | |
| 326 | 12 | __m128i upper_hi = _mm_unpackhi_epi16(upper, zero); | |
| 327 | 12 | __m128i center_lo = _mm_unpacklo_epi16(center, zero); | |
| 328 | 12 | __m128i center_hi = _mm_unpackhi_epi16(center, zero); | |
| 329 | |||
| 330 | 12 | __m128i result_lo = af_blend_uint16_t_sse41(upper_lo, center_lo, center_lo, center_weight, outer_weight, round_mask); | |
| 331 | 12 | __m128i result_hi = af_blend_uint16_t_sse41(upper_hi, center_hi, center_hi, center_weight, outer_weight, round_mask); | |
| 332 | |||
| 333 | 12 | __m128i result = _mm_packus_epi32(result_lo, result_hi); | |
| 334 | |||
| 335 | 12 | _mm_store_si128(reinterpret_cast<__m128i*>(dstp + x), result); | |
| 336 | } | ||
| 337 | 3 | } | |
| 338 | |||
| 339 | 3 | void af_vertical_sse2(BYTE* line_buf, BYTE* dstp, int height, int pitch, int width, int amount) { | |
| 340 | 3 | short t = (amount + 256) >> 9; | |
| 341 | 3 | __m128i center_weight = _mm_set1_epi16(t); | |
| 342 | 6 | __m128i outer_weight = _mm_set1_epi16(64 - t); | |
| 343 | 3 | __m128i round_mask = _mm_set1_epi16(0x40); | |
| 344 | 3 | __m128i zero = _mm_setzero_si128(); | |
| 345 | |||
| 346 |
2/2✓ Branch 75 → 17 taken 14 times.
✓ Branch 75 → 76 taken 3 times.
|
17 | for (int y = 0; y < height-1; ++y) { |
| 347 |
2/2✓ Branch 73 → 18 taken 56 times.
✓ Branch 73 → 74 taken 14 times.
|
70 | for (int x = 0; x < width; x+= 16) { |
| 348 | 56 | __m128i upper = _mm_load_si128(reinterpret_cast<const __m128i*>(line_buf+x)); | |
| 349 | 56 | __m128i center = _mm_load_si128(reinterpret_cast<const __m128i*>(dstp+x)); | |
| 350 | 56 | __m128i lower = _mm_load_si128(reinterpret_cast<const __m128i*>(dstp+pitch+x)); | |
| 351 | 56 | _mm_store_si128(reinterpret_cast<__m128i*>(line_buf+x), center); | |
| 352 | |||
| 353 | 56 | __m128i upper_lo = _mm_unpacklo_epi8(upper, zero); | |
| 354 | 56 | __m128i upper_hi = _mm_unpackhi_epi8(upper, zero); | |
| 355 | 56 | __m128i center_lo = _mm_unpacklo_epi8(center, zero); | |
| 356 | 56 | __m128i center_hi = _mm_unpackhi_epi8(center, zero); | |
| 357 | 56 | __m128i lower_lo = _mm_unpacklo_epi8(lower, zero); | |
| 358 | 56 | __m128i lower_hi = _mm_unpackhi_epi8(lower, zero); | |
| 359 | |||
| 360 | 56 | __m128i result_lo = af_blend_sse2(upper_lo, center_lo, lower_lo, center_weight, outer_weight, round_mask); | |
| 361 | 56 | __m128i result_hi = af_blend_sse2(upper_hi, center_hi, lower_hi, center_weight, outer_weight, round_mask); | |
| 362 | |||
| 363 | 56 | __m128i result = _mm_packus_epi16(result_lo, result_hi); | |
| 364 | |||
| 365 | 56 | _mm_store_si128(reinterpret_cast<__m128i*>(dstp+x), result); | |
| 366 | } | ||
| 367 | 14 | dstp += pitch; | |
| 368 | } | ||
| 369 | |||
| 370 | //last line | ||
| 371 |
2/2✓ Branch 125 → 77 taken 12 times.
✓ Branch 125 → 126 taken 3 times.
|
15 | for (int x = 0; x < width; x+= 16) { |
| 372 | 12 | __m128i upper = _mm_load_si128(reinterpret_cast<const __m128i*>(line_buf+x)); | |
| 373 | 24 | __m128i center = _mm_load_si128(reinterpret_cast<const __m128i*>(dstp+x)); | |
| 374 | |||
| 375 | 12 | __m128i upper_lo = _mm_unpacklo_epi8(upper, zero); | |
| 376 | 12 | __m128i upper_hi = _mm_unpackhi_epi8(upper, zero); | |
| 377 | 12 | __m128i center_lo = _mm_unpacklo_epi8(center, zero); | |
| 378 | 12 | __m128i center_hi = _mm_unpackhi_epi8(center, zero); | |
| 379 | |||
| 380 | 12 | __m128i result_lo = af_blend_sse2(upper_lo, center_lo, center_lo, center_weight, outer_weight, round_mask); | |
| 381 | 12 | __m128i result_hi = af_blend_sse2(upper_hi, center_hi, center_hi, center_weight, outer_weight, round_mask); | |
| 382 | |||
| 383 | 12 | __m128i result = _mm_packus_epi16(result_lo, result_hi); | |
| 384 | |||
| 385 | 12 | _mm_store_si128(reinterpret_cast<__m128i*>(dstp+x), result); | |
| 386 | } | ||
| 387 | 3 | } | |
| 388 | |||
| 389 | #ifdef X86_32 | ||
| 390 | |||
| 391 | static AVS_FORCEINLINE __m64 af_blend_mmx(__m64 &upper, __m64 ¢er, __m64 &lower, __m64 ¢er_weight, __m64 &outer_weight, __m64 &round_mask) { | ||
| 392 | __m64 outer_tmp = _mm_add_pi16(upper, lower); | ||
| 393 | __m64 center_tmp = _mm_mullo_pi16(center, center_weight); | ||
| 394 | |||
| 395 | outer_tmp = _mm_mullo_pi16(outer_tmp, outer_weight); | ||
| 396 | |||
| 397 | __m64 result = _mm_adds_pi16(center_tmp, outer_tmp); | ||
| 398 | result = _mm_adds_pi16(result, center_tmp); | ||
| 399 | result = _mm_adds_pi16(result, round_mask); | ||
| 400 | return _mm_srai_pi16(result, 7); | ||
| 401 | } | ||
| 402 | |||
| 403 | static AVS_FORCEINLINE __m64 af_unpack_blend_mmx(__m64 &left, __m64 ¢er, __m64 &right, __m64 ¢er_weight, __m64 &outer_weight, __m64 &round_mask, __m64 &zero) { | ||
| 404 | __m64 left_lo = _mm_unpacklo_pi8(left, zero); | ||
| 405 | __m64 left_hi = _mm_unpackhi_pi8(left, zero); | ||
| 406 | __m64 center_lo = _mm_unpacklo_pi8(center, zero); | ||
| 407 | __m64 center_hi = _mm_unpackhi_pi8(center, zero); | ||
| 408 | __m64 right_lo = _mm_unpacklo_pi8(right, zero); | ||
| 409 | __m64 right_hi = _mm_unpackhi_pi8(right, zero); | ||
| 410 | |||
| 411 | __m64 result_lo = af_blend_mmx(left_lo, center_lo, right_lo, center_weight, outer_weight, round_mask); | ||
| 412 | __m64 result_hi = af_blend_mmx(left_hi, center_hi, right_hi, center_weight, outer_weight, round_mask); | ||
| 413 | |||
| 414 | return _mm_packs_pu16(result_lo, result_hi); | ||
| 415 | } | ||
| 416 | |||
| 417 | void af_vertical_mmx(BYTE* line_buf, BYTE* dstp, int height, int pitch, int width, int amount) { | ||
| 418 | short t = (amount + 256) >> 9; | ||
| 419 | __m64 center_weight = _mm_set1_pi16(t); | ||
| 420 | __m64 outer_weight = _mm_set1_pi16(64 - t); | ||
| 421 | __m64 round_mask = _mm_set1_pi16(0x40); | ||
| 422 | __m64 zero = _mm_setzero_si64(); | ||
| 423 | |||
| 424 | for (int y = 0; y < height-1; ++y) { | ||
| 425 | for (int x = 0; x < width; x+= 8) { | ||
| 426 | __m64 upper = *reinterpret_cast<const __m64*>(line_buf+x); | ||
| 427 | __m64 center = *reinterpret_cast<const __m64*>(dstp+x); | ||
| 428 | __m64 lower = *reinterpret_cast<const __m64*>(dstp+pitch+x); | ||
| 429 | *reinterpret_cast<__m64*>(line_buf+x) = center; | ||
| 430 | |||
| 431 | __m64 result = af_unpack_blend_mmx(upper, center, lower, center_weight, outer_weight, round_mask, zero); | ||
| 432 | |||
| 433 | *reinterpret_cast<__m64*>(dstp+x) = result; | ||
| 434 | } | ||
| 435 | dstp += pitch; | ||
| 436 | } | ||
| 437 | |||
| 438 | //last line | ||
| 439 | for (int x = 0; x < width; x+= 8) { | ||
| 440 | __m64 upper = *reinterpret_cast<const __m64*>(line_buf+x); | ||
| 441 | __m64 center = *reinterpret_cast<const __m64*>(dstp+x); | ||
| 442 | |||
| 443 | __m64 upper_lo = _mm_unpacklo_pi8(upper, zero); | ||
| 444 | __m64 upper_hi = _mm_unpackhi_pi8(upper, zero); | ||
| 445 | __m64 center_lo = _mm_unpacklo_pi8(center, zero); | ||
| 446 | __m64 center_hi = _mm_unpackhi_pi8(center, zero); | ||
| 447 | |||
| 448 | __m64 result_lo = af_blend_mmx(upper_lo, center_lo, center_lo, center_weight, outer_weight, round_mask); | ||
| 449 | __m64 result_hi = af_blend_mmx(upper_hi, center_hi, center_hi, center_weight, outer_weight, round_mask); | ||
| 450 | |||
| 451 | __m64 result = _mm_packs_pu16(result_lo, result_hi); | ||
| 452 | |||
| 453 | *reinterpret_cast<__m64*>(dstp+x) = result; | ||
| 454 | } | ||
| 455 | _mm_empty(); | ||
| 456 | } | ||
| 457 | |||
| 458 | #endif | ||
| 459 | |||
| 460 | |||
| 461 | // -------------------------------------- | ||
| 462 | |||
| 463 | |||
| 464 | //implementation is not in-place. Unaligned reads will be slow on older intels but who cares | ||
| 465 | 2 | void af_horizontal_rgb32_sse2(BYTE* dstp, const BYTE* srcp, size_t dst_pitch, size_t src_pitch, size_t height, size_t width, size_t amount) { | |
| 466 | 2 | size_t width_bytes = width * 4; | |
| 467 | 2 | size_t loop_limit = width_bytes - 16; | |
| 468 | //int center_weight_c = int(amount*2); | ||
| 469 | //int outer_weight_c = int(32768-amount); | ||
| 470 | |||
| 471 | 2 | short t = short((amount + 256) >> 9); | |
| 472 | 2 | __m128i center_weight = _mm_set1_epi16(t); | |
| 473 | 4 | __m128i outer_weight = _mm_set1_epi16(64 - t); | |
| 474 | 2 | __m128i round_mask = _mm_set1_epi16(0x40); | |
| 475 | 2 | __m128i zero = _mm_setzero_si128(); | |
| 476 | 2 | __m128i left_mask = _mm_set_epi32(0, 0, 0, 0xFFFFFFFF); | |
| 477 | 2 | __m128i right_mask = _mm_set_epi32(0xFFFFFFFF, 0, 0, 0); | |
| 478 | |||
| 479 | __m128i center, right, left, result; | ||
| 480 | |||
| 481 |
2/2✓ Branch 197 → 21 taken 10 times.
✓ Branch 197 → 198 taken 2 times.
|
12 | for (size_t y = 0; y < height; ++y) { |
| 482 | 10 | center = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp)); | |
| 483 | 10 | right = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + 4)); | |
| 484 | 30 | left = _mm_or_si128(_mm_and_si128(center, left_mask), _mm_slli_si128(center, 4)); | |
| 485 | |||
| 486 | 10 | result = af_unpack_blend_sse2(left, center, right, center_weight, outer_weight, round_mask, zero); | |
| 487 | |||
| 488 | _mm_store_si128(reinterpret_cast< __m128i*>(dstp), result); | ||
| 489 | |||
| 490 |
1/2✗ Branch 137 → 80 not taken.
✓ Branch 137 → 138 taken 10 times.
|
10 | for (size_t x = 16; x < loop_limit; x+=16) { |
| 491 | ✗ | left = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + x - 4)); | |
| 492 | ✗ | center = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp + x)); | |
| 493 | ✗ | right = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + x + 4)); | |
| 494 | |||
| 495 | ✗ | result = af_unpack_blend_sse2(left, center, right, center_weight, outer_weight, round_mask, zero); | |
| 496 | |||
| 497 | ✗ | _mm_store_si128(reinterpret_cast< __m128i*>(dstp+x), result); | |
| 498 | } | ||
| 499 | |||
| 500 | 10 | left = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + loop_limit - 4)); | |
| 501 | 10 | center = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + loop_limit)); | |
| 502 | 30 | right = _mm_or_si128(_mm_and_si128(center, right_mask), _mm_srli_si128(center, 4)); | |
| 503 | |||
| 504 | 10 | result = af_unpack_blend_sse2(left, center, right, center_weight, outer_weight, round_mask, zero); | |
| 505 | |||
| 506 | 10 | _mm_storeu_si128(reinterpret_cast< __m128i*>(dstp + loop_limit), result); | |
| 507 | |||
| 508 | |||
| 509 | 10 | dstp += dst_pitch; | |
| 510 | 10 | srcp += src_pitch; | |
| 511 | } | ||
| 512 | 2 | } | |
| 513 | |||
| 514 | 2 | void af_horizontal_rgb64_sse2(BYTE* dstp, const BYTE* srcp, size_t dst_pitch, size_t src_pitch, size_t height, size_t width, size_t amount) { | |
| 515 | // width is really width | ||
| 516 | 2 | size_t width_bytes = width * 4 * sizeof(uint16_t); | |
| 517 | 2 | size_t loop_limit = width_bytes - 16; | |
| 518 | |||
| 519 | 2 | short t = short((amount + 256) >> 9); | |
| 520 | 2 | __m128i center_weight = _mm_set1_epi32(t); | |
| 521 | 4 | __m128i outer_weight = _mm_set1_epi32(64 - t); | |
| 522 | 2 | __m128i round_mask = _mm_set1_epi32(0x40); | |
| 523 | 2 | __m128i zero = _mm_setzero_si128(); | |
| 524 | 2 | __m128i left_mask = _mm_set_epi32(0, 0, 0xFFFFFFFF, 0xFFFFFFFF); | |
| 525 | 2 | __m128i right_mask = _mm_set_epi32(0xFFFFFFFF, 0xFFFFFFFF, 0, 0); | |
| 526 | |||
| 527 | __m128i center, right, left, result; | ||
| 528 | |||
| 529 |
2/2✓ Branch 323 → 21 taken 10 times.
✓ Branch 323 → 324 taken 2 times.
|
12 | for (size_t y = 0; y < height; ++y) { |
| 530 | 10 | center = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp)); | |
| 531 | 10 | right = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + 4*sizeof(uint16_t))); // move right by one 4*uint16_t pixelblock | |
| 532 | 30 | left = _mm_or_si128(_mm_and_si128(center, left_mask), _mm_slli_si128(center, 8)); | |
| 533 | |||
| 534 | 10 | result = af_unpack_blend_uint16_t_sse2(left, center, right, center_weight, outer_weight, round_mask, zero); | |
| 535 | |||
| 536 | _mm_store_si128(reinterpret_cast< __m128i*>(dstp), result); | ||
| 537 | |||
| 538 |
2/2✓ Branch 221 → 122 taken 10 times.
✓ Branch 221 → 222 taken 10 times.
|
20 | for (size_t x = 16; x < loop_limit; x += 16) { |
| 539 | 10 | left = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + x - 4 * sizeof(uint16_t))); | |
| 540 | 10 | center = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp + x)); | |
| 541 | 20 | right = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + x + 4 * sizeof(uint16_t))); | |
| 542 | |||
| 543 | 10 | result = af_unpack_blend_uint16_t_sse2(left, center, right, center_weight, outer_weight, round_mask, zero); | |
| 544 | |||
| 545 | 10 | _mm_store_si128(reinterpret_cast< __m128i*>(dstp + x), result); | |
| 546 | } | ||
| 547 | |||
| 548 | 10 | left = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + loop_limit - 4 * sizeof(uint16_t))); | |
| 549 | 10 | center = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + loop_limit)); | |
| 550 | 30 | right = _mm_or_si128(_mm_and_si128(center, right_mask), _mm_srli_si128(center, 4 * sizeof(uint16_t))); | |
| 551 | |||
| 552 | 10 | result = af_unpack_blend_uint16_t_sse2(left, center, right, center_weight, outer_weight, round_mask, zero); | |
| 553 | |||
| 554 | 10 | _mm_storeu_si128(reinterpret_cast< __m128i*>(dstp + loop_limit), result); | |
| 555 | |||
| 556 | |||
| 557 | 10 | dstp += dst_pitch; | |
| 558 | 10 | srcp += src_pitch; | |
| 559 | } | ||
| 560 | 2 | } | |
| 561 | |||
| 562 | #if defined(GCC) || defined(CLANG) | ||
| 563 | __attribute__((__target__("sse4.1"))) | ||
| 564 | #endif | ||
| 565 | 2 | void af_horizontal_rgb64_sse41(BYTE* dstp, const BYTE* srcp, size_t dst_pitch, size_t src_pitch, size_t height, size_t width, size_t amount) | |
| 566 | { | ||
| 567 | // width is really width | ||
| 568 | 2 | size_t width_bytes = width * 4 * sizeof(uint16_t); | |
| 569 | 2 | size_t loop_limit = width_bytes - 16; | |
| 570 | |||
| 571 | 2 | short t = short((amount + 256) >> 9); | |
| 572 | 2 | __m128i center_weight = _mm_set1_epi32(t); | |
| 573 | 4 | __m128i outer_weight = _mm_set1_epi32(64 - t); | |
| 574 | 2 | __m128i round_mask = _mm_set1_epi32(0x40); | |
| 575 | 2 | __m128i zero = _mm_setzero_si128(); | |
| 576 | 2 | __m128i left_mask = _mm_set_epi32(0, 0, 0xFFFFFFFF, 0xFFFFFFFF); | |
| 577 | 2 | __m128i right_mask = _mm_set_epi32(0xFFFFFFFF, 0xFFFFFFFF, 0, 0); | |
| 578 | |||
| 579 | __m128i center, right, left, result; | ||
| 580 | |||
| 581 |
2/2✓ Branch 197 → 21 taken 10 times.
✓ Branch 197 → 198 taken 2 times.
|
12 | for (size_t y = 0; y < height; ++y) { |
| 582 | 10 | center = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp)); | |
| 583 | 10 | right = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + 4 * sizeof(uint16_t))); // move right by one 4*uint16_t pixelblock | |
| 584 | 30 | left = _mm_or_si128(_mm_and_si128(center, left_mask), _mm_slli_si128(center, 8)); | |
| 585 | |||
| 586 | 10 | result = af_unpack_blend_uint16_t_sse41(left, center, right, center_weight, outer_weight, round_mask, zero); | |
| 587 | |||
| 588 | _mm_store_si128(reinterpret_cast<__m128i*>(dstp), result); | ||
| 589 | |||
| 590 |
2/2✓ Branch 137 → 80 taken 10 times.
✓ Branch 137 → 138 taken 10 times.
|
20 | for (size_t x = 16; x < loop_limit; x += 16) { |
| 591 | 10 | left = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + x - 4 * sizeof(uint16_t))); | |
| 592 | 10 | center = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp + x)); | |
| 593 | 20 | right = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + x + 4 * sizeof(uint16_t))); | |
| 594 | |||
| 595 | 10 | result = af_unpack_blend_uint16_t_sse41(left, center, right, center_weight, outer_weight, round_mask, zero); | |
| 596 | |||
| 597 | 10 | _mm_store_si128(reinterpret_cast<__m128i*>(dstp + x), result); | |
| 598 | } | ||
| 599 | |||
| 600 | 10 | left = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + loop_limit - 4 * sizeof(uint16_t))); | |
| 601 | 10 | center = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + loop_limit)); | |
| 602 | 30 | right = _mm_or_si128(_mm_and_si128(center, right_mask), _mm_srli_si128(center, 4 * sizeof(uint16_t))); | |
| 603 | |||
| 604 | 10 | result = af_unpack_blend_uint16_t_sse41(left, center, right, center_weight, outer_weight, round_mask, zero); | |
| 605 | |||
| 606 | 10 | _mm_storeu_si128(reinterpret_cast<__m128i*>(dstp + loop_limit), result); | |
| 607 | |||
| 608 | |||
| 609 | 10 | dstp += dst_pitch; | |
| 610 | 10 | srcp += src_pitch; | |
| 611 | } | ||
| 612 | 2 | } | |
| 613 | |||
| 614 | #ifdef X86_32 | ||
| 615 | |||
| 616 | void af_horizontal_rgb32_mmx(BYTE* dstp, const BYTE* srcp, size_t dst_pitch, size_t src_pitch, size_t height, size_t width, size_t amount) { | ||
| 617 | size_t width_bytes = width * 4; | ||
| 618 | size_t loop_limit = width_bytes - 8; | ||
| 619 | int center_weight_c = amount*2; | ||
| 620 | int outer_weight_c = 32768-amount; | ||
| 621 | |||
| 622 | short t = short((amount + 256) >> 9); | ||
| 623 | __m64 center_weight = _mm_set1_pi16(t); | ||
| 624 | __m64 outer_weight = _mm_set1_pi16(64 - t); | ||
| 625 | __m64 round_mask = _mm_set1_pi16(0x40); | ||
| 626 | __m64 zero = _mm_setzero_si64(); | ||
| 627 | __m64 left_mask = _mm_set_pi32(0, 0xFFFFFFFF); | ||
| 628 | __m64 right_mask = _mm_set_pi32(0xFFFFFFFF, 0); | ||
| 629 | |||
| 630 | __m64 center, right, left, result; | ||
| 631 | |||
| 632 | for (size_t y = 0; y < height; ++y) { | ||
| 633 | center = *reinterpret_cast<const __m64*>(srcp); | ||
| 634 | right = *reinterpret_cast<const __m64*>(srcp + 4); | ||
| 635 | left = _mm_or_si64(_mm_and_si64(center, left_mask), _mm_slli_si64(center, 32)); | ||
| 636 | |||
| 637 | result = af_unpack_blend_mmx(left, center, right, center_weight, outer_weight, round_mask, zero); | ||
| 638 | |||
| 639 | *reinterpret_cast< __m64*>(dstp) = result; | ||
| 640 | |||
| 641 | for (size_t x = 8; x < loop_limit; x+=8) { | ||
| 642 | left = *reinterpret_cast<const __m64*>(srcp + x - 4); | ||
| 643 | center = *reinterpret_cast<const __m64*>(srcp + x); | ||
| 644 | right = *reinterpret_cast<const __m64*>(srcp + x + 4); | ||
| 645 | |||
| 646 | result = af_unpack_blend_mmx(left, center, right, center_weight, outer_weight, round_mask, zero); | ||
| 647 | |||
| 648 | *reinterpret_cast< __m64*>(dstp+x) = result; | ||
| 649 | } | ||
| 650 | |||
| 651 | left = *reinterpret_cast<const __m64*>(srcp + loop_limit - 4); | ||
| 652 | center = *reinterpret_cast<const __m64*>(srcp + loop_limit); | ||
| 653 | right = _mm_or_si64(_mm_and_si64(center, right_mask), _mm_srli_si64(center, 32)); | ||
| 654 | |||
| 655 | result = af_unpack_blend_mmx(left, center, right, center_weight, outer_weight, round_mask, zero); | ||
| 656 | |||
| 657 | *reinterpret_cast< __m64*>(dstp + loop_limit) = result; | ||
| 658 | |||
| 659 | dstp += dst_pitch; | ||
| 660 | srcp += src_pitch; | ||
| 661 | } | ||
| 662 | _mm_empty(); | ||
| 663 | } | ||
| 664 | |||
| 665 | #endif | ||
| 666 | |||
| 667 | |||
| 668 | |||
| 669 | static AVS_FORCEINLINE __m128i af_blend_yuy2_sse2(__m128i &left, __m128i ¢er, __m128i &right, __m128i &luma_mask, | ||
| 670 | __m128i ¢er_weight, __m128i &outer_weight, __m128i &round_mask) { | ||
| 671 | 40 | __m128i left_luma = _mm_and_si128(left, luma_mask); //0 Y5 0 Y4 0 Y3 0 Y2 0 Y1 0 Y0 0 Y-1 0 Y-2 | |
| 672 | 20 | __m128i center_luma = _mm_and_si128(center, luma_mask); //0 Y7 0 Y6 0 Y5 0 Y4 0 Y3 0 Y2 0 Y1 0 Y0 | |
| 673 | 20 | __m128i right_luma = _mm_and_si128(right, luma_mask); //0 Y9 0 Y8 0 Y7 0 Y6 0 Y5 0 Y4 0 Y3 0 Y2 | |
| 674 | |||
| 675 | 20 | left_luma = _mm_or_si128( | |
| 676 | _mm_srli_si128(left_luma, 2), // 0 0 0 Y5 0 Y4 0 Y3 0 Y2 0 Y1 0 Y0 0 Y-1 | ||
| 677 | _mm_slli_si128(right_luma, 6) // 0 Y6 0 Y5 0 Y4 0 Y3 0 Y2 0 0 0 0 0 0 | ||
| 678 | ); // Y6..Y0 (Y-1) | ||
| 679 | |||
| 680 | 40 | right_luma = _mm_or_si128( | |
| 681 | _mm_srli_si128(center_luma, 2),//0 0 0 Y7 0 Y6 0 Y5 0 Y4 0 Y3 0 Y2 0 Y1 | ||
| 682 | _mm_slli_si128(right_luma, 2) //0 Y8 0 Y7 0 Y6 0 Y5 0 Y4 0 Y3 0 Y2 0 0 | ||
| 683 | ); // Y8..Y1 | ||
| 684 | |||
| 685 | 20 | __m128i result_luma = af_blend_sse2(left_luma, center_luma, right_luma, center_weight, outer_weight, round_mask); | |
| 686 | |||
| 687 | 20 | __m128i left_chroma = _mm_srli_epi16(left, 8); //0 V 0 U 0 V 0 U | |
| 688 | 20 | __m128i center_chroma = _mm_srli_epi16(center, 8); //0 V 0 U 0 V 0 U | |
| 689 | 40 | __m128i right_chroma = _mm_srli_epi16(right, 8); //0 V 0 U 0 V 0 U | |
| 690 | |||
| 691 | 20 | __m128i result_chroma = af_blend_sse2(left_chroma, center_chroma, right_chroma, center_weight, outer_weight, round_mask); | |
| 692 | |||
| 693 | 20 | __m128i lo_lu_hi_co = _mm_packus_epi16(result_luma, result_chroma); // U3 V3 U2 V2 U1 V1 U0 V0 Y7 Y6 Y5 Y4 Y3 Y2 Y1 Y0 | |
| 694 | 20 | __m128i result = _mm_unpacklo_epi8(lo_lu_hi_co, _mm_srli_si128(lo_lu_hi_co, 8)); // U3 Y7 V3 Y6 U2 Y5 V2 Y4 U1 Y3 V1 Y2 U0 Y1 V0 Y0 | |
| 695 | 20 | return result; | |
| 696 | } | ||
| 697 | |||
| 698 | |||
| 699 | 2 | void af_horizontal_yuy2_sse2(BYTE* dstp, const BYTE* srcp, size_t dst_pitch, size_t src_pitch, size_t height, size_t width, size_t amount) { | |
| 700 | 2 | size_t width_bytes = width * 2; | |
| 701 | 2 | size_t loop_limit = width_bytes - 16; | |
| 702 | |||
| 703 | 2 | short t = short((amount + 256) >> 9); | |
| 704 | 2 | __m128i center_weight = _mm_set1_epi16(t); | |
| 705 | 4 | __m128i outer_weight = _mm_set1_epi16(64 - t); | |
| 706 | 2 | __m128i round_mask = _mm_set1_epi16(0x40); | |
| 707 | 2 | __m128i left_mask = _mm_set_epi32(0, 0, 0, 0xFFFFFFFF); | |
| 708 | 2 | __m128i right_mask = _mm_set_epi32(0xFFFFFFFF, 0, 0, 0); | |
| 709 | 2 | __m128i left_mask_small = _mm_set_epi16(0, 0, 0, 0, 0, 0, 0x00FF, 0); | |
| 710 | 2 | __m128i right_mask_small = _mm_set_epi16(0, 0x00FF, 0, 0, 0, 0, 0, 0); | |
| 711 | 2 | __m128i luma_mask = _mm_set1_epi16(0xFF); | |
| 712 | |||
| 713 | __m128i center, right, left, result; | ||
| 714 | |||
| 715 |
2/2✓ Branch 230 → 27 taken 10 times.
✓ Branch 230 → 231 taken 2 times.
|
12 | for (size_t y = 0; y < height; ++y) { |
| 716 | 10 | center = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp));//V1 Y3 U1 Y2 V0 Y1 U0 Y0 | |
| 717 | 10 | right = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + 4));//V2 Y5 U2 Y4 V1 Y3 U1 Y2 | |
| 718 | |||
| 719 | //todo: now this is dumb | ||
| 720 | 20 | left = _mm_or_si128( | |
| 721 | _mm_and_si128(center, left_mask), | ||
| 722 | _mm_slli_si128(center, 4) | ||
| 723 | );//V0 Y1 U0 Y0 V0 Y1 U0 Y0 | ||
| 724 | 40 | left = _mm_or_si128( | |
| 725 | _mm_andnot_si128(left_mask_small, left), | ||
| 726 | _mm_and_si128(_mm_slli_si128(center, 2), left_mask_small) | ||
| 727 | );//V0 Y1 U0 Y0 V0 Y0 U0 Y0 | ||
| 728 | |||
| 729 | 10 | result = af_blend_yuy2_sse2(left, center, right, luma_mask, center_weight, outer_weight, round_mask); | |
| 730 | |||
| 731 | _mm_store_si128(reinterpret_cast< __m128i*>(dstp), result); | ||
| 732 | |||
| 733 |
1/2✗ Branch 159 → 97 not taken.
✓ Branch 159 → 160 taken 10 times.
|
10 | for (size_t x = 16; x < loop_limit; x+=16) { |
| 734 | ✗ | left = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + x - 4));//V0 Y1 U0 Y0 V-1 Y-1 U-1 Y-2 | |
| 735 | ✗ | center = _mm_load_si128(reinterpret_cast<const __m128i*>(srcp + x)); //V1 Y3 U1 Y2 V0 Y1 U0 Y0 | |
| 736 | ✗ | right = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + x + 4));//V2 Y5 U2 Y4 V1 Y3 U1 Y2 | |
| 737 | |||
| 738 | ✗ | result = af_blend_yuy2_sse2(left, center, right, luma_mask, center_weight, outer_weight, round_mask); | |
| 739 | |||
| 740 | ✗ | _mm_store_si128(reinterpret_cast< __m128i*>(dstp+x), result); | |
| 741 | } | ||
| 742 | |||
| 743 | 10 | left = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + loop_limit - 4)); | |
| 744 | 10 | center = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcp + loop_limit)); //V1 Y3 U1 Y2 V0 Y1 U0 Y0 | |
| 745 | |||
| 746 | //todo: now this is dumb2 | ||
| 747 | 20 | right = _mm_or_si128( | |
| 748 | _mm_and_si128(center, right_mask), | ||
| 749 | _mm_srli_si128(center, 4) | ||
| 750 | );//V1 Y3 U1 Y2 V1 Y3 U1 Y2 | ||
| 751 | |||
| 752 | 40 | right = _mm_or_si128( | |
| 753 | _mm_andnot_si128(right_mask_small, right), | ||
| 754 | _mm_and_si128(_mm_srli_si128(center, 2), right_mask_small) | ||
| 755 | );//V1 Y3 U1 Y3 V1 Y3 U1 Y2 | ||
| 756 | |||
| 757 | 10 | result = af_blend_yuy2_sse2(left, center, right, luma_mask, center_weight, outer_weight, round_mask); | |
| 758 | |||
| 759 | 10 | _mm_storeu_si128(reinterpret_cast< __m128i*>(dstp + loop_limit), result); | |
| 760 | |||
| 761 | 10 | dstp += dst_pitch; | |
| 762 | 10 | srcp += src_pitch; | |
| 763 | } | ||
| 764 | 2 | } | |
| 765 | |||
| 766 | |||
| 767 | |||
| 768 | #ifdef X86_32 | ||
| 769 | // ------------------------------------- | ||
| 770 | // Blur/Sharpen Horizontal YUY2 MMX Code | ||
| 771 | // ------------------------------------- | ||
| 772 | // | ||
| 773 | static AVS_FORCEINLINE __m64 af_blend_yuy2_mmx(__m64 &left, __m64 ¢er, __m64 &right, __m64 &luma_mask, | ||
| 774 | __m64 ¢er_weight, __m64 &outer_weight, __m64 &round_mask) { | ||
| 775 | __m64 left_luma = _mm_and_si64(left, luma_mask); //0 Y1 0 Y0 0 Y-1 0 Y-2 | ||
| 776 | __m64 center_luma = _mm_and_si64(center, luma_mask); //0 Y3 0 Y2 0 Y1 0 Y0 | ||
| 777 | __m64 right_luma = _mm_and_si64(right, luma_mask); //0 Y5 0 Y4 0 Y3 0 Y2 | ||
| 778 | |||
| 779 | left_luma = _mm_or_si64( | ||
| 780 | _mm_srli_si64(left_luma, 16), // 0 0 0 Y1 0 Y0 0 Y-1 | ||
| 781 | _mm_slli_si64(right_luma, 48) // 0 Y2 0 0 0 0 0 0 | ||
| 782 | ); | ||
| 783 | |||
| 784 | right_luma = _mm_or_si64( | ||
| 785 | _mm_srli_si64(center_luma, 16),//0 0 0 Y3 0 Y2 0 Y1 | ||
| 786 | _mm_slli_si64(right_luma, 16)//0 Y4 0 Y3 0 Y2 0 0 | ||
| 787 | ); | ||
| 788 | |||
| 789 | __m64 result_luma = af_blend_mmx(left_luma, center_luma, right_luma, center_weight, outer_weight, round_mask); | ||
| 790 | |||
| 791 | __m64 left_chroma = _mm_srli_pi16(left, 8); //0 V 0 U 0 V 0 U | ||
| 792 | __m64 center_chroma = _mm_srli_pi16(center, 8); //0 V 0 U 0 V 0 U | ||
| 793 | __m64 right_chroma = _mm_srli_pi16(right, 8); //0 V 0 U 0 V 0 U | ||
| 794 | |||
| 795 | __m64 result_chroma = af_blend_mmx(left_chroma, center_chroma, right_chroma, center_weight, outer_weight, round_mask); | ||
| 796 | |||
| 797 | __m64 lo_lu_hi_co = _m_packuswb(result_luma, result_chroma); // U1 V1 U0 V0 Y3 Y2 Y1 Y0 | ||
| 798 | __m64 result = _mm_unpacklo_pi8(lo_lu_hi_co, _mm_srli_si64(lo_lu_hi_co, 32)); // U1 Y3 V1 Y2 U0 Y1 V0 Y0 | ||
| 799 | return result; | ||
| 800 | } | ||
| 801 | |||
| 802 | |||
| 803 | void af_horizontal_yuy2_mmx(BYTE* dstp, const BYTE* srcp, size_t dst_pitch, size_t src_pitch, size_t height, size_t width, size_t amount) { | ||
| 804 | size_t width_bytes = width * 2; | ||
| 805 | size_t loop_limit = width_bytes - 8; | ||
| 806 | |||
| 807 | short t = short((amount + 256) >> 9); | ||
| 808 | __m64 center_weight = _mm_set1_pi16(t); | ||
| 809 | __m64 outer_weight = _mm_set1_pi16(64 - t); | ||
| 810 | __m64 round_mask = _mm_set1_pi16(0x40); | ||
| 811 | __m64 left_mask = _mm_set_pi32(0, 0xFFFFFFFF); | ||
| 812 | __m64 right_mask = _mm_set_pi32(0xFFFFFFFF, 0); | ||
| 813 | __m64 left_mask_small = _mm_set_pi16(0, 0, 0x00FF, 0); | ||
| 814 | __m64 right_mask_small = _mm_set_pi16(0, 0x00FF, 0, 0); | ||
| 815 | __m64 luma_mask = _mm_set1_pi16(0xFF); | ||
| 816 | |||
| 817 | __m64 center, right, left, result; | ||
| 818 | |||
| 819 | for (size_t y = 0; y < height; ++y) { | ||
| 820 | center = *reinterpret_cast<const __m64*>(srcp);//V1 Y3 U1 Y2 V0 Y1 U0 Y0 | ||
| 821 | right = *reinterpret_cast<const __m64*>(srcp + 4);//V2 Y5 U2 Y4 V1 Y3 U1 Y2 | ||
| 822 | |||
| 823 | //todo: now this is dumb | ||
| 824 | left = _mm_or_si64( | ||
| 825 | _mm_and_si64(center, left_mask), | ||
| 826 | _mm_slli_si64(center, 32) | ||
| 827 | );//V0 Y1 U0 Y0 V0 Y1 U0 Y0 | ||
| 828 | left = _mm_or_si64( | ||
| 829 | _mm_andnot_si64(left_mask_small, left), | ||
| 830 | _mm_and_si64(_mm_slli_si64(center, 16), left_mask_small) | ||
| 831 | );//V0 Y1 U0 Y0 V0 Y0 U0 Y0 | ||
| 832 | |||
| 833 | result = af_blend_yuy2_mmx(left, center, right, luma_mask, center_weight, outer_weight, round_mask); | ||
| 834 | |||
| 835 | *reinterpret_cast< __m64*>(dstp) = result; | ||
| 836 | |||
| 837 | for (size_t x = 8; x < loop_limit; x+=8) { | ||
| 838 | left = *reinterpret_cast<const __m64*>(srcp + x - 4);//V0 Y1 U0 Y0 V-1 Y-1 U-1 Y-2 | ||
| 839 | center = *reinterpret_cast<const __m64*>(srcp + x); //V1 Y3 U1 Y2 V0 Y1 U0 Y0 | ||
| 840 | right = *reinterpret_cast<const __m64*>(srcp + x + 4);//V2 Y5 U2 Y4 V1 Y3 U1 Y2 | ||
| 841 | |||
| 842 | __m64 result = af_blend_yuy2_mmx(left, center, right, luma_mask, center_weight, outer_weight, round_mask); | ||
| 843 | |||
| 844 | *reinterpret_cast< __m64*>(dstp+x) = result; | ||
| 845 | } | ||
| 846 | |||
| 847 | left = *reinterpret_cast<const __m64*>(srcp + loop_limit - 4); | ||
| 848 | center = *reinterpret_cast<const __m64*>(srcp + loop_limit); //V1 Y3 U1 Y2 V0 Y1 U0 Y0 | ||
| 849 | |||
| 850 | //todo: now this is dumb2 | ||
| 851 | right = _mm_or_si64( | ||
| 852 | _mm_and_si64(center, right_mask), | ||
| 853 | _mm_srli_si64(center, 32) | ||
| 854 | );//V1 Y3 U1 Y2 V1 Y3 U1 Y2 | ||
| 855 | right = _mm_or_si64( | ||
| 856 | _mm_andnot_si64(right_mask_small, right), | ||
| 857 | _mm_and_si64(_mm_srli_si64(center, 16), right_mask_small) | ||
| 858 | );//V1 Y3 U1 Y3 V1 Y3 U1 Y2 | ||
| 859 | |||
| 860 | result = af_blend_yuy2_mmx(left, center, right, luma_mask, center_weight, outer_weight, round_mask); | ||
| 861 | |||
| 862 | *reinterpret_cast< __m64*>(dstp + loop_limit) = result; | ||
| 863 | |||
| 864 | dstp += dst_pitch; | ||
| 865 | srcp += src_pitch; | ||
| 866 | } | ||
| 867 | _mm_empty(); | ||
| 868 | } | ||
| 869 | |||
| 870 | |||
| 871 | #endif | ||
| 872 | |||
| 873 | |||
| 874 | 3 | void af_horizontal_planar_sse2(BYTE* dstp, size_t height, size_t pitch, size_t width, size_t amount) { | |
| 875 | 3 | size_t mod16_width = (width / 16) * 16; | |
| 876 |
1/2✗ Branch 2 → 3 not taken.
✓ Branch 2 → 4 taken 3 times.
|
3 | size_t sse_loop_limit = width == mod16_width ? mod16_width - 16 : mod16_width; |
| 877 | 3 | int center_weight_c = int(amount*2); | |
| 878 | 3 | int outer_weight_c = int(32768-amount); | |
| 879 | |||
| 880 | 3 | short t = short((amount + 256) >> 9); | |
| 881 | 3 | __m128i center_weight = _mm_set1_epi16(t); | |
| 882 | 6 | __m128i outer_weight = _mm_set1_epi16(64 - t); | |
| 883 | 3 | __m128i round_mask = _mm_set1_epi16(0x40); | |
| 884 | 3 | __m128i zero = _mm_setzero_si128(); | |
| 885 | 3 | __m128i left_mask = _mm_set_epi32(0, 0, 0, 0xFF); | |
| 886 | 3 | __m128i right_mask = _mm_set_epi8((char)0xFF, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); | |
| 887 | |||
| 888 | __m128i left; | ||
| 889 | |||
| 890 |
2/2✓ Branch 215 → 24 taken 17 times.
✓ Branch 215 → 216 taken 3 times.
|
20 | for (size_t y = 0; y < height; ++y) { |
| 891 | //left border | ||
| 892 | 17 | __m128i center = _mm_load_si128(reinterpret_cast<const __m128i*>(dstp)); | |
| 893 | 17 | __m128i right = _mm_loadu_si128(reinterpret_cast<const __m128i*>(dstp+1)); | |
| 894 | 51 | left = _mm_or_si128(_mm_and_si128(center, left_mask), _mm_slli_si128(center, 1)); | |
| 895 | |||
| 896 | 17 | __m128i result = af_unpack_blend_sse2(left, center, right, center_weight, outer_weight, round_mask, zero); | |
| 897 | 34 | left = _mm_loadu_si128(reinterpret_cast<const __m128i*>(dstp+15)); | |
| 898 | _mm_store_si128(reinterpret_cast<__m128i*>(dstp), result); | ||
| 899 | |||
| 900 | //main processing loop | ||
| 901 |
2/2✓ Branch 142 → 85 taken 24 times.
✓ Branch 142 → 143 taken 17 times.
|
41 | for (size_t x = 16; x < sse_loop_limit; x+= 16) { |
| 902 | 24 | center = _mm_load_si128(reinterpret_cast<const __m128i*>(dstp+x)); | |
| 903 | 48 | right = _mm_loadu_si128(reinterpret_cast<const __m128i*>(dstp+x+1)); | |
| 904 | |||
| 905 | 24 | result = af_unpack_blend_sse2(left, center, right, center_weight, outer_weight, round_mask, zero); | |
| 906 | |||
| 907 | 24 | left = _mm_loadu_si128(reinterpret_cast<const __m128i*>(dstp+x+15)); | |
| 908 | |||
| 909 | 24 | _mm_store_si128(reinterpret_cast<__m128i*>(dstp+x), result); | |
| 910 | } | ||
| 911 | |||
| 912 | //right border | ||
| 913 |
1/2✗ Branch 143 → 144 not taken.
✓ Branch 143 → 201 taken 17 times.
|
17 | if(mod16_width == width) { //width is mod8, process with mmx |
| 914 | ✗ | center = _mm_load_si128(reinterpret_cast<const __m128i*>(dstp+mod16_width-16)); | |
| 915 | ✗ | right = _mm_or_si128(_mm_and_si128(center, right_mask), _mm_srli_si128(center, 1)); | |
| 916 | |||
| 917 | ✗ | result = af_unpack_blend_sse2(left, center, right, center_weight, outer_weight, round_mask, zero); | |
| 918 | |||
| 919 | ✗ | _mm_store_si128(reinterpret_cast<__m128i*>(dstp+mod16_width-16), result); | |
| 920 | } else { //some stuff left | ||
| 921 | 17 | BYTE l = _mm_cvtsi128_si32(left) & 0xFF; | |
| 922 | 17 | af_horizontal_planar_process_line_c<uint8_t>(l, dstp+mod16_width, width-mod16_width, center_weight_c, outer_weight_c); | |
| 923 | |||
| 924 | } | ||
| 925 | |||
| 926 | 17 | dstp += pitch; | |
| 927 | } | ||
| 928 | 3 | } | |
| 929 | |||
| 930 | 3 | void af_horizontal_planar_uint16_t_sse2(BYTE* dstp, size_t height, size_t pitch, size_t row_size, size_t amount, int bits_per_pixel) { | |
| 931 | 3 | size_t mod16_width = (row_size / 16) * 16; | |
| 932 |
1/2✗ Branch 2 → 3 not taken.
✓ Branch 2 → 4 taken 3 times.
|
3 | size_t sse_loop_limit = row_size == mod16_width ? mod16_width - 16 : mod16_width; |
| 933 | 3 | int center_weight_c = int(amount * 2); | |
| 934 | 3 | int outer_weight_c = int(32768 - amount); | |
| 935 | |||
| 936 | 3 | int t = int((amount + 256) >> 9); | |
| 937 | 3 | __m128i center_weight = _mm_set1_epi32(t); | |
| 938 | 6 | __m128i outer_weight = _mm_set1_epi32(64 - t); | |
| 939 | 3 | __m128i round_mask = _mm_set1_epi32(0x40); | |
| 940 | 3 | __m128i zero = _mm_setzero_si128(); | |
| 941 | 3 | __m128i left_mask = _mm_set_epi16(0, 0, 0, 0, 0, 0, 0, (short)0xFFFF); // 0, 0, 0, 0, 0, 0, 0, FFFF | |
| 942 | 3 | __m128i right_mask = _mm_set_epi16((short)0xFFFF, 0, 0, 0, 0, 0, 0, 0); | |
| 943 | |||
| 944 | __m128i left; | ||
| 945 | |||
| 946 |
2/2✓ Branch 341 → 24 taken 17 times.
✓ Branch 341 → 342 taken 3 times.
|
20 | for (size_t y = 0; y < height; ++y) { |
| 947 | //left border | ||
| 948 | 17 | __m128i center = _mm_load_si128(reinterpret_cast<const __m128i*>(dstp)); | |
| 949 | 17 | __m128i right = _mm_loadu_si128(reinterpret_cast<const __m128i*>(dstp + 2)); | |
| 950 | 51 | left = _mm_or_si128(_mm_and_si128(center, left_mask), _mm_slli_si128(center, 2)); | |
| 951 | |||
| 952 | 17 | __m128i result = af_unpack_blend_uint16_t_sse2(left, center, right, center_weight, outer_weight, round_mask, zero); | |
| 953 | 34 | left = _mm_loadu_si128(reinterpret_cast<const __m128i*>(dstp + (16 - 2))); | |
| 954 | _mm_store_si128(reinterpret_cast<__m128i*>(dstp), result); | ||
| 955 | |||
| 956 | //main processing loop | ||
| 957 |
2/2✓ Branch 226 → 127 taken 24 times.
✓ Branch 226 → 227 taken 17 times.
|
41 | for (size_t x = 16; x < sse_loop_limit; x += 16) { |
| 958 | 24 | center = _mm_load_si128(reinterpret_cast<const __m128i*>(dstp + x)); | |
| 959 | 48 | right = _mm_loadu_si128(reinterpret_cast<const __m128i*>(dstp + x + 2)); | |
| 960 | |||
| 961 | 24 | result = af_unpack_blend_uint16_t_sse2(left, center, right, center_weight, outer_weight, round_mask, zero); | |
| 962 | |||
| 963 | 24 | left = _mm_loadu_si128(reinterpret_cast<const __m128i*>(dstp + x + (16 - 2))); | |
| 964 | |||
| 965 | 24 | _mm_store_si128(reinterpret_cast<__m128i*>(dstp + x), result); | |
| 966 | } | ||
| 967 | |||
| 968 | //right border | ||
| 969 |
1/2✗ Branch 227 → 228 not taken.
✓ Branch 227 → 327 taken 17 times.
|
17 | if (mod16_width == row_size) { //width is mod8, process with mmx |
| 970 | ✗ | center = _mm_load_si128(reinterpret_cast<const __m128i*>(dstp + mod16_width - 16)); | |
| 971 | ✗ | right = _mm_or_si128(_mm_and_si128(center, right_mask), _mm_srli_si128(center, 2)); | |
| 972 | |||
| 973 | ✗ | result = af_unpack_blend_uint16_t_sse2(left, center, right, center_weight, outer_weight, round_mask, zero); | |
| 974 | |||
| 975 | ✗ | _mm_store_si128(reinterpret_cast<__m128i*>(dstp + mod16_width - 16), result); | |
| 976 | } | ||
| 977 | else { //some stuff left | ||
| 978 | 17 | uint16_t l = _mm_cvtsi128_si32(left) & 0xFFFF; | |
| 979 | 17 | af_horizontal_planar_process_line_uint16_c(l, dstp + mod16_width, row_size - mod16_width, center_weight_c, outer_weight_c, bits_per_pixel); | |
| 980 | } | ||
| 981 | |||
| 982 | 17 | dstp += pitch; | |
| 983 | } | ||
| 984 | 3 | } | |
| 985 | |||
| 986 | #if defined(GCC) || defined(CLANG) | ||
| 987 | __attribute__((__target__("sse4.1"))) | ||
| 988 | #endif | ||
| 989 | 3 | void af_horizontal_planar_uint16_t_sse41(BYTE* dstp, size_t height, size_t pitch, size_t row_size, size_t amount, int bits_per_pixel) | |
| 990 | { | ||
| 991 | 3 | size_t mod16_width = (row_size / 16) * 16; | |
| 992 |
1/2✗ Branch 2 → 3 not taken.
✓ Branch 2 → 4 taken 3 times.
|
3 | size_t sse_loop_limit = row_size == mod16_width ? mod16_width - 16 : mod16_width; |
| 993 | 3 | int center_weight_c = int(amount * 2); | |
| 994 | 3 | int outer_weight_c = int(32768 - amount); | |
| 995 | |||
| 996 | 3 | int t = int((amount + 256) >> 9); | |
| 997 | 3 | __m128i center_weight = _mm_set1_epi32(t); | |
| 998 | 6 | __m128i outer_weight = _mm_set1_epi32(64 - t); | |
| 999 | 3 | __m128i round_mask = _mm_set1_epi32(0x40); | |
| 1000 | 3 | __m128i zero = _mm_setzero_si128(); | |
| 1001 | 3 | __m128i left_mask = _mm_set_epi16(0, 0, 0, 0, 0, 0, 0, (short)0xFFFF); // 0, 0, 0, 0, 0, 0, 0, FFFF | |
| 1002 | 3 | __m128i right_mask = _mm_set_epi16((short)0xFFFF, 0, 0, 0, 0, 0, 0, 0); | |
| 1003 | |||
| 1004 | __m128i left; | ||
| 1005 | |||
| 1006 |
2/2✓ Branch 215 → 24 taken 17 times.
✓ Branch 215 → 216 taken 3 times.
|
20 | for (size_t y = 0; y < height; ++y) { |
| 1007 | //left border | ||
| 1008 | 17 | __m128i center = _mm_load_si128(reinterpret_cast<const __m128i*>(dstp)); | |
| 1009 | 17 | __m128i right = _mm_loadu_si128(reinterpret_cast<const __m128i*>(dstp + 2)); | |
| 1010 | 51 | left = _mm_or_si128(_mm_and_si128(center, left_mask), _mm_slli_si128(center, 2)); | |
| 1011 | |||
| 1012 | 17 | __m128i result = af_unpack_blend_uint16_t_sse41(left, center, right, center_weight, outer_weight, round_mask, zero); | |
| 1013 | 34 | left = _mm_loadu_si128(reinterpret_cast<const __m128i*>(dstp + (16 - 2))); | |
| 1014 | _mm_store_si128(reinterpret_cast<__m128i*>(dstp), result); | ||
| 1015 | |||
| 1016 | //main processing loop | ||
| 1017 |
2/2✓ Branch 142 → 85 taken 24 times.
✓ Branch 142 → 143 taken 17 times.
|
41 | for (size_t x = 16; x < sse_loop_limit; x += 16) { |
| 1018 | 24 | center = _mm_load_si128(reinterpret_cast<const __m128i*>(dstp + x)); | |
| 1019 | 48 | right = _mm_loadu_si128(reinterpret_cast<const __m128i*>(dstp + x + 2)); | |
| 1020 | |||
| 1021 | 24 | result = af_unpack_blend_uint16_t_sse41(left, center, right, center_weight, outer_weight, round_mask, zero); | |
| 1022 | |||
| 1023 | 24 | left = _mm_loadu_si128(reinterpret_cast<const __m128i*>(dstp + x + (16 - 2))); | |
| 1024 | |||
| 1025 | 24 | _mm_store_si128(reinterpret_cast<__m128i*>(dstp + x), result); | |
| 1026 | } | ||
| 1027 | |||
| 1028 | //right border | ||
| 1029 |
1/2✗ Branch 143 → 144 not taken.
✓ Branch 143 → 201 taken 17 times.
|
17 | if (mod16_width == row_size) { //width is mod8, process with mmx |
| 1030 | ✗ | center = _mm_load_si128(reinterpret_cast<const __m128i*>(dstp + mod16_width - 16)); | |
| 1031 | ✗ | right = _mm_or_si128(_mm_and_si128(center, right_mask), _mm_srli_si128(center, 2)); | |
| 1032 | |||
| 1033 | ✗ | result = af_unpack_blend_uint16_t_sse41(left, center, right, center_weight, outer_weight, round_mask, zero); | |
| 1034 | |||
| 1035 | ✗ | _mm_store_si128(reinterpret_cast<__m128i*>(dstp + mod16_width - 16), result); | |
| 1036 | } | ||
| 1037 | else { //some stuff left | ||
| 1038 | 17 | uint16_t l = _mm_cvtsi128_si32(left) & 0xFFFF; | |
| 1039 | 17 | af_horizontal_planar_process_line_uint16_c(l, dstp + mod16_width, row_size - mod16_width, center_weight_c, outer_weight_c, bits_per_pixel); | |
| 1040 | } | ||
| 1041 | |||
| 1042 | 17 | dstp += pitch; | |
| 1043 | } | ||
| 1044 | 3 | } | |
| 1045 | |||
| 1046 | 2 | void af_horizontal_planar_float_sse2(BYTE* dstp, size_t height, size_t pitch, size_t row_size, float amount) { | |
| 1047 | 2 | const float center_weight = amount; | |
| 1048 | 2 | const float outer_weight = (1.0f - amount) / 2.0f; | |
| 1049 | |||
| 1050 | 2 | __m128 center_weight_simd = _mm_set1_ps(center_weight); | |
| 1051 | 2 | __m128 outer_weight_simd = _mm_set1_ps(outer_weight); | |
| 1052 | |||
| 1053 | 2 | size_t mod16_width = (row_size / 16) * 16; | |
| 1054 |
1/2✗ Branch 6 → 7 not taken.
✓ Branch 6 → 8 taken 2 times.
|
2 | size_t sse_loop_limit = row_size == mod16_width ? mod16_width - 16 : mod16_width; |
| 1055 | |||
| 1056 | 2 | __m128i left_mask = _mm_set_epi32(0, 0, 0, 0xFFFFFFFF); | |
| 1057 | 2 | __m128i right_mask = _mm_set_epi32(0xFFFFFFFF, 0, 0, 0); | |
| 1058 | |||
| 1059 | __m128 left; | ||
| 1060 | |||
| 1061 |
2/2✓ Branch 94 → 14 taken 10 times.
✓ Branch 94 → 95 taken 2 times.
|
12 | for (size_t y = 0; y < height; ++y) { |
| 1062 | //left border | ||
| 1063 | 10 | __m128 center = _mm_load_ps(reinterpret_cast<const float*>(dstp)); | |
| 1064 | 10 | __m128 right = _mm_loadu_ps(reinterpret_cast<const float*>(dstp + sizeof(float))); | |
| 1065 | 60 | left = _mm_castsi128_ps(_mm_or_si128(_mm_and_si128(_mm_castps_si128(center), left_mask), _mm_slli_si128(_mm_castps_si128(center), sizeof(float)))); | |
| 1066 | |||
| 1067 | 10 | __m128 result = af_blend_float_sse2(left, center, right, center_weight_simd, outer_weight_simd); | |
| 1068 | 20 | left = _mm_loadu_ps(reinterpret_cast<const float*>(dstp + (16 - sizeof(float)))); | |
| 1069 | _mm_store_ps(reinterpret_cast<float*>(dstp), result); | ||
| 1070 | |||
| 1071 | //main processing loop | ||
| 1072 |
1/2✗ Branch 60 → 42 not taken.
✓ Branch 60 → 61 taken 10 times.
|
10 | for (size_t x = 16; x < sse_loop_limit; x += 16) { |
| 1073 | ✗ | center = _mm_load_ps(reinterpret_cast<const float*>(dstp + x)); | |
| 1074 | ✗ | right = _mm_loadu_ps(reinterpret_cast<const float*>(dstp + x + sizeof(float))); | |
| 1075 | |||
| 1076 | ✗ | result = af_blend_float_sse2(left, center, right, center_weight_simd, outer_weight_simd); | |
| 1077 | |||
| 1078 | ✗ | left = _mm_loadu_ps(reinterpret_cast<const float*>(dstp + x + (16 - sizeof(float)))); | |
| 1079 | |||
| 1080 | ✗ | _mm_store_ps(reinterpret_cast<float*>(dstp + x), result); | |
| 1081 | } | ||
| 1082 | |||
| 1083 | //right border | ||
| 1084 |
1/2✗ Branch 61 → 62 not taken.
✓ Branch 61 → 86 taken 10 times.
|
10 | if (mod16_width == row_size) { //width is mod8, process with mmx |
| 1085 | ✗ | center = _mm_load_ps(reinterpret_cast<const float*>(dstp + mod16_width - 16)); | |
| 1086 | ✗ | right = _mm_castsi128_ps(_mm_or_si128(_mm_and_si128(_mm_castps_si128(center), right_mask), _mm_srli_si128(_mm_castps_si128(center), sizeof(float)))); | |
| 1087 | |||
| 1088 | ✗ | result = af_blend_float_sse2(left, center, right, center_weight_simd, outer_weight_simd); | |
| 1089 | |||
| 1090 | ✗ | _mm_store_ps(reinterpret_cast<float*>(dstp + mod16_width - 16), result); | |
| 1091 | } | ||
| 1092 | else { //some stuff left | ||
| 1093 | 10 | float l = _mm_cvtss_f32(left); | |
| 1094 | 10 | af_horizontal_planar_process_line_float_c(l, (float *)(dstp + mod16_width), row_size - mod16_width, center_weight, outer_weight); | |
| 1095 | } | ||
| 1096 | |||
| 1097 | 10 | dstp += pitch; | |
| 1098 | } | ||
| 1099 | 2 | } | |
| 1100 | |||
| 1101 | |||
| 1102 | #ifdef X86_32 | ||
| 1103 | |||
| 1104 | void af_horizontal_planar_mmx(BYTE* dstp, size_t height, size_t pitch, size_t width, size_t amount) { | ||
| 1105 | size_t mod8_width = (width / 8) * 8; | ||
| 1106 | size_t mmx_loop_limit = width == mod8_width ? mod8_width - 8 : mod8_width; | ||
| 1107 | int center_weight_c = amount*2; | ||
| 1108 | int outer_weight_c = 32768-amount; | ||
| 1109 | |||
| 1110 | short t = short((amount + 256) >> 9); | ||
| 1111 | __m64 center_weight = _mm_set1_pi16(t); | ||
| 1112 | __m64 outer_weight = _mm_set1_pi16(64 - t); | ||
| 1113 | __m64 round_mask = _mm_set1_pi16(0x40); | ||
| 1114 | __m64 zero = _mm_setzero_si64(); | ||
| 1115 | __m64 left_mask = _mm_set_pi8(0, 0, 0, 0, 0, 0, 0, (char)0xFF); | ||
| 1116 | __m64 right_mask = _mm_set_pi8((char)0xFF, 0, 0, 0, 0, 0, 0, 0); | ||
| 1117 | |||
| 1118 | __m64 left; | ||
| 1119 | |||
| 1120 | for (size_t y = 0; y < height; ++y) { | ||
| 1121 | //left border | ||
| 1122 | __m64 center = *reinterpret_cast<const __m64*>(dstp); | ||
| 1123 | __m64 right = *reinterpret_cast<const __m64*>(dstp+1); | ||
| 1124 | left = _mm_or_si64(_mm_and_si64(center, left_mask), _mm_slli_si64(center, 8)); | ||
| 1125 | |||
| 1126 | __m64 result = af_unpack_blend_mmx(left, center, right, center_weight, outer_weight, round_mask, zero); | ||
| 1127 | left = *reinterpret_cast<const __m64*>(dstp+7); | ||
| 1128 | *reinterpret_cast<__m64*>(dstp) = result; | ||
| 1129 | |||
| 1130 | //main processing loop | ||
| 1131 | for (size_t x = 8; x < mmx_loop_limit; x+= 8) { | ||
| 1132 | center = *reinterpret_cast<const __m64*>(dstp+x); | ||
| 1133 | right = *reinterpret_cast<const __m64*>(dstp+x+1); | ||
| 1134 | |||
| 1135 | result = af_unpack_blend_mmx(left, center, right, center_weight, outer_weight, round_mask, zero); | ||
| 1136 | left = *reinterpret_cast<const __m64*>(dstp+x+7); | ||
| 1137 | |||
| 1138 | *reinterpret_cast<__m64*>(dstp+x) = result; | ||
| 1139 | } | ||
| 1140 | |||
| 1141 | //right border | ||
| 1142 | if(mod8_width == width) { //width is mod8, process with mmx | ||
| 1143 | center = *reinterpret_cast<const __m64*>(dstp+mod8_width-8); | ||
| 1144 | right = _mm_or_si64(_mm_and_si64(center, right_mask), _mm_srli_si64(center, 8)); | ||
| 1145 | |||
| 1146 | result = af_unpack_blend_mmx(left, center, right, center_weight, outer_weight, round_mask, zero); | ||
| 1147 | |||
| 1148 | *reinterpret_cast<__m64*>(dstp+mod8_width-8) = result; | ||
| 1149 | } else { //some stuff left | ||
| 1150 | BYTE l = _mm_cvtsi64_si32(left) & 0xFF; | ||
| 1151 | af_horizontal_planar_process_line_c<uint8_t>(l, dstp+mod8_width, width-mod8_width, center_weight_c, outer_weight_c); | ||
| 1152 | } | ||
| 1153 | |||
| 1154 | dstp += pitch; | ||
| 1155 | } | ||
| 1156 | _mm_empty(); | ||
| 1157 | } | ||
| 1158 | |||
| 1159 | |||
| 1160 | #endif | ||
| 1161 | |||
| 1162 | |||
| 1163 | |||
| 1164 | |||
| 1165 | |||
| 1166 | |||
| 1167 | |||
| 1168 | |||
| 1169 | |||
| 1170 | 100 | static inline __m128i _mm_cmple_epu8(__m128i x, __m128i y) | |
| 1171 | { | ||
| 1172 | // Returns 0xFF where x <= y: | ||
| 1173 | 200 | return _mm_cmpeq_epi8(_mm_min_epu8(x, y), x); | |
| 1174 | } | ||
| 1175 | |||
| 1176 | #if defined(GCC) || defined(CLANG) | ||
| 1177 | __attribute__((__target__("sse4.1"))) | ||
| 1178 | #endif | ||
| 1179 | 144 | static inline __m128i _mm_cmple_epu16_sse41(__m128i x, __m128i y) | |
| 1180 | { | ||
| 1181 | // Returns 0xFFFF where x <= y: | ||
| 1182 | 288 | return _mm_cmpeq_epi16(_mm_min_epu16(x, y), x); | |
| 1183 | } | ||
| 1184 | |||
| 1185 | 24 | static inline __m128i _mm_cmple_epu16_sse2(__m128i x, __m128i y) | |
| 1186 | { | ||
| 1187 | // Returns 0xFFFF where x <= y: | ||
| 1188 | 72 | return _mm_cmpeq_epi16(_mm_subs_epu16(x, y), _mm_setzero_si128()); | |
| 1189 | } | ||
| 1190 | |||
| 1191 | // fast: maxThreshold (255) simple accumulate for average | ||
| 1192 | template<bool maxThreshold> | ||
| 1193 | 2 | void accumulate_line_sse2(BYTE* c_plane, const BYTE** planeP, int planes, size_t width, int threshold, int div) { | |
| 1194 | // threshold: 8 bits: 2 bytes in a word. for YUY2: luma<<8 | chroma | ||
| 1195 | // 16 bits: 16 bit value (orig threshold scaled by bits_per_pixel) | ||
| 1196 | 2 | __m128i halfdiv_vector = _mm_set1_epi16(1); // High16(0x10000) | |
| 1197 | 2 | __m128i div_vector = _mm_set1_epi16(65536 / (planes + 1)); // mulhi | |
| 1198 | 2 | __m128i thresh = _mm_set1_epi16(threshold); | |
| 1199 | |||
| 1200 |
4/4void accumulate_line_sse2<false>(unsigned char*, unsigned char const**, int, unsigned long, int, int):
✓ Branch 70 → 15 taken 3 times.
✓ Branch 70 → 71 taken 1 time.
void accumulate_line_sse2<true>(unsigned char*, unsigned char const**, int, unsigned long, int, int):
✓ Branch 58 → 15 taken 3 times.
✓ Branch 58 → 59 taken 1 time.
|
8 | for (size_t x = 0; x < width; x+=16) { |
| 1201 | 12 | __m128i current = _mm_load_si128(reinterpret_cast<const __m128i*>(c_plane+x)); | |
| 1202 | 6 | __m128i zero = _mm_setzero_si128(); | |
| 1203 | 6 | __m128i low = _mm_unpacklo_epi8(current, zero); | |
| 1204 | 6 | __m128i high = _mm_unpackhi_epi8(current, zero); | |
| 1205 | |||
| 1206 |
4/4void accumulate_line_sse2<false>(unsigned char*, unsigned char const**, int, unsigned long, int, int):
✓ Branch 49 → 24 taken 6 times.
✓ Branch 49 → 50 taken 3 times.
void accumulate_line_sse2<true>(unsigned char*, unsigned char const**, int, unsigned long, int, int):
✓ Branch 37 → 24 taken 9 times.
✓ Branch 37 → 38 taken 3 times.
|
21 | for (int plane = planes - 1; plane >= 0; --plane) { |
| 1207 | 30 | __m128i p = _mm_load_si128(reinterpret_cast<const __m128i*>(planeP[plane] + x)); | |
| 1208 | |||
| 1209 | __m128i add_low, add_high; | ||
| 1210 | if (maxThreshold) { | ||
| 1211 | // fast: simple accumulate for average | ||
| 1212 | 9 | add_low = _mm_unpacklo_epi8(p, zero); | |
| 1213 | 9 | add_high = _mm_unpackhi_epi8(p, zero); | |
| 1214 | } else { | ||
| 1215 | 6 | auto pc = _mm_subs_epu8(p, current); // r2507- | |
| 1216 | 6 | auto cp = _mm_subs_epu8(current, p); | |
| 1217 | 6 | auto abs_cp = _mm_or_si128(pc, cp); | |
| 1218 | 6 | auto leq_thresh = _mm_cmple_epu8(abs_cp, thresh); | |
| 1219 | |||
| 1220 | 6 | __m128i andop = _mm_and_si128(leq_thresh, p); | |
| 1221 | 6 | __m128i andnop = _mm_andnot_si128(leq_thresh, current); | |
| 1222 | 6 | __m128i blended = _mm_or_si128(andop, andnop); //abs(p-c) <= thresh ? p : c | |
| 1223 | 6 | add_low = _mm_unpacklo_epi8(blended, zero); | |
| 1224 | 6 | add_high = _mm_unpackhi_epi8(blended, zero); | |
| 1225 | } | ||
| 1226 | |||
| 1227 | 15 | low = _mm_adds_epu16(low, add_low); | |
| 1228 | 15 | high = _mm_adds_epu16(high, add_high); | |
| 1229 | } | ||
| 1230 | |||
| 1231 | // non SSSE3, no _mm_mulhrs_epi16 | ||
| 1232 | // (x*2 * 65536/N + 65536) / 65536 / 2 | ||
| 1233 | // Hi16(x*2 * 65536/N + 1) >> 1 | ||
| 1234 | 12 | low = _mm_mulhi_epu16(_mm_slli_epi16(low, 1), div_vector); | |
| 1235 | 6 | low = _mm_adds_epu16(low, halfdiv_vector); | |
| 1236 | 6 | low = _mm_srli_epi16(low, 1); | |
| 1237 | 12 | high = _mm_mulhi_epu16(_mm_slli_epi16(high, 1), div_vector); | |
| 1238 | 6 | high = _mm_adds_epu16(high, halfdiv_vector); | |
| 1239 | 6 | high = _mm_srli_epi16(high, 1); | |
| 1240 | 6 | __m128i acc = _mm_packus_epi16(low, high); | |
| 1241 | |||
| 1242 | 6 | _mm_store_si128(reinterpret_cast<__m128i*>(c_plane+x), acc); | |
| 1243 | } | ||
| 1244 | 2 | } | |
| 1245 | |||
| 1246 | // instantiate | ||
| 1247 | template void accumulate_line_sse2<false>(BYTE* c_plane, const BYTE** planeP, int planes, size_t width, int threshold, int div); | ||
| 1248 | template void accumulate_line_sse2<true>(BYTE* c_plane, const BYTE** planeP, int planes, size_t width, int threshold, int div); | ||
| 1249 | |||
| 1250 | |||
| 1251 | template<bool maxThreshold> | ||
| 1252 | #if defined(GCC) || defined(CLANG) | ||
| 1253 | __attribute__((__target__("ssse3"))) | ||
| 1254 | #endif | ||
| 1255 | 13 | void accumulate_line_ssse3(BYTE* c_plane, const BYTE** planeP, int planes, size_t width, int threshold, int div) | |
| 1256 | { | ||
| 1257 | // threshold: 8 bits: 2 bytes in a word. for YUY2: luma<<8 | chroma | ||
| 1258 | // 16 bits: 16 bit value (orig threshold scaled by bits_per_pixel) | ||
| 1259 | 13 | __m128i div_vector = _mm_set1_epi16(div); | |
| 1260 | |||
| 1261 | 13 | __m128i thresh = _mm_set1_epi16(threshold); | |
| 1262 | |||
| 1263 |
4/4void accumulate_line_ssse3<false>(unsigned char*, unsigned char const**, int, unsigned long, int, int):
✓ Branch 54 → 11 taken 47 times.
✓ Branch 54 → 55 taken 12 times.
void accumulate_line_ssse3<true>(unsigned char*, unsigned char const**, int, unsigned long, int, int):
✓ Branch 42 → 11 taken 3 times.
✓ Branch 42 → 43 taken 1 time.
|
63 | for (size_t x = 0; x < width; x += 16) { |
| 1264 | 100 | __m128i current = _mm_load_si128(reinterpret_cast<const __m128i*>(c_plane + x)); | |
| 1265 | 50 | __m128i zero = _mm_setzero_si128(); | |
| 1266 | 50 | __m128i low = _mm_unpacklo_epi8(current, zero); | |
| 1267 | 50 | __m128i high = _mm_unpackhi_epi8(current, zero); | |
| 1268 | |||
| 1269 |
4/4void accumulate_line_ssse3<false>(unsigned char*, unsigned char const**, int, unsigned long, int, int):
✓ Branch 45 → 20 taken 94 times.
✓ Branch 45 → 46 taken 47 times.
void accumulate_line_ssse3<true>(unsigned char*, unsigned char const**, int, unsigned long, int, int):
✓ Branch 33 → 20 taken 9 times.
✓ Branch 33 → 34 taken 3 times.
|
153 | for (int plane = planes - 1; plane >= 0; --plane) { |
| 1270 | 206 | __m128i p = _mm_load_si128(reinterpret_cast<const __m128i*>(planeP[plane] + x)); | |
| 1271 | |||
| 1272 | __m128i add_low, add_high; | ||
| 1273 | if (maxThreshold) { | ||
| 1274 | // fast: simple accumulate for average | ||
| 1275 | 9 | add_low = _mm_unpacklo_epi8(p, zero); | |
| 1276 | 9 | add_high = _mm_unpackhi_epi8(p, zero); | |
| 1277 | } | ||
| 1278 | else { | ||
| 1279 | 94 | auto pc = _mm_subs_epu8(p, current); // r2507- | |
| 1280 | 94 | auto cp = _mm_subs_epu8(current, p); | |
| 1281 | 94 | auto abs_cp = _mm_or_si128(pc, cp); | |
| 1282 | 94 | auto leq_thresh = _mm_cmple_epu8(abs_cp, thresh); | |
| 1283 | |||
| 1284 | 94 | __m128i andop = _mm_and_si128(leq_thresh, p); | |
| 1285 | 94 | __m128i andnop = _mm_andnot_si128(leq_thresh, current); | |
| 1286 | 94 | __m128i blended = _mm_or_si128(andop, andnop); //abs(p-c) <= thresh ? p : c | |
| 1287 | 94 | add_low = _mm_unpacklo_epi8(blended, zero); | |
| 1288 | 94 | add_high = _mm_unpackhi_epi8(blended, zero); | |
| 1289 | } | ||
| 1290 | |||
| 1291 | 103 | low = _mm_adds_epu16(low, add_low); | |
| 1292 | 103 | high = _mm_adds_epu16(high, add_high); | |
| 1293 | } | ||
| 1294 | |||
| 1295 | // SSSE3: _mm_mulhrs_epi16: r0 := INT16(((a0 * b0) + 0x4000) >> 15) | ||
| 1296 | 50 | low = _mm_mulhrs_epi16(low, div_vector); | |
| 1297 | 50 | high = _mm_mulhrs_epi16(high, div_vector); | |
| 1298 | 50 | __m128i acc = _mm_packus_epi16(low, high); | |
| 1299 | |||
| 1300 | 50 | _mm_store_si128(reinterpret_cast<__m128i*>(c_plane + x), acc); | |
| 1301 | } | ||
| 1302 | 13 | } | |
| 1303 | |||
| 1304 | // instantiate | ||
| 1305 | template void accumulate_line_ssse3<false>(BYTE* c_plane, const BYTE** planeP, int planes, size_t width, int threshold, int div); | ||
| 1306 | template void accumulate_line_ssse3<true>(BYTE* c_plane, const BYTE** planeP, int planes, size_t width, int threshold, int div); | ||
| 1307 | |||
| 1308 | |||
| 1309 | // fast: maxThreshold (255) simple accumulate for average | ||
| 1310 | template<bool maxThreshold, bool lessThan16bit> | ||
| 1311 | 4 | void accumulate_line_16_sse2(BYTE* c_plane, const BYTE** planeP, int planes, size_t rowsize, int threshold, int div, int bits_per_pixel) { | |
| 1312 | // threshold: | ||
| 1313 | // 10-16 bits: orig threshold scaled by (bits_per_pixel-8) | ||
| 1314 | 4 | int max_pixel_value = (1 << bits_per_pixel) - 1; | |
| 1315 | 4 | __m128i limit = _mm_set1_epi16(max_pixel_value); //used for clamping when 10-14 bits | |
| 1316 | 4 | __m128 div_vector = _mm_set1_ps(1.0f / (planes + 1)); | |
| 1317 | 4 | __m128i thresh = _mm_set1_epi16(threshold); | |
| 1318 | |||
| 1319 | |||
| 1320 |
8/8void accumulate_line_16_sse2<false, false>(unsigned char*, unsigned char const**, int, unsigned long, int, int, int):
✓ Branch 81 → 13 taken 6 times.
✓ Branch 81 → 82 taken 1 time.
void accumulate_line_16_sse2<false, true>(unsigned char*, unsigned char const**, int, unsigned long, int, int, int):
✓ Branch 98 → 13 taken 6 times.
✓ Branch 98 → 99 taken 1 time.
void accumulate_line_16_sse2<true, false>(unsigned char*, unsigned char const**, int, unsigned long, int, int, int):
✓ Branch 69 → 13 taken 6 times.
✓ Branch 69 → 70 taken 1 time.
void accumulate_line_16_sse2<true, true>(unsigned char*, unsigned char const**, int, unsigned long, int, int, int):
✓ Branch 86 → 13 taken 6 times.
✓ Branch 86 → 87 taken 1 time.
|
28 | for (size_t x = 0; x < rowsize; x+=16) { |
| 1321 | 48 | __m128i current = _mm_load_si128(reinterpret_cast<const __m128i*>(c_plane+x)); | |
| 1322 | 24 | __m128i zero = _mm_setzero_si128(); | |
| 1323 | __m128i low, high; | ||
| 1324 | 24 | low = _mm_unpacklo_epi16(current, zero); | |
| 1325 | 24 | high = _mm_unpackhi_epi16(current, zero); | |
| 1326 | |||
| 1327 |
8/8void accumulate_line_16_sse2<false, false>(unsigned char*, unsigned char const**, int, unsigned long, int, int, int):
✓ Branch 47 → 22 taken 12 times.
✓ Branch 47 → 48 taken 6 times.
void accumulate_line_16_sse2<false, true>(unsigned char*, unsigned char const**, int, unsigned long, int, int, int):
✓ Branch 47 → 22 taken 12 times.
✓ Branch 47 → 48 taken 6 times.
void accumulate_line_16_sse2<true, false>(unsigned char*, unsigned char const**, int, unsigned long, int, int, int):
✓ Branch 35 → 22 taken 12 times.
✓ Branch 35 → 36 taken 6 times.
void accumulate_line_16_sse2<true, true>(unsigned char*, unsigned char const**, int, unsigned long, int, int, int):
✓ Branch 35 → 22 taken 18 times.
✓ Branch 35 → 36 taken 6 times.
|
78 | for (int plane = planes - 1; plane >= 0; --plane) { |
| 1328 | 108 | __m128i p = _mm_load_si128(reinterpret_cast<const __m128i*>(planeP[plane] + x)); | |
| 1329 | |||
| 1330 | __m128i add_low, add_high; | ||
| 1331 | if (maxThreshold) { | ||
| 1332 | // fast: simple accumulate for average | ||
| 1333 | 30 | add_low = _mm_unpacklo_epi16(p, zero); | |
| 1334 | 30 | add_high = _mm_unpackhi_epi16(p, zero); | |
| 1335 | } else { | ||
| 1336 | 24 | auto pc = _mm_subs_epu16(p, current); // r2507- | |
| 1337 | 24 | auto cp = _mm_subs_epu16(current, p); | |
| 1338 | 24 | auto abs_cp = _mm_or_si128(pc, cp); | |
| 1339 | 24 | auto leq_thresh = _mm_cmple_epu16_sse2(abs_cp, thresh); | |
| 1340 | /* | ||
| 1341 | __m128i p_greater_t = _mm_subs_epu16(p, thresh); | ||
| 1342 | __m128i c_greater_t = _mm_subs_epu16(current, thresh); | ||
| 1343 | __m128i over_thresh = _mm_or_si128(p_greater_t, c_greater_t); //abs(p-c) - t == (satsub(p,c) | satsub(c,p)) - t =kinda= satsub(p,t) | satsub(c,t) | ||
| 1344 | |||
| 1345 | __m128i leq_thresh = _mm_cmpeq_epi16(over_thresh, zero); //abs diff lower or equal to threshold | ||
| 1346 | */ | ||
| 1347 | |||
| 1348 | 24 | __m128i andop = _mm_and_si128(leq_thresh, p); | |
| 1349 | 24 | __m128i andnop = _mm_andnot_si128(leq_thresh, current); | |
| 1350 | 24 | __m128i blended = _mm_or_si128(andop, andnop); //abs(p-c) <= thresh ? p : c | |
| 1351 | |||
| 1352 | 24 | add_low = _mm_unpacklo_epi16(blended, zero); | |
| 1353 | 24 | add_high = _mm_unpackhi_epi16(blended, zero); | |
| 1354 | } | ||
| 1355 | 54 | low = _mm_add_epi32(low, add_low); | |
| 1356 | 54 | high = _mm_add_epi32(high, add_high); | |
| 1357 | } | ||
| 1358 | |||
| 1359 | __m128i acc; | ||
| 1360 | //__m128 half = _mm_set1_ps(0.5f); // no need rounder, _mm_cvtps_epi32 default is round-to-nearest, unless we use _mm_cvttps_epi32 which truncates | ||
| 1361 | 72 | low = _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(low), div_vector)); | |
| 1362 | 72 | high = _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(high), div_vector)); | |
| 1363 | 24 | acc = _MM_PACKUS_EPI32(low, high); // sse4.1 simul | |
| 1364 | if (lessThan16bit) | ||
| 1365 | 12 | acc = _MM_MIN_EPU16(acc, limit); // sse4.1 simul | |
| 1366 | |||
| 1367 | 24 | _mm_store_si128(reinterpret_cast<__m128i*>(c_plane+x), acc); | |
| 1368 | } | ||
| 1369 | 4 | } | |
| 1370 | |||
| 1371 | |||
| 1372 | // instantiate | ||
| 1373 | template void accumulate_line_16_sse2<false, false>(BYTE* c_plane, const BYTE** planeP, int planes, size_t rowsize, int threshold, int div, int bits_per_pixel); | ||
| 1374 | template void accumulate_line_16_sse2<false, true>(BYTE* c_plane, const BYTE** planeP, int planes, size_t rowsize, int threshold, int div, int bits_per_pixel); | ||
| 1375 | template void accumulate_line_16_sse2<true, false>(BYTE* c_plane, const BYTE** planeP, int planes, size_t rowsize, int threshold, int div, int bits_per_pixel); | ||
| 1376 | template void accumulate_line_16_sse2<true, true>(BYTE* c_plane, const BYTE** planeP, int planes, size_t rowsize, int threshold, int div, int bits_per_pixel); | ||
| 1377 | |||
| 1378 | |||
| 1379 | // fast: maxThreshold (255) simple accumulate for average | ||
| 1380 | template<bool maxThreshold, bool lessThan16bit> | ||
| 1381 | #if defined(GCC) || defined(CLANG) | ||
| 1382 | __attribute__((__target__("sse4.1"))) | ||
| 1383 | #endif | ||
| 1384 | 19 | void accumulate_line_16_sse41(BYTE* c_plane, const BYTE** planeP, int planes, size_t rowsize, int threshold, int div, int bits_per_pixel) | |
| 1385 | { | ||
| 1386 | // threshold: | ||
| 1387 | // 10-16 bits: orig threshold scaled by (bits_per_pixel-8) | ||
| 1388 | 19 | int max_pixel_value = (1 << bits_per_pixel) - 1; | |
| 1389 | 19 | __m128i limit = _mm_set1_epi16(max_pixel_value); //used for clamping when 10-14 bits | |
| 1390 | // halfdiv_vector = _mm_set1_epi32(1); // n/a | ||
| 1391 | 19 | __m128 div_vector = _mm_set1_ps(1.0f / (planes + 1)); | |
| 1392 | 19 | __m128i thresh = _mm_set1_epi16(threshold); | |
| 1393 | |||
| 1394 | |||
| 1395 |
8/8void accumulate_line_16_sse41<false, false>(unsigned char*, unsigned char const**, int, unsigned long, int, int, int):
✓ Branch 65 → 13 taken 66 times.
✓ Branch 65 → 66 taken 16 times.
void accumulate_line_16_sse41<false, true>(unsigned char*, unsigned char const**, int, unsigned long, int, int, int):
✓ Branch 66 → 13 taken 6 times.
✓ Branch 66 → 67 taken 1 time.
void accumulate_line_16_sse41<true, false>(unsigned char*, unsigned char const**, int, unsigned long, int, int, int):
✓ Branch 53 → 13 taken 6 times.
✓ Branch 53 → 54 taken 1 time.
void accumulate_line_16_sse41<true, true>(unsigned char*, unsigned char const**, int, unsigned long, int, int, int):
✓ Branch 54 → 13 taken 6 times.
✓ Branch 54 → 55 taken 1 time.
|
103 | for (size_t x = 0; x < rowsize; x += 16) { |
| 1396 | 168 | __m128i current = _mm_load_si128(reinterpret_cast<const __m128i*>(c_plane + x)); | |
| 1397 | 84 | __m128i zero = _mm_setzero_si128(); | |
| 1398 | __m128i low, high; | ||
| 1399 | 84 | low = _mm_unpacklo_epi16(current, zero); | |
| 1400 | 84 | high = _mm_unpackhi_epi16(current, zero); | |
| 1401 | |||
| 1402 |
8/8void accumulate_line_16_sse41<false, false>(unsigned char*, unsigned char const**, int, unsigned long, int, int, int):
✓ Branch 47 → 22 taken 132 times.
✓ Branch 47 → 48 taken 66 times.
void accumulate_line_16_sse41<false, true>(unsigned char*, unsigned char const**, int, unsigned long, int, int, int):
✓ Branch 47 → 22 taken 12 times.
✓ Branch 47 → 48 taken 6 times.
void accumulate_line_16_sse41<true, false>(unsigned char*, unsigned char const**, int, unsigned long, int, int, int):
✓ Branch 35 → 22 taken 12 times.
✓ Branch 35 → 36 taken 6 times.
void accumulate_line_16_sse41<true, true>(unsigned char*, unsigned char const**, int, unsigned long, int, int, int):
✓ Branch 35 → 22 taken 18 times.
✓ Branch 35 → 36 taken 6 times.
|
258 | for (int plane = planes - 1; plane >= 0; --plane) { |
| 1403 | 348 | __m128i p = _mm_load_si128(reinterpret_cast<const __m128i*>(planeP[plane] + x)); | |
| 1404 | |||
| 1405 | __m128i add_low, add_high; | ||
| 1406 | if (maxThreshold) { | ||
| 1407 | // fast: simple accumulate for average | ||
| 1408 | 30 | add_low = _mm_unpacklo_epi16(p, zero); | |
| 1409 | 30 | add_high = _mm_unpackhi_epi16(p, zero); | |
| 1410 | } | ||
| 1411 | else { | ||
| 1412 | 144 | auto pc = _mm_subs_epu16(p, current); // r2507- | |
| 1413 | 144 | auto cp = _mm_subs_epu16(current, p); | |
| 1414 | 144 | auto abs_cp = _mm_or_si128(pc, cp); | |
| 1415 | 144 | auto leq_thresh = _mm_cmple_epu16_sse41(abs_cp, thresh); | |
| 1416 | /* | ||
| 1417 | __m128i p_greater_t = _mm_subs_epu16(p, thresh); | ||
| 1418 | __m128i c_greater_t = _mm_subs_epu16(current, thresh); | ||
| 1419 | __m128i over_thresh = _mm_or_si128(p_greater_t, c_greater_t); //abs(p-c) - t == (satsub(p,c) | satsub(c,p)) - t =kinda= satsub(p,t) | satsub(c,t) | ||
| 1420 | |||
| 1421 | __m128i leq_thresh = _mm_cmpeq_epi16(over_thresh, zero); //abs diff lower or equal to threshold | ||
| 1422 | */ | ||
| 1423 | |||
| 1424 | 144 | __m128i andop = _mm_and_si128(leq_thresh, p); | |
| 1425 | 144 | __m128i andnop = _mm_andnot_si128(leq_thresh, current); | |
| 1426 | 144 | __m128i blended = _mm_or_si128(andop, andnop); //abs(p-c) <= thresh ? p : c | |
| 1427 | |||
| 1428 | 144 | add_low = _mm_unpacklo_epi16(blended, zero); | |
| 1429 | 144 | add_high = _mm_unpackhi_epi16(blended, zero); | |
| 1430 | } | ||
| 1431 | 174 | low = _mm_add_epi32(low, add_low); | |
| 1432 | 174 | high = _mm_add_epi32(high, add_high); | |
| 1433 | } | ||
| 1434 | |||
| 1435 | __m128i acc; | ||
| 1436 | //__m128 half = _mm_set1_ps(0.5f); // no need rounder, _mm_cvtps_epi32 default is round-to-nearest, unless we use _mm_cvttps_epi32 which truncates | ||
| 1437 | 252 | low = _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(low), div_vector)); | |
| 1438 | 252 | high = _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(high), div_vector)); | |
| 1439 | 84 | acc = _mm_packus_epi32(low, high); // sse41 | |
| 1440 | if (lessThan16bit) | ||
| 1441 | 12 | acc = _mm_min_epu16(acc, limit); // sse41 | |
| 1442 | |||
| 1443 | 84 | _mm_store_si128(reinterpret_cast<__m128i*>(c_plane + x), acc); | |
| 1444 | } | ||
| 1445 | 19 | } | |
| 1446 | |||
| 1447 | // instantiate | ||
| 1448 | // fast: maxThreshold (255) simple accumulate for average | ||
| 1449 | template void accumulate_line_16_sse41<false, false>(BYTE* c_plane, const BYTE** planeP, int planes, size_t rowsize, int threshold, int div, int bits_per_pixel); | ||
| 1450 | template void accumulate_line_16_sse41<false, true>(BYTE* c_plane, const BYTE** planeP, int planes, size_t rowsize, int threshold, int div, int bits_per_pixel); | ||
| 1451 | template void accumulate_line_16_sse41<true, false>(BYTE* c_plane, const BYTE** planeP, int planes, size_t rowsize, int threshold, int div, int bits_per_pixel); | ||
| 1452 | template void accumulate_line_16_sse41<true, true>(BYTE* c_plane, const BYTE** planeP, int planes, size_t rowsize, int threshold, int div, int bits_per_pixel); | ||
| 1453 | |||
| 1454 | |||
| 1455 | #ifdef X86_32 | ||
| 1456 | |||
| 1457 | static AVS_FORCEINLINE __m64 ts_multiply_repack_mmx(const __m64 &src, const __m64 &div, __m64 &halfdiv, __m64 &zero) { | ||
| 1458 | __m64 acc = _mm_madd_pi16(src, div); | ||
| 1459 | acc = _mm_add_pi32(acc, halfdiv); | ||
| 1460 | acc = _mm_srli_pi32(acc, 15); | ||
| 1461 | acc = _mm_packs_pi32(acc, acc); | ||
| 1462 | return _mm_packs_pu16(acc, zero); | ||
| 1463 | } | ||
| 1464 | |||
| 1465 | //thresh and div must always be 16-bit integers. Thresh is 2 packed bytes and div is a single 16-bit number | ||
| 1466 | void accumulate_line_mmx(BYTE* c_plane, const BYTE** planeP, int planes, size_t width, int threshold, int div) { | ||
| 1467 | __m64 halfdiv_vector = _mm_set1_pi32(16384); | ||
| 1468 | __m64 div_vector = _mm_set1_pi16(div); | ||
| 1469 | |||
| 1470 | for (size_t x = 0; x < width; x+=8) { | ||
| 1471 | __m64 current = *reinterpret_cast<const __m64*>(c_plane+x); | ||
| 1472 | __m64 zero = _mm_setzero_si64(); | ||
| 1473 | __m64 low = _mm_unpacklo_pi8(current, zero); | ||
| 1474 | __m64 high = _mm_unpackhi_pi8(current, zero); | ||
| 1475 | __m64 thresh = _mm_set1_pi16(threshold); | ||
| 1476 | |||
| 1477 | for(int plane = planes-1; plane >= 0; --plane) { | ||
| 1478 | __m64 p = *reinterpret_cast<const __m64*>(planeP[plane]+x); | ||
| 1479 | |||
| 1480 | __m64 p_greater_t = _mm_subs_pu8(p, thresh); | ||
| 1481 | __m64 c_greater_t = _mm_subs_pu8(current, thresh); | ||
| 1482 | __m64 over_thresh = _mm_or_si64(p_greater_t, c_greater_t); //abs(p-c) - t == (satsub(p,c) | satsub(c,p)) - t =kinda= satsub(p,t) | satsub(c,t) | ||
| 1483 | |||
| 1484 | __m64 leq_thresh = _mm_cmpeq_pi8(over_thresh, zero); //abs diff lower or equal to threshold | ||
| 1485 | |||
| 1486 | __m64 andop = _mm_and_si64(leq_thresh, p); | ||
| 1487 | __m64 andnop = _mm_andnot_si64(leq_thresh, current); | ||
| 1488 | __m64 blended = _mm_or_si64(andop, andnop); //abs(p-c) <= thresh ? p : c | ||
| 1489 | |||
| 1490 | __m64 add_low = _mm_unpacklo_pi8(blended, zero); | ||
| 1491 | __m64 add_high = _mm_unpackhi_pi8(blended, zero); | ||
| 1492 | |||
| 1493 | low = _mm_adds_pu16(low, add_low); | ||
| 1494 | high = _mm_adds_pu16(high, add_high); | ||
| 1495 | } | ||
| 1496 | |||
| 1497 | __m64 low_low = ts_multiply_repack_mmx(_mm_unpacklo_pi16(low, zero), div_vector, halfdiv_vector, zero); | ||
| 1498 | __m64 low_high = ts_multiply_repack_mmx(_mm_unpackhi_pi16(low, zero), div_vector, halfdiv_vector, zero); | ||
| 1499 | __m64 high_low = ts_multiply_repack_mmx(_mm_unpacklo_pi16(high, zero), div_vector, halfdiv_vector, zero); | ||
| 1500 | __m64 high_high = ts_multiply_repack_mmx(_mm_unpackhi_pi16(high, zero), div_vector, halfdiv_vector, zero); | ||
| 1501 | |||
| 1502 | low = _mm_unpacklo_pi16(low_low, low_high); | ||
| 1503 | high = _mm_unpacklo_pi16(high_low, high_high); | ||
| 1504 | |||
| 1505 | __m64 acc = _mm_unpacklo_pi32(low, high); | ||
| 1506 | |||
| 1507 | *reinterpret_cast<__m64*>(c_plane+x) = acc; | ||
| 1508 | } | ||
| 1509 | _mm_empty(); | ||
| 1510 | } | ||
| 1511 | |||
| 1512 | #endif | ||
| 1513 | |||
| 1514 | |||
| 1515 | |||
| 1516 | // may also used from conditionalfunctions | ||
| 1517 | // packed rgb template masks out alpha plane for RGB32 | ||
| 1518 | template<bool packedRGB3264> | ||
| 1519 | 4 | int calculate_sad_sse2(const BYTE* cur_ptr, const BYTE* other_ptr, int cur_pitch, int other_pitch, size_t rowsize, size_t height) | |
| 1520 | { | ||
| 1521 | 4 | size_t mod16_width = rowsize / 16 * 16; | |
| 1522 | 4 | int result = 0; | |
| 1523 | 4 | __m128i sum = _mm_setzero_si128(); | |
| 1524 | |||
| 1525 | __m128i rgb_mask; | ||
| 1526 | if (packedRGB3264) { | ||
| 1527 | 2 | rgb_mask = _mm_set1_epi32(0x00FFFFFF); | |
| 1528 | } | ||
| 1529 | |||
| 1530 |
4/4int calculate_sad_sse2<false>(unsigned char const*, unsigned char const*, int, int, unsigned long, unsigned long):
✓ Branch 23 → 6 taken 12 times.
✓ Branch 23 → 24 taken 2 times.
int calculate_sad_sse2<true>(unsigned char const*, unsigned char const*, int, int, unsigned long, unsigned long):
✓ Branch 29 → 9 taken 12 times.
✓ Branch 29 → 30 taken 2 times.
|
28 | for (size_t y = 0; y < height; ++y) { |
| 1531 |
4/4int calculate_sad_sse2<false>(unsigned char const*, unsigned char const*, int, int, unsigned long, unsigned long):
✓ Branch 17 → 7 taken 31 times.
✓ Branch 17 → 18 taken 12 times.
int calculate_sad_sse2<true>(unsigned char const*, unsigned char const*, int, int, unsigned long, unsigned long):
✓ Branch 23 → 10 taken 24 times.
✓ Branch 23 → 24 taken 12 times.
|
79 | for (size_t x = 0; x < mod16_width; x+=16) { |
| 1532 | 55 | __m128i cur = _mm_load_si128(reinterpret_cast<const __m128i*>(cur_ptr + x)); | |
| 1533 | 110 | __m128i other = _mm_load_si128(reinterpret_cast<const __m128i*>(other_ptr + x)); | |
| 1534 | if (packedRGB3264) { | ||
| 1535 | 24 | cur = _mm_and_si128(cur, rgb_mask); // mask out A channel | |
| 1536 | 24 | other = _mm_and_si128(other, rgb_mask); | |
| 1537 | } | ||
| 1538 | 55 | __m128i sad = _mm_sad_epu8(cur, other); | |
| 1539 | 55 | sum = _mm_add_epi32(sum, sad); | |
| 1540 | } | ||
| 1541 |
3/4int calculate_sad_sse2<false>(unsigned char const*, unsigned char const*, int, int, unsigned long, unsigned long):
✓ Branch 18 → 19 taken 12 times.
✗ Branch 18 → 22 not taken.
int calculate_sad_sse2<true>(unsigned char const*, unsigned char const*, int, int, unsigned long, unsigned long):
✓ Branch 24 → 25 taken 7 times.
✓ Branch 24 → 28 taken 5 times.
|
24 | if (mod16_width != rowsize) { |
| 1542 | if (packedRGB3264) | ||
| 1543 |
2/2✓ Branch 27 → 26 taken 7 times.
✓ Branch 27 → 28 taken 7 times.
|
14 | for (size_t x = mod16_width / 4; x < rowsize / 4; x += 4) { |
| 1544 | 7 | result += std::abs(cur_ptr[x*4+0] - other_ptr[x*4+0]) + | |
| 1545 | 7 | std::abs(cur_ptr[x*4+1] - other_ptr[x*4+1]) + | |
| 1546 | 7 | std::abs(cur_ptr[x*4+2] - other_ptr[x*4+2]); | |
| 1547 | // no alpha | ||
| 1548 | } | ||
| 1549 | else | ||
| 1550 |
2/2✓ Branch 21 → 20 taken 60 times.
✓ Branch 21 → 22 taken 12 times.
|
72 | for (size_t x = mod16_width; x < rowsize; ++x) { |
| 1551 | 60 | result += std::abs(cur_ptr[x] - other_ptr[x]); | |
| 1552 | } | ||
| 1553 | } | ||
| 1554 | 24 | cur_ptr += cur_pitch; | |
| 1555 | 24 | other_ptr += other_pitch; | |
| 1556 | } | ||
| 1557 | 16 | __m128i upper = _mm_castps_si128(_mm_movehl_ps(_mm_setzero_ps(), _mm_castsi128_ps(sum))); | |
| 1558 | 4 | sum = _mm_add_epi32(sum, upper); | |
| 1559 | 4 | result += _mm_cvtsi128_si32(sum); | |
| 1560 | 4 | return result; | |
| 1561 | } | ||
| 1562 | |||
| 1563 | // instantiate | ||
| 1564 | template int calculate_sad_sse2<false>(const BYTE* cur_ptr, const BYTE* other_ptr, int cur_pitch, int other_pitch, size_t rowsize, size_t height); | ||
| 1565 | template int calculate_sad_sse2<true>(const BYTE* cur_ptr, const BYTE* other_ptr, int cur_pitch, int other_pitch, size_t rowsize, size_t height); | ||
| 1566 | |||
| 1567 | |||
| 1568 | // works for uint8_t, but there is a specific, bit faster function above | ||
| 1569 | // also used from conditionalfunctions | ||
| 1570 | // packed rgb template masks out alpha plane for RGB32/RGB64 | ||
| 1571 | template<typename pixel_t, bool packedRGB3264> | ||
| 1572 | 6 | int64_t calculate_sad_8_or_16_sse2(const BYTE* cur_ptr, const BYTE* other_ptr, int cur_pitch, int other_pitch, size_t rowsize, size_t height) | |
| 1573 | { | ||
| 1574 | 6 | size_t mod16_width = rowsize / 16 * 16; | |
| 1575 | |||
| 1576 | 6 | __m128i zero = _mm_setzero_si128(); | |
| 1577 | 6 | int64_t totalsum = 0; // fullframe SAD exceeds int32 at 8+ bit | |
| 1578 | |||
| 1579 | __m128i rgb_mask; | ||
| 1580 | if (packedRGB3264) { | ||
| 1581 | if constexpr(sizeof(pixel_t) == 1) | ||
| 1582 | ✗ | rgb_mask = _mm_set1_epi32(0x00FFFFFF); | |
| 1583 | else | ||
| 1584 | 2 | rgb_mask = _mm_set_epi32(0x0000FFFF,0xFFFFFFFF,0x0000FFFF,0xFFFFFFFF); | |
| 1585 | } | ||
| 1586 | |||
| 1587 |
6/8long calculate_sad_8_or_16_sse2<unsigned char, false>(unsigned char const*, unsigned char const*, int, int, unsigned long, unsigned long):
✓ Branch 31 → 6 taken 12 times.
✓ Branch 31 → 32 taken 2 times.
long calculate_sad_8_or_16_sse2<unsigned char, true>(unsigned char const*, unsigned char const*, int, int, unsigned long, unsigned long):
✗ Branch 37 → 9 not taken.
✗ Branch 37 → 38 not taken.
long calculate_sad_8_or_16_sse2<unsigned short, false>(unsigned char const*, unsigned char const*, int, int, unsigned long, unsigned long):
✓ Branch 47 → 6 taken 12 times.
✓ Branch 47 → 48 taken 2 times.
long calculate_sad_8_or_16_sse2<unsigned short, true>(unsigned char const*, unsigned char const*, int, int, unsigned long, unsigned long):
✓ Branch 51 → 7 taken 12 times.
✓ Branch 51 → 52 taken 2 times.
|
42 | for ( size_t y = 0; y < height; y++ ) |
| 1588 | { | ||
| 1589 | 36 | __m128i sum = _mm_setzero_si128(); // for one row int is enough | |
| 1590 |
6/8long calculate_sad_8_or_16_sse2<unsigned char, false>(unsigned char const*, unsigned char const*, int, int, unsigned long, unsigned long):
✓ Branch 19 → 9 taken 31 times.
✓ Branch 19 → 20 taken 12 times.
long calculate_sad_8_or_16_sse2<unsigned char, true>(unsigned char const*, unsigned char const*, int, int, unsigned long, unsigned long):
✗ Branch 25 → 12 not taken.
✗ Branch 25 → 26 not taken.
long calculate_sad_8_or_16_sse2<unsigned short, false>(unsigned char const*, unsigned char const*, int, int, unsigned long, unsigned long):
✓ Branch 29 → 9 taken 62 times.
✓ Branch 29 → 30 taken 12 times.
long calculate_sad_8_or_16_sse2<unsigned short, true>(unsigned char const*, unsigned char const*, int, int, unsigned long, unsigned long):
✓ Branch 33 → 10 taken 48 times.
✓ Branch 33 → 34 taken 12 times.
|
177 | for ( size_t x = 0; x < mod16_width; x+=16 ) |
| 1591 | { | ||
| 1592 | __m128i src1, src2; | ||
| 1593 | 141 | src1 = _mm_load_si128((__m128i *) (cur_ptr + x)); // 16 bytes or 8 words | |
| 1594 | 282 | src2 = _mm_load_si128((__m128i *) (other_ptr + x)); | |
| 1595 | if (packedRGB3264) { | ||
| 1596 | 48 | src1 = _mm_and_si128(src1, rgb_mask); // mask out A channel | |
| 1597 | 48 | src2 = _mm_and_si128(src2, rgb_mask); | |
| 1598 | } | ||
| 1599 | if constexpr(sizeof(pixel_t) == 1) { | ||
| 1600 | // this is uint_16 specific, but leave here for sample | ||
| 1601 | 62 | sum = _mm_add_epi32(sum, _mm_sad_epu8(src1, src2)); // sum0_32, 0, sum1_32, 0 | |
| 1602 | } | ||
| 1603 | else if constexpr(sizeof(pixel_t) == 2) { | ||
| 1604 | 110 | __m128i greater_t = _mm_subs_epu16(src1, src2); // unsigned sub with saturation | |
| 1605 | 110 | __m128i smaller_t = _mm_subs_epu16(src2, src1); | |
| 1606 | 110 | __m128i absdiff = _mm_or_si128(greater_t, smaller_t); //abs(s1-s2) == (satsub(s1,s2) | satsub(s2,s1)) | |
| 1607 | // 8 x uint16 absolute differences | ||
| 1608 | 220 | sum = _mm_add_epi32(sum, _mm_unpacklo_epi16(absdiff, zero)); | |
| 1609 | 220 | sum = _mm_add_epi32(sum, _mm_unpackhi_epi16(absdiff, zero)); | |
| 1610 | // sum0_32, sum1_32, sum2_32, sum3_32 | ||
| 1611 | } | ||
| 1612 | } | ||
| 1613 | // summing up partial sums | ||
| 1614 | if constexpr(sizeof(pixel_t) == 2) { | ||
| 1615 | // at 16 bits: we have 4 integers for sum: a0 a1 a2 a3 | ||
| 1616 | 24 | __m128i a0_a1 = _mm_unpacklo_epi32(sum, zero); // a0 0 a1 0 | |
| 1617 | 24 | __m128i a2_a3 = _mm_unpackhi_epi32(sum, zero); // a2 0 a3 0 | |
| 1618 | 24 | sum = _mm_add_epi32( a0_a1, a2_a3 ); // a0+a2, 0, a1+a3, 0 | |
| 1619 | /* SSSE3: told to be not too fast | ||
| 1620 | sum = _mm_hadd_epi32(sum, zero); // A1+A2, B1+B2, 0+0, 0+0 | ||
| 1621 | sum = _mm_hadd_epi32(sum, zero); // A1+A2+B1+B2, 0+0+0+0, 0+0+0+0, 0+0+0+0 | ||
| 1622 | */ | ||
| 1623 | } | ||
| 1624 | |||
| 1625 | // sum here: two 32 bit partial result: sum1 0 sum2 0 | ||
| 1626 | 36 | __m128i sum_hi = _mm_unpackhi_epi64(sum, zero); | |
| 1627 | // or: __m128i sum_hi = _mm_castps_si128(_mm_movehl_ps(_mm_setzero_ps(), _mm_castsi128_ps(sum))); | ||
| 1628 | 36 | sum = _mm_add_epi32(sum, sum_hi); | |
| 1629 | 36 | int rowsum = _mm_cvtsi128_si32(sum); | |
| 1630 | |||
| 1631 | // rest | ||
| 1632 |
4/8long calculate_sad_8_or_16_sse2<unsigned char, false>(unsigned char const*, unsigned char const*, int, int, unsigned long, unsigned long):
✓ Branch 26 → 27 taken 12 times.
✗ Branch 26 → 30 not taken.
long calculate_sad_8_or_16_sse2<unsigned char, true>(unsigned char const*, unsigned char const*, int, int, unsigned long, unsigned long):
✗ Branch 32 → 33 not taken.
✗ Branch 32 → 36 not taken.
long calculate_sad_8_or_16_sse2<unsigned short, false>(unsigned char const*, unsigned char const*, int, int, unsigned long, unsigned long):
✓ Branch 42 → 43 taken 12 times.
✗ Branch 42 → 46 not taken.
long calculate_sad_8_or_16_sse2<unsigned short, true>(unsigned char const*, unsigned char const*, int, int, unsigned long, unsigned long):
✓ Branch 46 → 47 taken 7 times.
✓ Branch 46 → 50 taken 5 times.
|
36 | if (mod16_width != rowsize) { |
| 1633 | if (packedRGB3264) | ||
| 1634 |
2/4long calculate_sad_8_or_16_sse2<unsigned char, true>(unsigned char const*, unsigned char const*, int, int, unsigned long, unsigned long):
✗ Branch 35 → 34 not taken.
✗ Branch 35 → 36 not taken.
long calculate_sad_8_or_16_sse2<unsigned short, true>(unsigned char const*, unsigned char const*, int, int, unsigned long, unsigned long):
✓ Branch 49 → 48 taken 7 times.
✓ Branch 49 → 50 taken 7 times.
|
14 | for (size_t x = mod16_width / sizeof(pixel_t) / 4; x < rowsize / sizeof(pixel_t) / 4; x += 4) { |
| 1635 | 7 | rowsum += std::abs(reinterpret_cast<const pixel_t *>(cur_ptr)[x*4+0] - reinterpret_cast<const pixel_t *>(other_ptr)[x*4+0]) + | |
| 1636 | 7 | std::abs(reinterpret_cast<const pixel_t *>(cur_ptr)[x*4+1] - reinterpret_cast<const pixel_t *>(other_ptr)[x*4+1]) + | |
| 1637 | 7 | std::abs(reinterpret_cast<const pixel_t *>(cur_ptr)[x*4+2] - reinterpret_cast<const pixel_t *>(other_ptr)[x*4+2]); | |
| 1638 | // no alpha | ||
| 1639 | } | ||
| 1640 | else | ||
| 1641 |
4/4long calculate_sad_8_or_16_sse2<unsigned char, false>(unsigned char const*, unsigned char const*, int, int, unsigned long, unsigned long):
✓ Branch 29 → 28 taken 60 times.
✓ Branch 29 → 30 taken 12 times.
long calculate_sad_8_or_16_sse2<unsigned short, false>(unsigned char const*, unsigned char const*, int, int, unsigned long, unsigned long):
✓ Branch 45 → 44 taken 60 times.
✓ Branch 45 → 46 taken 12 times.
|
144 | for (size_t x = mod16_width / sizeof(pixel_t); x < rowsize / sizeof(pixel_t); ++x) { |
| 1642 | 120 | rowsum += std::abs(reinterpret_cast<const pixel_t *>(cur_ptr)[x] - reinterpret_cast<const pixel_t *>(other_ptr)[x]); | |
| 1643 | } | ||
| 1644 | } | ||
| 1645 | |||
| 1646 | 36 | totalsum += rowsum; | |
| 1647 | |||
| 1648 | 36 | cur_ptr += cur_pitch; | |
| 1649 | 36 | other_ptr += other_pitch; | |
| 1650 | } | ||
| 1651 | 6 | return totalsum; | |
| 1652 | } | ||
| 1653 | |||
| 1654 | // instantiate | ||
| 1655 | template int64_t calculate_sad_8_or_16_sse2<uint8_t, false>(const BYTE* cur_ptr, const BYTE* other_ptr, int cur_pitch, int other_pitch, size_t rowsize, size_t height); | ||
| 1656 | template int64_t calculate_sad_8_or_16_sse2<uint8_t, true>(const BYTE* cur_ptr, const BYTE* other_ptr, int cur_pitch, int other_pitch, size_t rowsize, size_t height); | ||
| 1657 | template int64_t calculate_sad_8_or_16_sse2<uint16_t, false>(const BYTE* cur_ptr, const BYTE* other_ptr, int cur_pitch, int other_pitch, size_t rowsize, size_t height); | ||
| 1658 | template int64_t calculate_sad_8_or_16_sse2<uint16_t, true>(const BYTE* cur_ptr, const BYTE* other_ptr, int cur_pitch, int other_pitch, size_t rowsize, size_t height); | ||
| 1659 | |||
| 1660 | |||
| 1661 | #ifdef X86_32 | ||
| 1662 | int calculate_sad_isse(const BYTE* cur_ptr, const BYTE* other_ptr, int cur_pitch, int other_pitch, size_t rowsize, size_t height) | ||
| 1663 | { | ||
| 1664 | size_t mod8_width = rowsize / 8 * 8; | ||
| 1665 | int result = 0; | ||
| 1666 | __m64 sum = _mm_setzero_si64(); | ||
| 1667 | for (size_t y = 0; y < height; ++y) { | ||
| 1668 | for (size_t x = 0; x < mod8_width; x+=8) { | ||
| 1669 | __m64 cur = *reinterpret_cast<const __m64*>(cur_ptr + x); | ||
| 1670 | __m64 other = *reinterpret_cast<const __m64*>(other_ptr + x); | ||
| 1671 | __m64 sad = _mm_sad_pu8(cur, other); | ||
| 1672 | sum = _mm_add_pi32(sum, sad); | ||
| 1673 | } | ||
| 1674 | if (mod8_width != rowsize) { | ||
| 1675 | for (size_t x = mod8_width; x < rowsize; ++x) { | ||
| 1676 | result += std::abs(cur_ptr[x] - other_ptr[x]); | ||
| 1677 | } | ||
| 1678 | } | ||
| 1679 | |||
| 1680 | cur_ptr += cur_pitch; | ||
| 1681 | other_ptr += other_pitch; | ||
| 1682 | } | ||
| 1683 | result += _mm_cvtsi64_si32(sum); | ||
| 1684 | _mm_empty(); | ||
| 1685 | return result; | ||
| 1686 | } | ||
| 1687 | #endif | ||
| 1688 | |||
| 1689 | |||
| 1690 | |||
| 1691 | |||
| 1692 | |||
| 1693 | |||
| 1694 | |||
| 1695 |