filters/intel/resample_sse.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // Avisynth v2.5. Copyright 2002 Ben Rudiak-Gould et al. | ||
| 2 | // http://avisynth.nl | ||
| 3 | |||
| 4 | // This program is free software; you can redistribute it and/or modify | ||
| 5 | // it under the terms of the GNU General Public License as published by | ||
| 6 | // the Free Software Foundation; either version 2 of the License, or | ||
| 7 | // (at your option) any later version. | ||
| 8 | // | ||
| 9 | // This program is distributed in the hope that it will be useful, | ||
| 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| 12 | // GNU General Public License for more details. | ||
| 13 | // | ||
| 14 | // You should have received a copy of the GNU General Public License | ||
| 15 | // along with this program; if not, write to the Free Software | ||
| 16 | // Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA, or visit | ||
| 17 | // http://www.gnu.org/copyleft/gpl.html . | ||
| 18 | // | ||
| 19 | // Linking Avisynth statically or dynamically with other modules is making a | ||
| 20 | // combined work based on Avisynth. Thus, the terms and conditions of the GNU | ||
| 21 | // General Public License cover the whole combination. | ||
| 22 | // | ||
| 23 | // As a special exception, the copyright holders of Avisynth give you | ||
| 24 | // permission to link Avisynth with independent modules that communicate with | ||
| 25 | // Avisynth solely through the interfaces defined in avisynth.h, regardless of the license | ||
| 26 | // terms of these independent modules, and to copy and distribute the | ||
| 27 | // resulting combined work under terms of your choice, provided that | ||
| 28 | // every copy of the combined work is accompanied by a complete copy of | ||
| 29 | // the source code of Avisynth (the version of Avisynth used to produce the | ||
| 30 | // combined work), being distributed under the terms of the GNU General | ||
| 31 | // Public License plus this exception. An independent module is a module | ||
| 32 | // which is not derived from or based on Avisynth, such as 3rd-party filters, | ||
| 33 | // import and export plugins, or graphical user interfaces. | ||
| 34 | |||
| 35 | |||
| 36 | #include <avisynth.h> | ||
| 37 | #include <avs/config.h> | ||
| 38 | |||
| 39 | #include "../resample.h" | ||
| 40 | |||
| 41 | // Intrinsics base header + really required extension headers | ||
| 42 | #if defined(_MSC_VER) | ||
| 43 | #include <intrin.h> // MSVC | ||
| 44 | #else | ||
| 45 | #include <x86intrin.h> // GCC/MinGW/Clang/LLVM | ||
| 46 | #endif | ||
| 47 | #include <tmmintrin.h> // SSSE3 | ||
| 48 | |||
| 49 | #include "../../core/internal.h" | ||
| 50 | |||
| 51 | |||
| 52 | #ifdef X86_32 | ||
| 53 | void resize_v_mmx_planar(BYTE* dst, const BYTE* src, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int target_height, int bits_per_pixel) | ||
| 54 | { | ||
| 55 | int filter_size = program->filter_size; | ||
| 56 | short* current_coeff = program->pixel_coefficient; | ||
| 57 | |||
| 58 | int wMod8 = (width / 8) * 8; | ||
| 59 | |||
| 60 | __m64 zero = _mm_setzero_si64(); | ||
| 61 | |||
| 62 | const int kernel_size = program->filter_size_real; // not the aligned | ||
| 63 | const int kernel_size_mod2 = (kernel_size / 2) * 2; | ||
| 64 | const bool notMod2 = kernel_size_mod2 < kernel_size; | ||
| 65 | |||
| 66 | for (int y = 0; y < target_height; y++) { | ||
| 67 | int offset = program->pixel_offset[y]; | ||
| 68 | const BYTE* src_ptr = src + offset * src_pitch; | ||
| 69 | |||
| 70 | for (int x = 0; x < wMod8; x += 8) { | ||
| 71 | __m64 result_1 = _mm_set1_pi32(8192); // Init. with rounder (16384/2 = 8192) | ||
| 72 | __m64 result_2 = result_1; | ||
| 73 | __m64 result_3 = result_1; | ||
| 74 | __m64 result_4 = result_1; | ||
| 75 | |||
| 76 | for (int i = 0; i < kernel_size_mod2; i += 2) { | ||
| 77 | __m64 src_p1 = *(reinterpret_cast<const __m64*>(src_ptr + i * src_pitch + x)); // For detailed explanation please see SSE2 version. | ||
| 78 | __m64 src_p2 = *(reinterpret_cast<const __m64*>(src_ptr + (i + 1) * src_pitch + x)); | ||
| 79 | |||
| 80 | __m64 src_l = _mm_unpacklo_pi8(src_p1, src_p2); | ||
| 81 | __m64 src_h = _mm_unpackhi_pi8(src_p1, src_p2); | ||
| 82 | |||
| 83 | __m64 src_1 = _mm_unpacklo_pi8(src_l, zero); | ||
| 84 | __m64 src_2 = _mm_unpackhi_pi8(src_l, zero); | ||
| 85 | __m64 src_3 = _mm_unpacklo_pi8(src_h, zero); | ||
| 86 | __m64 src_4 = _mm_unpackhi_pi8(src_h, zero); | ||
| 87 | |||
| 88 | // two 16 bit short coeffs | ||
| 89 | __m64 coeff = _mm_cvtsi32_si64(*reinterpret_cast<const int*>(current_coeff + i)); | ||
| 90 | coeff = _mm_unpacklo_pi32(coeff, coeff); | ||
| 91 | |||
| 92 | __m64 dst_1 = _mm_madd_pi16(src_1, coeff); | ||
| 93 | __m64 dst_2 = _mm_madd_pi16(src_2, coeff); | ||
| 94 | __m64 dst_3 = _mm_madd_pi16(src_3, coeff); | ||
| 95 | __m64 dst_4 = _mm_madd_pi16(src_4, coeff); | ||
| 96 | |||
| 97 | result_1 = _mm_add_pi32(result_1, dst_1); | ||
| 98 | result_2 = _mm_add_pi32(result_2, dst_2); | ||
| 99 | result_3 = _mm_add_pi32(result_3, dst_3); | ||
| 100 | result_4 = _mm_add_pi32(result_4, dst_4); | ||
| 101 | } | ||
| 102 | |||
| 103 | if (notMod2) { // do last odd row | ||
| 104 | __m64 src_p = *(reinterpret_cast<const __m64*>(src_ptr + kernel_size_mod2 * src_pitch + x)); | ||
| 105 | |||
| 106 | __m64 src_l = _mm_unpacklo_pi8(src_p, zero); | ||
| 107 | __m64 src_h = _mm_unpackhi_pi8(src_p, zero); | ||
| 108 | |||
| 109 | __m64 coeff = _mm_set1_pi16(current_coeff[kernel_size_mod2]); | ||
| 110 | |||
| 111 | __m64 dst_ll = _mm_mullo_pi16(src_l, coeff); // Multiply by coefficient | ||
| 112 | __m64 dst_lh = _mm_mulhi_pi16(src_l, coeff); | ||
| 113 | __m64 dst_hl = _mm_mullo_pi16(src_h, coeff); | ||
| 114 | __m64 dst_hh = _mm_mulhi_pi16(src_h, coeff); | ||
| 115 | |||
| 116 | __m64 dst_1 = _mm_unpacklo_pi16(dst_ll, dst_lh); // Unpack to 32-bit integer | ||
| 117 | __m64 dst_2 = _mm_unpackhi_pi16(dst_ll, dst_lh); | ||
| 118 | __m64 dst_3 = _mm_unpacklo_pi16(dst_hl, dst_hh); | ||
| 119 | __m64 dst_4 = _mm_unpackhi_pi16(dst_hl, dst_hh); | ||
| 120 | |||
| 121 | result_1 = _mm_add_pi32(result_1, dst_1); | ||
| 122 | result_2 = _mm_add_pi32(result_2, dst_2); | ||
| 123 | result_3 = _mm_add_pi32(result_3, dst_3); | ||
| 124 | result_4 = _mm_add_pi32(result_4, dst_4); | ||
| 125 | } | ||
| 126 | |||
| 127 | // Divide by 16348 (FPRound) | ||
| 128 | result_1 = _mm_srai_pi32(result_1, 14); | ||
| 129 | result_2 = _mm_srai_pi32(result_2, 14); | ||
| 130 | result_3 = _mm_srai_pi32(result_3, 14); | ||
| 131 | result_4 = _mm_srai_pi32(result_4, 14); | ||
| 132 | |||
| 133 | // Pack and store | ||
| 134 | __m64 result_l = _mm_packs_pi32(result_1, result_2); | ||
| 135 | __m64 result_h = _mm_packs_pi32(result_3, result_4); | ||
| 136 | __m64 result = _mm_packs_pu16(result_l, result_h); | ||
| 137 | |||
| 138 | *(reinterpret_cast<__m64*>(dst + x)) = result; | ||
| 139 | } | ||
| 140 | |||
| 141 | // Leftover | ||
| 142 | for (int x = wMod8; x < width; x++) { | ||
| 143 | int result = 0; | ||
| 144 | for (int i = 0; i < kernel_size; i++) { | ||
| 145 | result += (src_ptr + i * src_pitch)[x] * current_coeff[i]; | ||
| 146 | } | ||
| 147 | result = ((result + 8192) / 16384); | ||
| 148 | result = result > 255 ? 255 : result < 0 ? 0 : result; | ||
| 149 | dst[x] = (BYTE)result; | ||
| 150 | } | ||
| 151 | |||
| 152 | dst += dst_pitch; | ||
| 153 | current_coeff += filter_size; | ||
| 154 | } | ||
| 155 | |||
| 156 | _mm_empty(); | ||
| 157 | } | ||
| 158 | #endif | ||
| 159 | |||
| 160 | // On x86-32 keep the 1×8 (or 2-lane/8-pixel) kernel | ||
| 161 | // On x86-64 use the 2×8 (4-lane/16-pixel) kernel. | ||
| 162 | // On 32-bit the only 8 XMM registers are available, 2x8 kernel causes register pressure issues. | ||
| 163 | |||
| 164 | // 1x8 pixel kernel, hoped to have less register pressure on x86-32 thus slower, but the 2x8 is faster anyway. | ||
| 165 | // Left here for reference. | ||
| 166 | ✗ | void resize_v_sse2_planar_pix8(BYTE* dst8, const BYTE* src, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int target_height, int bits_per_pixel) | |
| 167 | { | ||
| 168 | AVS_UNUSED(bits_per_pixel); | ||
| 169 | |||
| 170 | ✗ | const int filter_size = program->filter_size; | |
| 171 | ✗ | uint8_t* AVS_RESTRICT dst = (uint8_t * AVS_RESTRICT)dst8; | |
| 172 | ✗ | const short* AVS_RESTRICT current_coeff = program->pixel_coefficient; | |
| 173 | |||
| 174 | ✗ | const __m128i zero = _mm_setzero_si128(); | |
| 175 | ✗ | const __m128i rounder = _mm_set1_epi32(1 << (FPScale8bits - 1)); | |
| 176 | |||
| 177 | ✗ | const int kernel_size = program->filter_size_real; // not the aligned | |
| 178 | ✗ | const int kernel_size_mod2 = (kernel_size / 2) * 2; | |
| 179 | ✗ | const bool notMod2 = kernel_size_mod2 < kernel_size; | |
| 180 | |||
| 181 | ✗ | for (int y = 0; y < target_height; y++) { | |
| 182 | ✗ | const int offset = program->pixel_offset[y]; | |
| 183 | ✗ | const BYTE* src_ptr = src + offset * src_pitch; | |
| 184 | |||
| 185 | // alignment is safe till 64 bytes | ||
| 186 | // 8 pixels at a time | ||
| 187 | ✗ | for (int x = 0; x < width; x += 8) { | |
| 188 | ✗ | __m128i result_single_lo = rounder; | |
| 189 | ✗ | __m128i result_single_hi = rounder; | |
| 190 | |||
| 191 | ✗ | const uint8_t* AVS_RESTRICT src2_ptr = src_ptr + x; | |
| 192 | |||
| 193 | // Process pairs of rows for better efficiency (2 coeffs/cycle) | ||
| 194 | ✗ | int i = 0; | |
| 195 | ✗ | for (; i < kernel_size_mod2; i += 2) { | |
| 196 | // Load _two_ coefficients as a single packed value and broadcast | ||
| 197 | ✗ | __m128i coeff = _mm_set1_epi32(*reinterpret_cast<const int*>(current_coeff + i)); // CO|co|CO|co|CO|co|CO|co | |
| 198 | |||
| 199 | ✗ | __m128i src_even = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(src2_ptr)); // 8x 8bit pixels | |
| 200 | ✗ | __m128i src_odd = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(src2_ptr + src_pitch)); // 8x 8bit pixels | |
| 201 | ✗ | src_even = _mm_unpacklo_epi8(src_even, zero); | |
| 202 | ✗ | src_odd = _mm_unpacklo_epi8(src_odd, zero); | |
| 203 | ✗ | __m128i src_lo = _mm_unpacklo_epi16(src_even, src_odd); | |
| 204 | ✗ | __m128i src_hi = _mm_unpackhi_epi16(src_even, src_odd); | |
| 205 | ✗ | result_single_lo = _mm_add_epi32(result_single_lo, _mm_madd_epi16(src_lo, coeff)); // a*b + c | |
| 206 | ✗ | result_single_hi = _mm_add_epi32(result_single_hi, _mm_madd_epi16(src_hi, coeff)); // a*b + c | |
| 207 | ✗ | src2_ptr += 2 * src_pitch; // Move to the next pair of rows | |
| 208 | } | ||
| 209 | |||
| 210 | // Process the last odd row if needed | ||
| 211 | ✗ | if (notMod2) { | |
| 212 | // Load a single coefficients as a single packed value and broadcast | ||
| 213 | ✗ | __m128i coeff = _mm_set1_epi16(*reinterpret_cast<const short*>(current_coeff + i)); // 0|co|0|co|0|co|0|co | |
| 214 | |||
| 215 | ✗ | __m128i src_even = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(src2_ptr)); // 8x 8bit pixels | |
| 216 | ✗ | src_even = _mm_unpacklo_epi8(src_even, zero); | |
| 217 | ✗ | __m128i src_lo = _mm_unpacklo_epi16(src_even, zero); | |
| 218 | ✗ | __m128i src_hi = _mm_unpackhi_epi16(src_even, zero); | |
| 219 | ✗ | result_single_lo = _mm_add_epi32(result_single_lo, _mm_madd_epi16(src_lo, coeff)); // a*b + c | |
| 220 | ✗ | result_single_hi = _mm_add_epi32(result_single_hi, _mm_madd_epi16(src_hi, coeff)); // a*b + c | |
| 221 | } | ||
| 222 | |||
| 223 | // scale back, store | ||
| 224 | ✗ | __m128i result_lo = result_single_lo; | |
| 225 | ✗ | __m128i result_hi = result_single_hi; | |
| 226 | // shift back integer arithmetic 14 bits precision | ||
| 227 | ✗ | result_lo = _mm_srai_epi32(result_lo, FPScale8bits); | |
| 228 | ✗ | result_hi = _mm_srai_epi32(result_hi, FPScale8bits); | |
| 229 | |||
| 230 | // Note: SSE4.1 simulations for SSE2: _mm_packus_epi32 | ||
| 231 | ✗ | __m128i result_8x_uint16 = _MM_PACKUS_EPI32(result_lo, result_hi); // 8*32 => 8*16 | |
| 232 | ✗ | __m128i result_8x_uint8 = _mm_packus_epi16(result_8x_uint16, result_8x_uint16); // 8*16 => 8*8 | |
| 233 | ✗ | _mm_storel_epi64(reinterpret_cast<__m128i*>(dst + x), result_8x_uint8); | |
| 234 | } | ||
| 235 | |||
| 236 | ✗ | dst += dst_pitch; | |
| 237 | ✗ | current_coeff += filter_size; | |
| 238 | } | ||
| 239 | ✗ | } | |
| 240 | |||
| 241 | // 2x8 (4-lane/16-pixel) kernel. | ||
| 242 | // For some reason, this is actually faster even on x86-32 contrary to the register pressure worries. | ||
| 243 | 5 | void resize_v_sse2_planar_pix16(BYTE* dst8, const BYTE* src, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int target_height, int bits_per_pixel) | |
| 244 | { | ||
| 245 | AVS_UNUSED(bits_per_pixel); | ||
| 246 | |||
| 247 | 5 | const int filter_size = program->filter_size; | |
| 248 | 5 | uint8_t* AVS_RESTRICT dst = (uint8_t * AVS_RESTRICT)dst8; | |
| 249 | 5 | const short* AVS_RESTRICT current_coeff = program->pixel_coefficient; | |
| 250 | |||
| 251 | 5 | const __m128i zero = _mm_setzero_si128(); | |
| 252 | 5 | const __m128i rounder = _mm_set1_epi32(1 << (FPScale8bits - 1)); | |
| 253 | |||
| 254 | 5 | const int kernel_size = program->filter_size_real; // not the aligned | |
| 255 | 5 | const int kernel_size_mod2 = (kernel_size / 2) * 2; | |
| 256 | 5 | const bool notMod2 = kernel_size_mod2 < kernel_size; | |
| 257 | |||
| 258 |
2/2✓ Branch 173 → 9 taken 29 times.
✓ Branch 173 → 174 taken 5 times.
|
34 | for (int y = 0; y < target_height; y++) { |
| 259 | 29 | const int offset = program->pixel_offset[y]; | |
| 260 | 29 | const BYTE* src_ptr = src + offset * src_pitch; | |
| 261 | |||
| 262 | // alignment is safe till 64 | ||
| 263 | // 16 pixels at a time | ||
| 264 |
2/2✓ Branch 171 → 11 taken 106 times.
✓ Branch 171 → 172 taken 29 times.
|
135 | for (int x = 0; x < width; x += 16) { |
| 265 | 106 | __m128i result_single_lo = rounder; | |
| 266 | 106 | __m128i result_single_hi = rounder; | |
| 267 | |||
| 268 | 106 | __m128i result_single2_lo = rounder; | |
| 269 | 106 | __m128i result_single2_hi = rounder; | |
| 270 | |||
| 271 | 106 | const uint8_t* AVS_RESTRICT src2_ptr = src_ptr + x; | |
| 272 | |||
| 273 | // Process pairs of rows for better efficiency (2 coeffs/cycle) | ||
| 274 | 106 | int i = 0; | |
| 275 |
2/2✓ Branch 73 → 12 taken 322 times.
✓ Branch 73 → 74 taken 106 times.
|
428 | for (; i < kernel_size_mod2; i += 2) { |
| 276 | // Load _two_ coefficients as a single packed value and broadcast | ||
| 277 | 644 | __m128i coeff = _mm_set1_epi32(*reinterpret_cast<const int*>(current_coeff + i)); // CO|co|CO|co|CO|co|CO|co | |
| 278 | |||
| 279 | 322 | __m128i src_even = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(src2_ptr)); // 8x 8bit pixels | |
| 280 | 322 | __m128i src_odd = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(src2_ptr + src_pitch)); // 8x 8bit pixels | |
| 281 | |||
| 282 | 322 | __m128i src_even2 = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(src2_ptr + 8)); // 8x 8bit pixels | |
| 283 | 644 | __m128i src_odd2 = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(src2_ptr + src_pitch + 8)); // 8x 8bit pixels | |
| 284 | |||
| 285 | 322 | src_even = _mm_unpacklo_epi8(src_even, zero); | |
| 286 | 322 | src_odd = _mm_unpacklo_epi8(src_odd, zero); | |
| 287 | |||
| 288 | 322 | src_even2 = _mm_unpacklo_epi8(src_even2, zero); | |
| 289 | 322 | src_odd2 = _mm_unpacklo_epi8(src_odd2, zero); | |
| 290 | |||
| 291 | 322 | __m128i src_lo = _mm_unpacklo_epi16(src_even, src_odd); | |
| 292 | 322 | __m128i src_hi = _mm_unpackhi_epi16(src_even, src_odd); | |
| 293 | |||
| 294 | 322 | __m128i src_lo2 = _mm_unpacklo_epi16(src_even2, src_odd2); | |
| 295 | 322 | __m128i src_hi2 = _mm_unpackhi_epi16(src_even2, src_odd2); | |
| 296 | |||
| 297 | 644 | result_single_lo = _mm_add_epi32(result_single_lo, _mm_madd_epi16(src_lo, coeff)); // a*b + c | |
| 298 | 644 | result_single_hi = _mm_add_epi32(result_single_hi, _mm_madd_epi16(src_hi, coeff)); // a*b + c | |
| 299 | |||
| 300 | 644 | result_single2_lo = _mm_add_epi32(result_single2_lo, _mm_madd_epi16(src_lo2, coeff)); // a*b + c | |
| 301 | 322 | result_single2_hi = _mm_add_epi32(result_single2_hi, _mm_madd_epi16(src_hi2, coeff)); // a*b + c | |
| 302 | |||
| 303 | 322 | src2_ptr += 2 * src_pitch; // Move to the next pair of rows | |
| 304 | } | ||
| 305 | |||
| 306 | // Process the last odd row if needed | ||
| 307 |
2/2✓ Branch 74 → 75 taken 82 times.
✓ Branch 74 → 120 taken 24 times.
|
106 | if (notMod2) { |
| 308 | // Load a single coefficients as a single packed value and broadcast | ||
| 309 | 164 | __m128i coeff = _mm_set1_epi16(*reinterpret_cast<const short*>(current_coeff + i)); // 0|co|0|co|0|co|0|co | |
| 310 | |||
| 311 | 82 | __m128i src_even = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(src2_ptr)); // 8x 8bit pixels | |
| 312 | 164 | __m128i src_even2 = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(src2_ptr + 8)); // 8x 8bit pixels | |
| 313 | |||
| 314 | 82 | src_even = _mm_unpacklo_epi8(src_even, zero); | |
| 315 | 82 | src_even2 = _mm_unpacklo_epi8(src_even2, zero); | |
| 316 | |||
| 317 | 82 | __m128i src_lo = _mm_unpacklo_epi16(src_even, zero); | |
| 318 | 82 | __m128i src_hi = _mm_unpackhi_epi16(src_even, zero); | |
| 319 | 82 | __m128i src_lo2 = _mm_unpacklo_epi16(src_even2, zero); | |
| 320 | 82 | __m128i src_hi2 = _mm_unpackhi_epi16(src_even2, zero); | |
| 321 | |||
| 322 | 164 | result_single_lo = _mm_add_epi32(result_single_lo, _mm_madd_epi16(src_lo, coeff)); // a*b + c | |
| 323 | 164 | result_single_hi = _mm_add_epi32(result_single_hi, _mm_madd_epi16(src_hi, coeff)); // a*b + c | |
| 324 | 164 | result_single2_lo = _mm_add_epi32(result_single2_lo, _mm_madd_epi16(src_lo2, coeff)); // a*b + c | |
| 325 | 164 | result_single2_hi = _mm_add_epi32(result_single2_hi, _mm_madd_epi16(src_hi2, coeff)); // a*b + c | |
| 326 | |||
| 327 | } | ||
| 328 | |||
| 329 | // scale back, store | ||
| 330 | 106 | __m128i result_lo = result_single_lo; | |
| 331 | 106 | __m128i result_hi = result_single_hi; | |
| 332 | 106 | __m128i result_lo2 = result_single2_lo; | |
| 333 | 106 | __m128i result_hi2 = result_single2_hi; | |
| 334 | |||
| 335 | // shift back integer arithmetic 14 bits precision | ||
| 336 | 106 | result_lo = _mm_srai_epi32(result_lo, FPScale8bits); | |
| 337 | 106 | result_hi = _mm_srai_epi32(result_hi, FPScale8bits); | |
| 338 | 106 | result_lo2 = _mm_srai_epi32(result_lo2, FPScale8bits); | |
| 339 | 106 | result_hi2 = _mm_srai_epi32(result_hi2, FPScale8bits); | |
| 340 | |||
| 341 | // Note: SSE4.1 simulations for SSE2: _mm_packus_epi32 | ||
| 342 | 106 | __m128i result_8x_uint16 = _MM_PACKUS_EPI32(result_lo, result_hi); // 8*32 => 8*16 | |
| 343 | 106 | __m128i result_8x_uint8 = _mm_packus_epi16(result_8x_uint16, result_8x_uint16); // 8*16 => 8*8 | |
| 344 | 106 | _mm_storel_epi64(reinterpret_cast<__m128i*>(dst + x), result_8x_uint8); | |
| 345 | |||
| 346 | 106 | __m128i result2_8x_uint16 = _MM_PACKUS_EPI32(result_lo2, result_hi2); // 8*32 => 8*16 | |
| 347 | 106 | __m128i result2_8x_uint8 = _mm_packus_epi16(result2_8x_uint16, result2_8x_uint16); // 8*16 => 8*8 | |
| 348 | 106 | _mm_storel_epi64(reinterpret_cast<__m128i*>(dst + x + 8), result2_8x_uint8); | |
| 349 | |||
| 350 | } | ||
| 351 | |||
| 352 | 29 | dst += dst_pitch; | |
| 353 | 29 | current_coeff += filter_size; | |
| 354 | } | ||
| 355 | 5 | } | |
| 356 | |||
| 357 | #if defined(X86_64) | ||
| 358 | 5 | static void resize_v_sse2_planar_impl(BYTE* dst8, const BYTE* src, int dst_pitch, int src_pitch, | |
| 359 | ResamplingProgram* program, int width, int target_height, int bits_per_pixel) | ||
| 360 | { | ||
| 361 | 5 | resize_v_sse2_planar_pix16(dst8, src, dst_pitch, src_pitch, program, width, target_height, bits_per_pixel); | |
| 362 | 5 | } | |
| 363 | #elif defined(X86_32) | ||
| 364 | static void resize_v_sse2_planar_impl(BYTE* dst8, const BYTE* src, int dst_pitch, int src_pitch, | ||
| 365 | ResamplingProgram* program, int width, int target_height, int bits_per_pixel) | ||
| 366 | { | ||
| 367 | // Use the 2x8 kernel even on x86-32 as it is faster despite register pressure concerns. | ||
| 368 | resize_v_sse2_planar_pix16(dst8, src, dst_pitch, src_pitch, program, width, target_height, bits_per_pixel); | ||
| 369 | //resize_v_sse2_planar_pix8(dst8, src, dst_pitch, src_pitch, program, width, target_height, bits_per_pixel); | ||
| 370 | } | ||
| 371 | #else | ||
| 372 | #error Unsupported target for resize_v_sse2_planar | ||
| 373 | #endif | ||
| 374 | |||
| 375 | 5 | void resize_v_sse2_planar(BYTE* dst8, const BYTE* src, int dst_pitch, int src_pitch, | |
| 376 | ResamplingProgram* program, int width, int target_height, int bits_per_pixel) | ||
| 377 | { | ||
| 378 | 5 | resize_v_sse2_planar_impl(dst8, src, dst_pitch, src_pitch, program, width, target_height, bits_per_pixel); | |
| 379 | 5 | } | |
| 380 | |||
| 381 | // like the AVX2 version, but only 8 pixels at a time | ||
| 382 | template<bool lessthan16bit> | ||
| 383 | 8 | void resize_v_sse2_planar_uint16_t(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int target_height, int bits_per_pixel) | |
| 384 | { | ||
| 385 | AVS_UNUSED(bits_per_pixel); | ||
| 386 | |||
| 387 | 8 | const int filter_size = program->filter_size; | |
| 388 | 8 | const short* AVS_RESTRICT current_coeff = program->pixel_coefficient; | |
| 389 | |||
| 390 | 8 | const __m128i zero = _mm_setzero_si128(); | |
| 391 | |||
| 392 | // for 16 bits only | ||
| 393 | 8 | [[maybe_unused]] const __m128i shifttosigned = _mm_set1_epi16(-32768); | |
| 394 | 8 | [[maybe_unused]] const __m128i shiftfromsigned = _mm_set1_epi32(32768 << FPScale16bits); | |
| 395 | |||
| 396 | 8 | const __m128i rounder = _mm_set1_epi32(1 << (FPScale16bits - 1)); | |
| 397 | |||
| 398 | 8 | const uint16_t* src = (uint16_t*)src8; | |
| 399 | 8 | uint16_t* AVS_RESTRICT dst = (uint16_t* AVS_RESTRICT)dst8; | |
| 400 | 8 | dst_pitch = dst_pitch / sizeof(uint16_t); | |
| 401 | 8 | src_pitch = src_pitch / sizeof(uint16_t); | |
| 402 | |||
| 403 | 8 | const int limit = (1 << bits_per_pixel) - 1; | |
| 404 | 8 | __m128i clamp_limit = _mm_set1_epi16((short)limit); // clamp limit for <16 bits | |
| 405 | |||
| 406 | 8 | const int kernel_size = program->filter_size_real; // not the aligned | |
| 407 | 8 | const int kernel_size_mod2 = (kernel_size / 2) * 2; | |
| 408 | 8 | const bool notMod2 = kernel_size_mod2 < kernel_size; | |
| 409 | |||
| 410 |
4/4void resize_v_sse2_planar_uint16_t<false>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 102 → 21 taken 23 times.
✓ Branch 102 → 103 taken 4 times.
void resize_v_sse2_planar_uint16_t<true>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 110 → 21 taken 23 times.
✓ Branch 110 → 111 taken 4 times.
|
54 | for (int y = 0; y < target_height; y++) { |
| 411 | 46 | const int offset = program->pixel_offset[y]; | |
| 412 | 46 | const uint16_t* src_ptr = src + offset * src_pitch; | |
| 413 | |||
| 414 | // 16 byte 8 word (half as many as AVX2) | ||
| 415 | // no need wmod8, alignment is safe at least 32 | ||
| 416 |
4/4void resize_v_sse2_planar_uint16_t<false>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 100 → 23 taken 128 times.
✓ Branch 100 → 101 taken 23 times.
void resize_v_sse2_planar_uint16_t<true>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 108 → 23 taken 128 times.
✓ Branch 108 → 109 taken 23 times.
|
302 | for (int x = 0; x < width; x += 8) { |
| 417 | |||
| 418 | 256 | __m128i result_single_lo = rounder; | |
| 419 | 256 | __m128i result_single_hi = rounder; | |
| 420 | |||
| 421 | 256 | const uint16_t* AVS_RESTRICT src2_ptr = src_ptr + x; | |
| 422 | |||
| 423 | // Process pairs of rows for better efficiency (2 coeffs/cycle) | ||
| 424 | 256 | int i = 0; | |
| 425 |
4/4void resize_v_sse2_planar_uint16_t<false>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 49 → 24 taken 452 times.
✓ Branch 49 → 50 taken 128 times.
void resize_v_sse2_planar_uint16_t<true>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 45 → 24 taken 344 times.
✓ Branch 45 → 46 taken 128 times.
|
1052 | for (; i < kernel_size_mod2; i += 2) { |
| 426 | // Load _two_ coefficients as a single packed value and broadcast | ||
| 427 | 1592 | __m128i coeff = _mm_set1_epi32(*reinterpret_cast<const int*>(current_coeff + i)); // CO|co|CO|co|CO|co|CO|co | |
| 428 | |||
| 429 | 796 | __m128i src_even = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src2_ptr)); // 8x 16bit pixels | |
| 430 | 1592 | __m128i src_odd = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src2_ptr + src_pitch)); // 8x 16bit pixels | |
| 431 | if constexpr (!lessthan16bit) { | ||
| 432 | 452 | src_even = _mm_add_epi16(src_even, shifttosigned); | |
| 433 | 452 | src_odd = _mm_add_epi16(src_odd, shifttosigned); | |
| 434 | } | ||
| 435 | 796 | __m128i src_lo = _mm_unpacklo_epi16(src_even, src_odd); | |
| 436 | 796 | __m128i src_hi = _mm_unpackhi_epi16(src_even, src_odd); | |
| 437 | 1592 | result_single_lo = _mm_add_epi32(result_single_lo, _mm_madd_epi16(src_lo, coeff)); // a*b + c | |
| 438 | 796 | result_single_hi = _mm_add_epi32(result_single_hi, _mm_madd_epi16(src_hi, coeff)); // a*b + c | |
| 439 | 796 | src2_ptr += 2 * src_pitch; // Move to the next pair of rows | |
| 440 | } | ||
| 441 | |||
| 442 | // Process the last odd row if needed | ||
| 443 |
3/4void resize_v_sse2_planar_uint16_t<false>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 50 → 51 taken 92 times.
✓ Branch 50 → 72 taken 36 times.
void resize_v_sse2_planar_uint16_t<true>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 46 → 47 taken 128 times.
✗ Branch 46 → 66 not taken.
|
256 | if (notMod2) { |
| 444 | // Load a single coefficients as a single packed value and broadcast | ||
| 445 | 440 | __m128i coeff = _mm_set1_epi16(*reinterpret_cast<const short*>(current_coeff + i)); // 0|co|0|co|0|co|0|co | |
| 446 | |||
| 447 | 220 | __m128i src_even = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src2_ptr)); // 8x 16bit pixels | |
| 448 | if constexpr (!lessthan16bit) { | ||
| 449 | 92 | src_even = _mm_add_epi16(src_even, shifttosigned); | |
| 450 | } | ||
| 451 | 220 | __m128i src_lo = _mm_unpacklo_epi16(src_even, zero); | |
| 452 | 220 | __m128i src_hi = _mm_unpackhi_epi16(src_even, zero); | |
| 453 | 440 | result_single_lo = _mm_add_epi32(result_single_lo, _mm_madd_epi16(src_lo, coeff)); // a*b + c | |
| 454 | 440 | result_single_hi = _mm_add_epi32(result_single_hi, _mm_madd_epi16(src_hi, coeff)); // a*b + c | |
| 455 | } | ||
| 456 | |||
| 457 | // correct if signed, scale back, store | ||
| 458 | 256 | __m128i result_lo = result_single_lo; | |
| 459 | 256 | __m128i result_hi = result_single_hi; | |
| 460 | if constexpr (!lessthan16bit) { | ||
| 461 | 128 | result_lo = _mm_add_epi32(result_lo, shiftfromsigned); | |
| 462 | 128 | result_hi = _mm_add_epi32(result_hi, shiftfromsigned); | |
| 463 | } | ||
| 464 | // shift back integer arithmetic 13 bits precision | ||
| 465 | 256 | result_lo = _mm_srai_epi32(result_lo, FPScale16bits); | |
| 466 | 256 | result_hi = _mm_srai_epi32(result_hi, FPScale16bits); | |
| 467 | |||
| 468 | // Note: SSE4.1 simulations for SSE2: _mm_packus_epi32, _mm_min_epu16 | ||
| 469 | 256 | __m128i result_8x_uint16 = _MM_PACKUS_EPI32(result_lo, result_hi); // 8*32 => 8*16 | |
| 470 | if constexpr (lessthan16bit) { | ||
| 471 | 128 | result_8x_uint16 = _MM_MIN_EPU16(result_8x_uint16, clamp_limit); // extra clamp for 10-14 bit | |
| 472 | } | ||
| 473 | 256 | _mm_stream_si128(reinterpret_cast<__m128i*>(dst + x), result_8x_uint16); | |
| 474 | } | ||
| 475 | |||
| 476 | 46 | dst += dst_pitch; | |
| 477 | 46 | current_coeff += filter_size; | |
| 478 | } | ||
| 479 | 8 | } | |
| 480 | //-------- 128 bit float Horizontals | ||
| 481 | |||
| 482 | AVS_FORCEINLINE static void process_two_8pixels_h_float(const float* src, int begin1, int begin2, int i, float* current_coeff, int filter_size, __m128& result1, __m128& result2) { | ||
| 483 | 184 | __m128 data_1_low = _mm_loadu_ps(src + begin1 + i); // Load first 4 floats | |
| 484 | 92 | __m128 data_1_high = _mm_loadu_ps(src + begin1 + i + 4); // Load next 4 floats | |
| 485 | 92 | __m128 data_2_low = _mm_loadu_ps(src + begin2 + i); // Load first 4 floats | |
| 486 | 184 | __m128 data_2_high = _mm_loadu_ps(src + begin2 + i + 4); // Load next 4 floats | |
| 487 | |||
| 488 | 92 | __m128 coeff_1_low = _mm_load_ps(current_coeff); // Load first 4 coefficients | |
| 489 | 92 | __m128 coeff_1_high = _mm_load_ps(current_coeff + 4); // Load next 4 coefficients | |
| 490 | 92 | __m128 coeff_2_low = _mm_load_ps(current_coeff + filter_size); // Load first 4 coefficients for second pixel | |
| 491 | 184 | __m128 coeff_2_high = _mm_load_ps(current_coeff + filter_size + 4); // Load next 4 coefficients for second pixel | |
| 492 | |||
| 493 | 184 | result1 = _mm_add_ps(result1, _mm_mul_ps(data_1_low, coeff_1_low)); // a*b + c for first 4 floats | |
| 494 | 184 | result1 = _mm_add_ps(result1, _mm_mul_ps(data_1_high, coeff_1_high)); // a*b + c for next 4 floats | |
| 495 | 184 | result2 = _mm_add_ps(result2, _mm_mul_ps(data_2_low, coeff_2_low)); // a*b + c for first 4 floats | |
| 496 | 92 | result2 = _mm_add_ps(result2, _mm_mul_ps(data_2_high, coeff_2_high)); // a*b + c for next 4 floats | |
| 497 | 92 | } | |
| 498 | |||
| 499 | template<bool safe_aligned_mode> | ||
| 500 | AVS_FORCEINLINE static void process_two_pixels_h_float(const float* src_ptr, int begin1, int begin2, float* current_coeff, int filter_size, __m128& result1, __m128& result2, int kernel_size) { | ||
| 501 | int ksmod8; | ||
| 502 | // 32 bytes contain 8 floats | ||
| 503 | if constexpr (safe_aligned_mode) | ||
| 504 | 92 | ksmod8 = filter_size / 8 * 8; | |
| 505 | else | ||
| 506 | 36 | ksmod8 = kernel_size / 8 * 8; // danger zone, scanline overread possible. Use exact unaligned kernel_size | |
| 507 | 128 | const float* src_ptr1 = src_ptr + begin1; | |
| 508 | 128 | const float* src_ptr2 = src_ptr + begin2; | |
| 509 | 128 | int i = 0; | |
| 510 | |||
| 511 | // Process 8 elements at a time | ||
| 512 |
12/16✓ Branch 48 → 14 taken 23 times.
✓ Branch 48 → 49 taken 23 times.
✓ Branch 89 → 55 taken 23 times.
✓ Branch 89 → 90 taken 23 times.
✓ Branch 132 → 98 taken 23 times.
✓ Branch 132 → 133 taken 23 times.
✓ Branch 173 → 139 taken 23 times.
✓ Branch 173 → 174 taken 23 times.
✗ Branch 224 → 190 not taken.
✓ Branch 224 → 225 taken 9 times.
✗ Branch 298 → 264 not taken.
✓ Branch 298 → 299 taken 9 times.
✗ Branch 374 → 340 not taken.
✓ Branch 374 → 375 taken 9 times.
✗ Branch 448 → 414 not taken.
✓ Branch 448 → 449 taken 9 times.
|
220 | for (; i < ksmod8; i += 8) { |
| 513 | 92 | process_two_8pixels_h_float(src_ptr, begin1, begin2, i, current_coeff + i, filter_size, result1, result2); | |
| 514 | } | ||
| 515 | |||
| 516 | if constexpr (!safe_aligned_mode) { | ||
| 517 | // working with the original, unaligned kernel_size | ||
| 518 |
4/8✗ Branch 225 → 226 not taken.
✓ Branch 225 → 227 taken 9 times.
✗ Branch 299 → 300 not taken.
✓ Branch 299 → 301 taken 9 times.
✗ Branch 375 → 376 not taken.
✓ Branch 375 → 377 taken 9 times.
✗ Branch 449 → 450 not taken.
✓ Branch 449 → 451 taken 9 times.
|
36 | if (i == kernel_size) return; |
| 519 | |||
| 520 | 36 | float* current_coeff2 = current_coeff + filter_size; // Points to second pixel's coefficients | |
| 521 | 36 | const int ksmod4 = kernel_size / 4 * 4; | |
| 522 | |||
| 523 | // Process 4 elements if needed | ||
| 524 |
8/8✓ Branch 227 → 228 taken 4 times.
✓ Branch 227 → 246 taken 5 times.
✓ Branch 301 → 302 taken 4 times.
✓ Branch 301 → 320 taken 5 times.
✓ Branch 377 → 378 taken 4 times.
✓ Branch 377 → 396 taken 5 times.
✓ Branch 451 → 452 taken 4 times.
✓ Branch 451 → 470 taken 5 times.
|
36 | if (i < ksmod4) { |
| 525 | // Process 4 elements for first pixel | ||
| 526 | 16 | __m128 data_1 = _mm_loadu_ps(src_ptr1 + i); | |
| 527 | 32 | __m128 coeff_1 = _mm_load_ps(current_coeff + i); | |
| 528 | 16 | __m128 temp_result1 = _mm_mul_ps(data_1, coeff_1); | |
| 529 | |||
| 530 | // Process 4 elements for second pixel | ||
| 531 | 16 | __m128 data_2 = _mm_loadu_ps(src_ptr2 + i); | |
| 532 | 32 | __m128 coeff_2 = _mm_load_ps(current_coeff2 + i); | |
| 533 | 16 | __m128 temp_result2 = _mm_mul_ps(data_2, coeff_2); | |
| 534 | |||
| 535 | // update result vectors | ||
| 536 | 16 | result1 = _mm_add_ps(result1, temp_result1); | |
| 537 | 16 | result2 = _mm_add_ps(result2, temp_result2); | |
| 538 | |||
| 539 | 16 | i += 4; | |
| 540 |
4/8✓ Branch 244 → 245 taken 4 times.
✗ Branch 244 → 246 not taken.
✓ Branch 318 → 319 taken 4 times.
✗ Branch 318 → 320 not taken.
✓ Branch 394 → 395 taken 4 times.
✗ Branch 394 → 396 not taken.
✓ Branch 468 → 469 taken 4 times.
✗ Branch 468 → 470 not taken.
|
16 | if (i == kernel_size) return; |
| 541 | } | ||
| 542 | |||
| 543 | // Process remaining elements with scalar operations | ||
| 544 |
4/8✓ Branch 246 → 247 taken 5 times.
✗ Branch 246 → 259 not taken.
✓ Branch 320 → 321 taken 5 times.
✗ Branch 320 → 333 not taken.
✓ Branch 396 → 397 taken 5 times.
✗ Branch 396 → 409 not taken.
✓ Branch 470 → 471 taken 5 times.
✗ Branch 470 → 483 not taken.
|
20 | if (i < kernel_size) { |
| 545 | 20 | float scalar_sum1[4] = { 0, 0, 0, 0 }; // like an __m128 | |
| 546 | 20 | float scalar_sum2[4] = { 0, 0, 0, 0 }; | |
| 547 | |||
| 548 |
8/8✓ Branch 249 → 248 taken 15 times.
✓ Branch 249 → 250 taken 5 times.
✓ Branch 323 → 322 taken 15 times.
✓ Branch 323 → 324 taken 5 times.
✓ Branch 399 → 398 taken 15 times.
✓ Branch 399 → 400 taken 5 times.
✓ Branch 473 → 472 taken 15 times.
✓ Branch 473 → 474 taken 5 times.
|
80 | for (; i < kernel_size; i++) { |
| 549 | 60 | scalar_sum1[i % 4] += src_ptr1[i] * current_coeff[i]; | |
| 550 | 60 | scalar_sum2[i % 4] += src_ptr2[i] * current_coeff2[i]; | |
| 551 | } | ||
| 552 | |||
| 553 | // Convert scalar results to SIMD and add to result vectors | ||
| 554 | 20 | __m128 temp_result1 = _mm_loadu_ps(scalar_sum1); | |
| 555 | 20 | __m128 temp_result2 = _mm_loadu_ps(scalar_sum2); | |
| 556 | |||
| 557 | 20 | result1 = _mm_add_ps(result1, temp_result1); | |
| 558 | 40 | result2 = _mm_add_ps(result2, temp_result2); | |
| 559 | } | ||
| 560 | } | ||
| 561 | 92 | } | |
| 562 | |||
| 563 | template<bool is_safe> | ||
| 564 | #if defined(GCC) || defined(CLANG) | ||
| 565 | __attribute__((__target__("ssse3"))) | ||
| 566 | #endif | ||
| 567 | AVS_FORCEINLINE static void process_eight_pixels_h_float(const float* src, int x, float* current_coeff_base, int filter_size, | ||
| 568 | __m128& zero128, | ||
| 569 | float* dst, | ||
| 570 | ResamplingProgram* program) | ||
| 571 | { | ||
| 572 |
2/4✗ Branch 9 → 10 not taken.
✓ Branch 9 → 11 taken 23 times.
✗ Branch 185 → 186 not taken.
✓ Branch 185 → 187 taken 9 times.
|
32 | assert(program->filter_size_alignment >= 8); // code assumes this |
| 573 | |||
| 574 | 32 | float* current_coeff = current_coeff_base + x * filter_size; | |
| 575 | 32 | const int unaligned_kernel_size = program->filter_size_real; | |
| 576 | |||
| 577 | // Unrolled processing of all 8 pixels | ||
| 578 | |||
| 579 | // 0 & 1 | ||
| 580 | 32 | __m128 result0 = zero128; | |
| 581 | 32 | __m128 result1 = zero128; | |
| 582 | 32 | int begin0 = program->pixel_offset[x + 0]; | |
| 583 | 32 | int begin1 = program->pixel_offset[x + 1]; | |
| 584 | process_two_pixels_h_float<is_safe>(src, begin0, begin1, current_coeff, filter_size, result0, result1, unaligned_kernel_size); | ||
| 585 | 32 | current_coeff += 2 * filter_size; | |
| 586 | 32 | __m128 sumQuad12 = _mm_hadd_ps(result0, result1); // L1L1L1L1L1L1L1L1 + L2L2L2L2L2L2L2L2L2 = L1L1 L2L2 L1L1 L2L2 | |
| 587 | |||
| 588 | // 2 & 3 | ||
| 589 | 32 | result0 = zero128; | |
| 590 | 32 | result1 = zero128; | |
| 591 | 32 | begin0 = program->pixel_offset[x + 2]; | |
| 592 | 32 | begin1 = program->pixel_offset[x + 3]; | |
| 593 | process_two_pixels_h_float<is_safe>(src, begin0, begin1, current_coeff, filter_size, result0, result1, unaligned_kernel_size); | ||
| 594 | 32 | current_coeff += 2 * filter_size; | |
| 595 | 64 | __m128 sumQuad1234 = _mm_hadd_ps(sumQuad12, _mm_hadd_ps(result0, result1)); | |
| 596 | |||
| 597 | 32 | __m128 result_lo = sumQuad1234; // L1 L2 L3 L4 | |
| 598 | |||
| 599 | // 4 & 5 | ||
| 600 | 32 | result0 = zero128; | |
| 601 | 32 | result1 = zero128; | |
| 602 | 32 | begin0 = program->pixel_offset[x + 4]; | |
| 603 | 32 | begin1 = program->pixel_offset[x + 5]; | |
| 604 | process_two_pixels_h_float<is_safe>(src, begin0, begin1, current_coeff, filter_size, result0, result1, unaligned_kernel_size); | ||
| 605 | 32 | current_coeff += 2 * filter_size; | |
| 606 | 32 | __m128 sumQuad56 = _mm_hadd_ps(result0, result1); // L1L1L1L1L1L1L1L1 + L2L2L2L2L2L2L2L2L2 = L1L1 L2L2 L1L1 L2L2 | |
| 607 | |||
| 608 | // 6 & 7 | ||
| 609 | 32 | result0 = zero128; | |
| 610 | 32 | result1 = zero128; | |
| 611 | 32 | begin0 = program->pixel_offset[x + 6]; | |
| 612 | 32 | begin1 = program->pixel_offset[x + 7]; | |
| 613 | process_two_pixels_h_float<is_safe>(src, begin0, begin1, current_coeff, filter_size, result0, result1, unaligned_kernel_size); | ||
| 614 | //current_coeff += 2 * filter_size; | ||
| 615 | 64 | __m128 sumQuad5678 = _mm_hadd_ps(sumQuad56, _mm_hadd_ps(result0, result1)); | |
| 616 | |||
| 617 | 32 | __m128 result_hi = sumQuad5678; // L1 L2 L3 L4 | |
| 618 | |||
| 619 | 32 | _mm_stream_ps(reinterpret_cast<float*>(dst + x), result_lo); // 8 results at a time | |
| 620 | 32 | _mm_stream_ps(reinterpret_cast<float*>(dst + x + 4), result_hi); // 8 results at a time | |
| 621 | |||
| 622 | 32 | } | |
| 623 | |||
| 624 | #if defined(GCC) || defined(CLANG) | ||
| 625 | __attribute__((__target__("ssse3"))) | ||
| 626 | #endif | ||
| 627 | 2 | void resizer_h_ssse3_generic_float(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) { | |
| 628 | AVS_UNUSED(bits_per_pixel); | ||
| 629 | 2 | int filter_size = program->filter_size; | |
| 630 | 2 | __m128 zero128 = _mm_setzero_ps(); | |
| 631 | |||
| 632 | 2 | const float* src = (float*)src8; | |
| 633 | 2 | float* dst = (float*)dst8; | |
| 634 | 2 | dst_pitch = dst_pitch / sizeof(float); | |
| 635 | 2 | src_pitch = src_pitch / sizeof(float); | |
| 636 | |||
| 637 |
1/2✓ Branch 4 → 5 taken 2 times.
✗ Branch 4 → 6 not taken.
|
2 | const int w_safe_mod8 = (program->safelimit_filter_size_aligned.overread_possible ? program->safelimit_filter_size_aligned.source_overread_beyond_targetx : width) / 8 * 8; |
| 638 | |||
| 639 |
2/2✓ Branch 493 → 8 taken 9 times.
✓ Branch 493 → 494 taken 2 times.
|
11 | for (int y = 0; y < height; y++) { |
| 640 | 9 | float* current_coeff_base = program->pixel_coefficient_float; | |
| 641 | |||
| 642 | // Process safe aligned pixels | ||
| 643 |
2/2✓ Branch 183 → 9 taken 23 times.
✓ Branch 183 → 184 taken 9 times.
|
32 | for (int x = 0; x < w_safe_mod8; x += 8) { |
| 644 | process_eight_pixels_h_float<true>(src, x, current_coeff_base, filter_size, zero128, dst, program); | ||
| 645 | } | ||
| 646 | |||
| 647 | // Process up to the actual kernel size instead of the aligned filter_size to prevent overreading beyond the last source pixel. | ||
| 648 | // We assume extra offset entries were added to the p->pixel_offset array (aligned to 8 during initialization). | ||
| 649 | // This may store 1-7 false pixels, but they are ignored since Avisynth will not read beyond the width. | ||
| 650 |
2/2✓ Branch 491 → 185 taken 9 times.
✓ Branch 491 → 492 taken 9 times.
|
18 | for (int x = w_safe_mod8; x < width; x += 8) { |
| 651 | process_eight_pixels_h_float<false>(src, x, current_coeff_base, filter_size, zero128, dst, program); | ||
| 652 | } | ||
| 653 | |||
| 654 | 9 | dst += dst_pitch; | |
| 655 | 9 | src += src_pitch; | |
| 656 | } | ||
| 657 | 2 | } | |
| 658 | |||
| 659 | //-------- 32 bit float Vertical | ||
| 660 | |||
| 661 | // Process each row with its coefficient | ||
| 662 | 2 | void resize_v_sse2_planar_float(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int target_height, int bits_per_pixel) | |
| 663 | { | ||
| 664 | AVS_UNUSED(bits_per_pixel); | ||
| 665 | |||
| 666 | 2 | const int filter_size = program->filter_size; | |
| 667 | 2 | const float* AVS_RESTRICT current_coeff = program->pixel_coefficient_float; | |
| 668 | |||
| 669 | 2 | const float* src = (const float*)src8; | |
| 670 | 2 | float* AVS_RESTRICT dst = (float*)dst8; | |
| 671 | 2 | dst_pitch = dst_pitch / sizeof(float); | |
| 672 | 2 | src_pitch = src_pitch / sizeof(float); | |
| 673 | |||
| 674 | 2 | const int kernel_size = program->filter_size_real; // not the aligned | |
| 675 | 2 | const int kernel_size_mod2 = (kernel_size / 2) * 2; // Process pairs of rows for better efficiency | |
| 676 | 2 | const bool notMod2 = kernel_size_mod2 < kernel_size; | |
| 677 | |||
| 678 |
2/2✓ Branch 69 → 3 taken 11 times.
✓ Branch 69 → 70 taken 2 times.
|
13 | for (int y = 0; y < target_height; y++) { |
| 679 | 11 | int offset = program->pixel_offset[y]; | |
| 680 | 11 | const float* src_ptr = src + offset * src_pitch; | |
| 681 | |||
| 682 | // use 8 pixels, like AVX2, by utilizing 2x2 ps registers (speed) | ||
| 683 |
2/2✓ Branch 67 → 5 taken 55 times.
✓ Branch 67 → 68 taken 11 times.
|
66 | for (int x = 0; x < width; x += 8) { |
| 684 | 55 | __m128 result_single_even = _mm_setzero_ps(); | |
| 685 | 55 | __m128 result_single_odd = _mm_setzero_ps(); | |
| 686 | 55 | __m128 result_single_even_b = _mm_setzero_ps(); | |
| 687 | 55 | __m128 result_single_odd_b = _mm_setzero_ps(); | |
| 688 | |||
| 689 | 55 | const float* AVS_RESTRICT src2_ptr = src_ptr + x; // __restrict here | |
| 690 | |||
| 691 | // Process pairs of rows for better efficiency (2 coeffs/cycle) | ||
| 692 | 55 | int i = 0; | |
| 693 |
2/2✓ Branch 43 → 14 taken 55 times.
✓ Branch 43 → 44 taken 55 times.
|
110 | for (; i < kernel_size_mod2; i += 2) { |
| 694 | 55 | __m128 coeff_even = _mm_set1_ps(current_coeff[i]); | |
| 695 | 110 | __m128 coeff_odd = _mm_set1_ps(current_coeff[i + 1]); | |
| 696 | |||
| 697 | 55 | __m128 src_even = _mm_loadu_ps(src2_ptr); | |
| 698 | 110 | __m128 src_odd = _mm_loadu_ps(src2_ptr + src_pitch); | |
| 699 | |||
| 700 | 55 | __m128 mul_even = _mm_mul_ps(src_even, coeff_even); | |
| 701 | 55 | __m128 mul_odd = _mm_mul_ps(src_odd, coeff_odd); | |
| 702 | |||
| 703 | 55 | result_single_even = _mm_add_ps(result_single_even, mul_even); | |
| 704 | 55 | result_single_odd = _mm_add_ps(result_single_odd, mul_odd); | |
| 705 | |||
| 706 | 55 | __m128 src_even_b = _mm_loadu_ps(src2_ptr + 4); | |
| 707 | 110 | __m128 src_odd_b = _mm_loadu_ps(src2_ptr + 4 + src_pitch); | |
| 708 | |||
| 709 | 55 | __m128 mul_even_b = _mm_mul_ps(src_even_b, coeff_even); | |
| 710 | 55 | __m128 mul_odd_b = _mm_mul_ps(src_odd_b, coeff_odd); | |
| 711 | |||
| 712 | 55 | result_single_even_b = _mm_add_ps(result_single_even_b, mul_even_b); | |
| 713 | 55 | result_single_odd_b = _mm_add_ps(result_single_odd_b, mul_odd_b); | |
| 714 | |||
| 715 | 55 | src2_ptr += 2 * src_pitch; | |
| 716 | } | ||
| 717 | |||
| 718 | 55 | result_single_even = _mm_add_ps(result_single_even, result_single_odd); | |
| 719 | 55 | result_single_even_b = _mm_add_ps(result_single_even_b, result_single_odd_b); | |
| 720 | |||
| 721 | // Process the last odd row if needed | ||
| 722 |
1/2✓ Branch 48 → 49 taken 55 times.
✗ Branch 48 → 64 not taken.
|
55 | if (notMod2) { |
| 723 | 110 | __m128 coeff = _mm_set1_ps(current_coeff[i]); | |
| 724 | 55 | __m128 src_val = _mm_loadu_ps(src2_ptr); | |
| 725 | 110 | __m128 src_val_b = _mm_loadu_ps(src2_ptr + 4); | |
| 726 | |||
| 727 | 110 | result_single_even = _mm_add_ps(result_single_even, _mm_mul_ps(src_val, coeff)); | |
| 728 | 110 | result_single_even_b = _mm_add_ps(result_single_even_b, _mm_mul_ps(src_val_b, coeff)); | |
| 729 | } | ||
| 730 | |||
| 731 | // Store result | ||
| 732 | 55 | _mm_stream_ps(dst + x, result_single_even); | |
| 733 | 55 | _mm_stream_ps(dst + x + 4, result_single_even_b); | |
| 734 | } | ||
| 735 | |||
| 736 | 11 | dst += dst_pitch; | |
| 737 | 11 | current_coeff += filter_size; | |
| 738 | } | ||
| 739 | 2 | } | |
| 740 | |||
| 741 | // ----------------------------------------------- | ||
| 742 | // 8 bit Horizontal. | ||
| 743 | // Dual line processing, use template until alignment and end conditions allow. | ||
| 744 | |||
| 745 | // Based on AVX2 code, but without the filter_size alignment template | ||
| 746 | |||
| 747 | template<typename pixel_t, bool lessthan16bit> | ||
| 748 | AVS_FORCEINLINE static void process_two_16pixels_h_uint8_16_core(const pixel_t* AVS_RESTRICT src, int begin1, int begin2, int i, const short* AVS_RESTRICT current_coeff, int filter_size, __m128i& result1, __m128i& result2, | ||
| 749 | const __m128i& shifttosigned_or_zero128) { | ||
| 750 | |||
| 751 | __m128i data_1_lo, data_1_hi, data_2_lo, data_2_hi; | ||
| 752 | |||
| 753 | if constexpr (sizeof(pixel_t) == 1) { | ||
| 754 | // pixel_t is uint8_t | ||
| 755 | 3624 | __m128i data_1 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src + begin1 + i)); | |
| 756 | 1812 | __m128i data_2 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src + begin2 + i)); | |
| 757 | |||
| 758 | 1812 | data_1_lo = _mm_unpacklo_epi8(data_1, shifttosigned_or_zero128); | |
| 759 | 1812 | data_1_hi = _mm_unpackhi_epi8(data_1, shifttosigned_or_zero128); | |
| 760 | 1812 | data_2_lo = _mm_unpacklo_epi8(data_2, shifttosigned_or_zero128); | |
| 761 | 3624 | data_2_hi = _mm_unpackhi_epi8(data_2, shifttosigned_or_zero128); | |
| 762 | } | ||
| 763 | else { | ||
| 764 | // pixel_t is uint16_t, at exact 16 bit size an unsigned -> signed 16 bit conversion needed | ||
| 765 | 7184 | data_1_lo = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src + begin1 + i)); | |
| 766 | 3592 | data_1_hi = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src + begin1 + i + 8)); | |
| 767 | if constexpr (!lessthan16bit) { | ||
| 768 | 1812 | data_1_lo = _mm_add_epi16(data_1_lo, shifttosigned_or_zero128); | |
| 769 | 1812 | data_1_hi = _mm_add_epi16(data_1_hi, shifttosigned_or_zero128); | |
| 770 | } | ||
| 771 | 3592 | data_2_lo = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src + begin2 + i)); | |
| 772 | 5372 | data_2_hi = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src + begin2 + i + 8)); | |
| 773 | if constexpr (!lessthan16bit) { | ||
| 774 | 1812 | data_2_lo = _mm_add_epi16(data_2_lo, shifttosigned_or_zero128); | |
| 775 | 3624 | data_2_hi = _mm_add_epi16(data_2_hi, shifttosigned_or_zero128); | |
| 776 | } | ||
| 777 | } | ||
| 778 | |||
| 779 | 5404 | __m128i coeff_1_lo = _mm_load_si128(reinterpret_cast<const __m128i*>(current_coeff)); // 8 coeffs | |
| 780 | 5404 | __m128i coeff_1_hi = _mm_load_si128(reinterpret_cast<const __m128i*>(current_coeff + 8)); // next 8 coeffs | |
| 781 | 5404 | __m128i coeff_2_lo = _mm_load_si128(reinterpret_cast<const __m128i*>(current_coeff + 1 * filter_size)); // 8x second pixel's coefficients | |
| 782 | 10808 | __m128i coeff_2_hi = _mm_load_si128(reinterpret_cast<const __m128i*>(current_coeff + 1 * filter_size + 8)); // next 8x second pixel's coefficients | |
| 783 | |||
| 784 | 10808 | result1 = _mm_add_epi32(result1, _mm_madd_epi16(data_1_lo, coeff_1_lo)); | |
| 785 | 10808 | result1 = _mm_add_epi32(result1, _mm_madd_epi16(data_1_hi, coeff_1_hi)); | |
| 786 | 10808 | result2 = _mm_add_epi32(result2, _mm_madd_epi16(data_2_lo, coeff_2_lo)); | |
| 787 | 5404 | result2 = _mm_add_epi32(result2, _mm_madd_epi16(data_2_hi, coeff_2_hi)); | |
| 788 | 5404 | } | |
| 789 | |||
| 790 | template<bool safe_aligned_mode, typename pixel_t, bool lessthan16bit> | ||
| 791 | AVS_FORCEINLINE static void process_two_pixels_h_uint8_16(const pixel_t* AVS_RESTRICT src_ptr, int begin1, int begin2, const short* AVS_RESTRICT current_coeff, int filter_size, __m128i& result1, __m128i& result2, int kernel_size, | ||
| 792 | const __m128i& shifttosigned_or_zero128) { | ||
| 793 | int ksmod16; | ||
| 794 | if constexpr (safe_aligned_mode) | ||
| 795 | 3964 | ksmod16 = filter_size / 16 * 16; | |
| 796 | else | ||
| 797 | 692 | ksmod16 = kernel_size / 16 * 16; // danger zone, scanline overread possible. Use exact unaligned kernel_size | |
| 798 | 4656 | const pixel_t* src_ptr1 = src_ptr + begin1; | |
| 799 | 4656 | const pixel_t* src_ptr2 = src_ptr + begin2; | |
| 800 | 4656 | int i = 0; | |
| 801 | |||
| 802 | // Process 16 elements at a time | ||
| 803 |
48/48✓ Branch 58 → 24 taken 425 times.
✓ Branch 58 → 59 taken 325 times.
✓ Branch 60 → 22 taken 433 times.
✓ Branch 60 → 61 taken 333 times.
✓ Branch 66 → 24 taken 433 times.
✓ Branch 66 → 67 taken 333 times.
✓ Branch 99 → 65 taken 425 times.
✓ Branch 99 → 100 taken 325 times.
✓ Branch 105 → 67 taken 433 times.
✓ Branch 105 → 106 taken 333 times.
✓ Branch 115 → 73 taken 433 times.
✓ Branch 115 → 116 taken 333 times.
✓ Branch 142 → 108 taken 425 times.
✓ Branch 142 → 143 taken 325 times.
✓ Branch 152 → 114 taken 433 times.
✓ Branch 152 → 153 taken 333 times.
✓ Branch 166 → 124 taken 433 times.
✓ Branch 166 → 167 taken 333 times.
✓ Branch 183 → 149 taken 425 times.
✓ Branch 183 → 184 taken 325 times.
✓ Branch 197 → 159 taken 433 times.
✓ Branch 197 → 198 taken 333 times.
✓ Branch 215 → 173 taken 433 times.
✓ Branch 215 → 216 taken 333 times.
✓ Branch 273 → 239 taken 20 times.
✓ Branch 273 → 274 taken 55 times.
✓ Branch 275 → 237 taken 20 times.
✓ Branch 275 → 276 taken 59 times.
✓ Branch 303 → 261 taken 20 times.
✓ Branch 303 → 304 taken 59 times.
✓ Branch 382 → 348 taken 20 times.
✓ Branch 382 → 383 taken 55 times.
✓ Branch 400 → 362 taken 20 times.
✓ Branch 400 → 401 taken 59 times.
✓ Branch 428 → 386 taken 20 times.
✓ Branch 428 → 429 taken 59 times.
✓ Branch 493 → 459 taken 20 times.
✓ Branch 493 → 494 taken 55 times.
✓ Branch 527 → 489 taken 20 times.
✓ Branch 527 → 528 taken 59 times.
✓ Branch 555 → 513 taken 20 times.
✓ Branch 555 → 556 taken 59 times.
✓ Branch 602 → 568 taken 20 times.
✓ Branch 602 → 603 taken 55 times.
✓ Branch 652 → 614 taken 20 times.
✓ Branch 652 → 653 taken 59 times.
✓ Branch 680 → 638 taken 20 times.
✓ Branch 680 → 681 taken 59 times.
|
10060 | for (; i < ksmod16; i += 16) { |
| 804 | 5404 | process_two_16pixels_h_uint8_16_core<pixel_t, lessthan16bit>(src_ptr, begin1, begin2, i, current_coeff + i, filter_size, result1, result2, shifttosigned_or_zero128); | |
| 805 | } | ||
| 806 | |||
| 807 | if constexpr (!safe_aligned_mode) { | ||
| 808 | // working with the original, unaligned kernel_size | ||
| 809 |
12/24✗ Branch 274 → 275 not taken.
✓ Branch 274 → 276 taken 55 times.
✗ Branch 276 → 277 not taken.
✓ Branch 276 → 278 taken 59 times.
✗ Branch 304 → 305 not taken.
✓ Branch 304 → 306 taken 59 times.
✗ Branch 383 → 384 not taken.
✓ Branch 383 → 385 taken 55 times.
✗ Branch 401 → 402 not taken.
✓ Branch 401 → 403 taken 59 times.
✗ Branch 429 → 430 not taken.
✓ Branch 429 → 431 taken 59 times.
✗ Branch 494 → 495 not taken.
✓ Branch 494 → 496 taken 55 times.
✗ Branch 528 → 529 not taken.
✓ Branch 528 → 530 taken 59 times.
✗ Branch 556 → 557 not taken.
✓ Branch 556 → 558 taken 59 times.
✗ Branch 603 → 604 not taken.
✓ Branch 603 → 605 taken 55 times.
✗ Branch 653 → 654 not taken.
✓ Branch 653 → 655 taken 59 times.
✗ Branch 681 → 682 not taken.
✓ Branch 681 → 683 taken 59 times.
|
692 | if (i == kernel_size) return; |
| 810 | |||
| 811 | 692 | const short* current_coeff2 = current_coeff + filter_size; // Points to second pixel's coefficients | |
| 812 | 692 | const int ksmod8 = kernel_size / 8 * 8; | |
| 813 | 692 | const int ksmod4 = kernel_size / 4 * 4; | |
| 814 | |||
| 815 | // Process 8 elements if needed | ||
| 816 |
24/24✓ Branch 276 → 277 taken 10 times.
✓ Branch 276 → 295 taken 45 times.
✓ Branch 278 → 279 taken 10 times.
✓ Branch 278 → 309 taken 49 times.
✓ Branch 306 → 307 taken 10 times.
✓ Branch 306 → 329 taken 49 times.
✓ Branch 385 → 386 taken 10 times.
✓ Branch 385 → 404 taken 45 times.
✓ Branch 403 → 404 taken 10 times.
✓ Branch 403 → 434 taken 49 times.
✓ Branch 431 → 432 taken 10 times.
✓ Branch 431 → 454 taken 49 times.
✓ Branch 496 → 497 taken 10 times.
✓ Branch 496 → 515 taken 45 times.
✓ Branch 530 → 531 taken 10 times.
✓ Branch 530 → 561 taken 49 times.
✓ Branch 558 → 559 taken 10 times.
✓ Branch 558 → 581 taken 49 times.
✓ Branch 605 → 606 taken 10 times.
✓ Branch 605 → 624 taken 45 times.
✓ Branch 655 → 656 taken 10 times.
✓ Branch 655 → 686 taken 49 times.
✓ Branch 683 → 684 taken 10 times.
✓ Branch 683 → 706 taken 49 times.
|
692 | if (i < ksmod8) { |
| 817 | // Process 8 elements for first pixel | ||
| 818 | __m128i data_1; | ||
| 819 | if constexpr(sizeof(pixel_t) == 1) | ||
| 820 | 80 | data_1 = _mm_unpacklo_epi8(_mm_loadl_epi64(reinterpret_cast<const __m128i*>(src_ptr1 + i)), shifttosigned_or_zero128); | |
| 821 | else { | ||
| 822 | // uint16_t | ||
| 823 | 80 | data_1 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src_ptr1 + i)); | |
| 824 | if constexpr (!lessthan16bit) | ||
| 825 | 40 | data_1 = _mm_add_epi16(data_1, shifttosigned_or_zero128); // unsigned -> signed | |
| 826 | } | ||
| 827 | |||
| 828 | 240 | __m128i coeff_1 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(current_coeff + i)); | |
| 829 | 120 | __m128i temp_result1 = _mm_madd_epi16(data_1, coeff_1); | |
| 830 | |||
| 831 | // Process 8 elements for second pixel | ||
| 832 | __m128i data_2; | ||
| 833 | if constexpr (sizeof(pixel_t) == 1) | ||
| 834 | 80 | data_2 = _mm_unpacklo_epi8(_mm_loadl_epi64(reinterpret_cast<const __m128i*>(src_ptr2 + i)), shifttosigned_or_zero128); | |
| 835 | else { | ||
| 836 | // uint16_t | ||
| 837 | 80 | data_2 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src_ptr2 + i)); | |
| 838 | if constexpr (!lessthan16bit) | ||
| 839 | 40 | data_2 = _mm_add_epi16(data_2, shifttosigned_or_zero128); // unsigned -> signed | |
| 840 | } | ||
| 841 | |||
| 842 | 240 | __m128i coeff_2 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(current_coeff2 + i)); | |
| 843 | 120 | __m128i temp_result2 = _mm_madd_epi16(data_2, coeff_2); | |
| 844 | |||
| 845 | // update result vectors | ||
| 846 | 120 | result1 = _mm_add_epi32(result1, temp_result1); | |
| 847 | 120 | result2 = _mm_add_epi32(result2, temp_result2); | |
| 848 | |||
| 849 | 120 | i += 8; | |
| 850 |
12/24✗ Branch 293 → 294 not taken.
✓ Branch 293 → 295 taken 10 times.
✗ Branch 307 → 308 not taken.
✓ Branch 307 → 309 taken 10 times.
✗ Branch 327 → 328 not taken.
✓ Branch 327 → 329 taken 10 times.
✗ Branch 402 → 403 not taken.
✓ Branch 402 → 404 taken 10 times.
✗ Branch 432 → 433 not taken.
✓ Branch 432 → 434 taken 10 times.
✗ Branch 452 → 453 not taken.
✓ Branch 452 → 454 taken 10 times.
✗ Branch 513 → 514 not taken.
✓ Branch 513 → 515 taken 10 times.
✗ Branch 559 → 560 not taken.
✓ Branch 559 → 561 taken 10 times.
✗ Branch 579 → 580 not taken.
✓ Branch 579 → 581 taken 10 times.
✗ Branch 622 → 623 not taken.
✓ Branch 622 → 624 taken 10 times.
✗ Branch 684 → 685 not taken.
✓ Branch 684 → 686 taken 10 times.
✗ Branch 704 → 705 not taken.
✓ Branch 704 → 706 taken 10 times.
|
120 | if (i == kernel_size) return; |
| 851 | } | ||
| 852 | |||
| 853 | // Process 4 elements if needed | ||
| 854 |
24/24✓ Branch 295 → 296 taken 45 times.
✓ Branch 295 → 330 taken 10 times.
✓ Branch 309 → 310 taken 49 times.
✓ Branch 309 → 344 taken 10 times.
✓ Branch 329 → 330 taken 49 times.
✓ Branch 329 → 368 taken 10 times.
✓ Branch 404 → 405 taken 45 times.
✓ Branch 404 → 439 taken 10 times.
✓ Branch 434 → 435 taken 49 times.
✓ Branch 434 → 469 taken 10 times.
✓ Branch 454 → 455 taken 49 times.
✓ Branch 454 → 493 taken 10 times.
✓ Branch 515 → 516 taken 45 times.
✓ Branch 515 → 550 taken 10 times.
✓ Branch 561 → 562 taken 49 times.
✓ Branch 561 → 596 taken 10 times.
✓ Branch 581 → 582 taken 49 times.
✓ Branch 581 → 620 taken 10 times.
✓ Branch 624 → 625 taken 45 times.
✓ Branch 624 → 659 taken 10 times.
✓ Branch 686 → 687 taken 49 times.
✓ Branch 686 → 721 taken 10 times.
✓ Branch 706 → 707 taken 49 times.
✓ Branch 706 → 745 taken 10 times.
|
692 | if (i < ksmod4) { |
| 855 | // Process 4 elements for first pixel | ||
| 856 | __m128i data_1; | ||
| 857 | if constexpr (sizeof(pixel_t) == 1) | ||
| 858 | 392 | data_1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*reinterpret_cast<const int*>(src_ptr1 + i)), shifttosigned_or_zero128); | |
| 859 | else { | ||
| 860 | 376 | data_1 = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(src_ptr1 + i)); | |
| 861 | if constexpr (!lessthan16bit) | ||
| 862 | 196 | data_1 = _mm_add_epi16(data_1, shifttosigned_or_zero128); // unsigned -> signed | |
| 863 | } | ||
| 864 | |||
| 865 | 1144 | __m128i coeff_1 = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(current_coeff + i)); | |
| 866 | 572 | __m128i temp_result1 = _mm_madd_epi16(data_1, coeff_1); | |
| 867 | |||
| 868 | // Process 4 elements for second pixel | ||
| 869 | __m128i data_2; | ||
| 870 | if constexpr (sizeof(pixel_t) == 1) | ||
| 871 | 392 | data_2 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(*reinterpret_cast<const int*>(src_ptr2 + i)), shifttosigned_or_zero128); | |
| 872 | else { | ||
| 873 | 376 | data_2 = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(src_ptr2 + i)); | |
| 874 | if constexpr (!lessthan16bit) | ||
| 875 | 196 | data_2 = _mm_add_epi16(data_2, shifttosigned_or_zero128); // unsigned -> signed | |
| 876 | } | ||
| 877 | 1144 | __m128i coeff_2 = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(current_coeff2 + i)); | |
| 878 | 572 | __m128i temp_result2 = _mm_madd_epi16(data_2, coeff_2); | |
| 879 | |||
| 880 | // update result vectors | ||
| 881 | 572 | result1 = _mm_add_epi32(result1, temp_result1); | |
| 882 | 572 | result2 = _mm_add_epi32(result2, temp_result2); | |
| 883 | |||
| 884 | 572 | i += 4; | |
| 885 |
24/24✓ Branch 328 → 329 taken 30 times.
✓ Branch 328 → 330 taken 15 times.
✓ Branch 342 → 343 taken 34 times.
✓ Branch 342 → 344 taken 15 times.
✓ Branch 366 → 367 taken 34 times.
✓ Branch 366 → 368 taken 15 times.
✓ Branch 437 → 438 taken 30 times.
✓ Branch 437 → 439 taken 15 times.
✓ Branch 467 → 468 taken 34 times.
✓ Branch 467 → 469 taken 15 times.
✓ Branch 491 → 492 taken 34 times.
✓ Branch 491 → 493 taken 15 times.
✓ Branch 548 → 549 taken 30 times.
✓ Branch 548 → 550 taken 15 times.
✓ Branch 594 → 595 taken 34 times.
✓ Branch 594 → 596 taken 15 times.
✓ Branch 618 → 619 taken 34 times.
✓ Branch 618 → 620 taken 15 times.
✓ Branch 657 → 658 taken 30 times.
✓ Branch 657 → 659 taken 15 times.
✓ Branch 719 → 720 taken 34 times.
✓ Branch 719 → 721 taken 15 times.
✓ Branch 743 → 744 taken 34 times.
✓ Branch 743 → 745 taken 15 times.
|
572 | if (i == kernel_size) return; |
| 886 | } | ||
| 887 | |||
| 888 | // Process remaining elements with scalar operations | ||
| 889 |
12/24✓ Branch 330 → 331 taken 25 times.
✗ Branch 330 → 343 not taken.
✓ Branch 344 → 345 taken 25 times.
✗ Branch 344 → 357 not taken.
✓ Branch 368 → 369 taken 25 times.
✗ Branch 368 → 381 not taken.
✓ Branch 439 → 440 taken 25 times.
✗ Branch 439 → 452 not taken.
✓ Branch 469 → 470 taken 25 times.
✗ Branch 469 → 482 not taken.
✓ Branch 493 → 494 taken 25 times.
✗ Branch 493 → 506 not taken.
✓ Branch 550 → 551 taken 25 times.
✗ Branch 550 → 563 not taken.
✓ Branch 596 → 597 taken 25 times.
✗ Branch 596 → 609 not taken.
✓ Branch 620 → 621 taken 25 times.
✗ Branch 620 → 633 not taken.
✓ Branch 659 → 660 taken 25 times.
✗ Branch 659 → 672 not taken.
✓ Branch 721 → 722 taken 25 times.
✗ Branch 721 → 734 not taken.
✓ Branch 745 → 746 taken 25 times.
✗ Branch 745 → 758 not taken.
|
300 | if (i < kernel_size) { |
| 890 | 300 | int scalar_sum1[4] = { 0, 0, 0, 0 }; // like an __m128i | |
| 891 | 300 | int scalar_sum2[4] = { 0, 0, 0, 0 }; | |
| 892 | |||
| 893 |
24/24✓ Branch 333 → 332 taken 60 times.
✓ Branch 333 → 334 taken 25 times.
✓ Branch 347 → 346 taken 60 times.
✓ Branch 347 → 348 taken 25 times.
✓ Branch 371 → 370 taken 60 times.
✓ Branch 371 → 372 taken 25 times.
✓ Branch 442 → 441 taken 60 times.
✓ Branch 442 → 443 taken 25 times.
✓ Branch 472 → 471 taken 60 times.
✓ Branch 472 → 473 taken 25 times.
✓ Branch 496 → 495 taken 60 times.
✓ Branch 496 → 497 taken 25 times.
✓ Branch 553 → 552 taken 60 times.
✓ Branch 553 → 554 taken 25 times.
✓ Branch 599 → 598 taken 60 times.
✓ Branch 599 → 600 taken 25 times.
✓ Branch 623 → 622 taken 60 times.
✓ Branch 623 → 624 taken 25 times.
✓ Branch 662 → 661 taken 60 times.
✓ Branch 662 → 663 taken 25 times.
✓ Branch 724 → 723 taken 60 times.
✓ Branch 724 → 725 taken 25 times.
✓ Branch 748 → 747 taken 60 times.
✓ Branch 748 → 749 taken 25 times.
|
1020 | for (; i < kernel_size; i++) { |
| 894 | if constexpr (sizeof(pixel_t) == 1) { | ||
| 895 | 240 | scalar_sum1[i % 4] += src_ptr1[i] * current_coeff[i]; | |
| 896 | 240 | scalar_sum2[i % 4] += src_ptr2[i] * current_coeff2[i]; | |
| 897 | } else { | ||
| 898 | 480 | uint16_t pix1 = src_ptr1[i]; | |
| 899 | 480 | uint16_t pix2 = src_ptr2[i]; | |
| 900 | |||
| 901 | if constexpr (!lessthan16bit) { | ||
| 902 | 240 | pix1 -= 32768; | |
| 903 | 240 | pix2 -= 32768; | |
| 904 | } | ||
| 905 | |||
| 906 | 480 | scalar_sum1[i % 4] += (short)pix1 * current_coeff[i]; | |
| 907 | 480 | scalar_sum2[i % 4] += (short)pix2 * current_coeff2[i]; | |
| 908 | } | ||
| 909 | } | ||
| 910 | |||
| 911 | // Convert scalar results to SIMD and add to result vectors | ||
| 912 | 300 | __m128i temp_result1 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(scalar_sum1)); | |
| 913 | 300 | __m128i temp_result2 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(scalar_sum2)); | |
| 914 | |||
| 915 | // update result vectors | ||
| 916 | 300 | result1 = _mm_add_epi32(result1, temp_result1); | |
| 917 | 600 | result2 = _mm_add_epi32(result2, temp_result2); | |
| 918 | } | ||
| 919 | } | ||
| 920 | 3964 | } | |
| 921 | |||
| 922 | template<bool is_safe, typename pixel_t, bool lessthan16bit> | ||
| 923 | #if defined(GCC) || defined(CLANG) | ||
| 924 | __attribute__((__target__("ssse3"))) | ||
| 925 | #endif | ||
| 926 | AVS_FORCEINLINE static void process_eight_pixels_h_uint8_16(const pixel_t* AVS_RESTRICT src, int x, const short* current_coeff_base, int filter_size, | ||
| 927 | __m128i& rounder128, __m128i& shifttosigned_or_zero128, __m128i& clamp_limit, | ||
| 928 | pixel_t* AVS_RESTRICT dst, | ||
| 929 | ResamplingProgram* program) | ||
| 930 | { | ||
| 931 |
5/10✗ Branch 17 → 18 not taken.
✓ Branch 17 → 19 taken 333 times.
✗ Branch 19 → 20 not taken.
✓ Branch 19 → 21 taken 658 times.
✗ Branch 232 → 233 not taken.
✓ Branch 232 → 234 taken 59 times.
✗ Branch 234 → 235 not taken.
✓ Branch 234 → 236 taken 55 times.
✗ Branch 256 → 257 not taken.
✓ Branch 256 → 258 taken 59 times.
|
1164 | assert(program->filter_size_alignment >= 16); // code assumes this |
| 932 | |||
| 933 | 1164 | const short* AVS_RESTRICT current_coeff = current_coeff_base + x * filter_size; | |
| 934 | 1164 | const int unaligned_kernel_size = program->filter_size_real; | |
| 935 | |||
| 936 | // Unrolled processing of all 8 pixels | ||
| 937 | |||
| 938 | // 0 & 1 | ||
| 939 | 1164 | __m128i result0 = rounder128; | |
| 940 | 1164 | __m128i result1 = rounder128; | |
| 941 | 1164 | int begin0 = program->pixel_offset[x + 0]; | |
| 942 | 1164 | int begin1 = program->pixel_offset[x + 1]; | |
| 943 | process_two_pixels_h_uint8_16<is_safe, pixel_t, lessthan16bit>(src, begin0, begin1, current_coeff, filter_size, result0, result1, unaligned_kernel_size, shifttosigned_or_zero128); | ||
| 944 | 1164 | current_coeff += 2 * filter_size; | |
| 945 | 1164 | __m128i sumQuad12 = _mm_hadd_epi32(result0, result1); | |
| 946 | |||
| 947 | // 2 & 3 | ||
| 948 | 1164 | result0 = rounder128; | |
| 949 | 1164 | result1 = rounder128; | |
| 950 | 1164 | begin0 = program->pixel_offset[x + 2]; | |
| 951 | 1164 | begin1 = program->pixel_offset[x + 3]; | |
| 952 | process_two_pixels_h_uint8_16<is_safe, pixel_t, lessthan16bit>(src, begin0, begin1, current_coeff, filter_size, result0, result1, unaligned_kernel_size, shifttosigned_or_zero128); | ||
| 953 | 1164 | current_coeff += 2 * filter_size; | |
| 954 | 2328 | __m128i sumQuad1234 = _mm_hadd_epi32(sumQuad12, _mm_hadd_epi32(result0, result1)); | |
| 955 | |||
| 956 | // 4 & 5 | ||
| 957 | 1164 | result0 = rounder128; | |
| 958 | 1164 | result1 = rounder128; | |
| 959 | 1164 | begin0 = program->pixel_offset[x + 4]; | |
| 960 | 1164 | begin1 = program->pixel_offset[x + 5]; | |
| 961 | process_two_pixels_h_uint8_16<is_safe, pixel_t, lessthan16bit>(src, begin0, begin1, current_coeff, filter_size, result0, result1, unaligned_kernel_size, shifttosigned_or_zero128); | ||
| 962 | 1164 | current_coeff += 2 * filter_size; | |
| 963 | 1164 | __m128i sumQuad56 = _mm_hadd_epi32(result0, result1); | |
| 964 | |||
| 965 | // 6 & 7 | ||
| 966 | 1164 | result0 = rounder128; | |
| 967 | 1164 | result1 = rounder128; | |
| 968 | 1164 | begin0 = program->pixel_offset[x + 6]; | |
| 969 | 1164 | begin1 = program->pixel_offset[x + 7]; | |
| 970 | process_two_pixels_h_uint8_16<is_safe, pixel_t, lessthan16bit>(src, begin0, begin1, current_coeff, filter_size, result0, result1, unaligned_kernel_size, shifttosigned_or_zero128); | ||
| 971 | //current_coeff += 2 * filter_size; | ||
| 972 | 2328 | __m128i sumQuad5678 = _mm_hadd_epi32(sumQuad56, _mm_hadd_epi32(result0, result1)); | |
| 973 | |||
| 974 | 1164 | __m128i pix1234 = sumQuad1234; | |
| 975 | 392 | __m128i pix5678 = sumQuad5678; | |
| 976 | |||
| 977 | // correct if signed, scale back, store | ||
| 978 | if constexpr (sizeof(pixel_t) == 2 && !lessthan16bit) { | ||
| 979 | 392 | const __m128i shiftfromsigned = _mm_set1_epi32(+32768 << FPScale16bits); // yes, 32 bit data. for 16 bits only | |
| 980 | 392 | pix1234 = _mm_add_epi32(pix1234, shiftfromsigned); | |
| 981 | 392 | pix5678 = _mm_add_epi32(pix5678, shiftfromsigned); | |
| 982 | } | ||
| 983 | |||
| 984 | 1164 | const int current_fp_scale_bits = (sizeof(pixel_t) == 1) ? FPScale8bits : FPScale16bits; | |
| 985 | // scale back, shuffle, store | ||
| 986 | 1164 | __m128i result1234 = _mm_srai_epi32(pix1234, current_fp_scale_bits); | |
| 987 | 1164 | __m128i result5678 = _mm_srai_epi32(pix5678, current_fp_scale_bits); | |
| 988 | 1164 | __m128i result_2x4x_uint16_128 = _MM_PACKUS_EPI32(result1234, result5678); | |
| 989 | if constexpr (sizeof(pixel_t) == 1) { | ||
| 990 | 392 | __m128i result_2x4x_uint8 = _mm_packus_epi16(result_2x4x_uint16_128, shifttosigned_or_zero128); | |
| 991 | 392 | _mm_storel_epi64(reinterpret_cast<__m128i*>(dst + x), result_2x4x_uint8); | |
| 992 | } | ||
| 993 | else { | ||
| 994 | // uint16_t | ||
| 995 | if constexpr (lessthan16bit) | ||
| 996 | 380 | result_2x4x_uint16_128 = _MM_MIN_EPU16(result_2x4x_uint16_128, clamp_limit); // extra clamp for 10-14 bits | |
| 997 | |||
| 998 | 772 | _mm_store_si128(reinterpret_cast<__m128i*>(dst + x), result_2x4x_uint16_128); | |
| 999 | |||
| 1000 | } | ||
| 1001 | 1164 | } | |
| 1002 | |||
| 1003 | //-------- uint8/16_t Horizontal | ||
| 1004 | // 4 pixels at a time. | ||
| 1005 | // ssse3: _mm_hadd_epi32 | ||
| 1006 | template<typename pixel_t, bool lessthan16bit> | ||
| 1007 | #if defined(GCC) || defined(CLANG) | ||
| 1008 | __attribute__((__target__("ssse3"))) | ||
| 1009 | #endif | ||
| 1010 | 14 | void resizer_h_ssse3_generic_uint8_16(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) { | |
| 1011 | 14 | int filter_size = program->filter_size; | |
| 1012 | 14 | const int current_fp_scale_bits = (sizeof(pixel_t) == 1) ? FPScale8bits : FPScale16bits; | |
| 1013 | 14 | __m128i rounder128 = _mm_setr_epi32(1 << (current_fp_scale_bits - 1), 0, 0, 0); | |
| 1014 | |||
| 1015 | __m128i shifttosigned_or_zero128; | ||
| 1016 | if constexpr(sizeof(pixel_t) == 1) | ||
| 1017 | 5 | shifttosigned_or_zero128 = _mm_setzero_si128(); | |
| 1018 | else | ||
| 1019 | 9 | shifttosigned_or_zero128 = _mm_set1_epi16(-32768); // for 16 bits only | |
| 1020 | 14 | __m128i clamp_limit = _mm_set1_epi16((short)((1 << bits_per_pixel) - 1)); // clamp limit for 8< <16 bits | |
| 1021 | |||
| 1022 | 14 | const pixel_t* src = reinterpret_cast<const pixel_t* AVS_RESTRICT>(src8); | |
| 1023 | 14 | pixel_t* dst = reinterpret_cast<pixel_t* AVS_RESTRICT>(dst8); | |
| 1024 | 14 | dst_pitch /= sizeof(pixel_t); | |
| 1025 | 14 | src_pitch /= sizeof(pixel_t); | |
| 1026 | |||
| 1027 |
3/6void resizer_h_ssse3_generic_uint8_16<unsigned char, true>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 12 → 13 taken 5 times.
✗ Branch 12 → 14 not taken.
void resizer_h_ssse3_generic_uint8_16<unsigned short, false>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 14 → 15 taken 5 times.
✗ Branch 14 → 16 not taken.
void resizer_h_ssse3_generic_uint8_16<unsigned short, true>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 14 → 15 taken 4 times.
✗ Branch 14 → 16 not taken.
|
14 | const int w_safe_mod8 = (program->safelimit_filter_size_aligned.overread_possible ? program->safelimit_filter_size_aligned.source_overread_beyond_targetx : width) / 8 * 8; |
| 1028 | |||
| 1029 |
6/6void resizer_h_ssse3_generic_uint8_16<unsigned char, true>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 767 → 16 taken 24 times.
✓ Branch 767 → 768 taken 5 times.
void resizer_h_ssse3_generic_uint8_16<unsigned short, false>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 797 → 18 taken 24 times.
✓ Branch 797 → 798 taken 5 times.
void resizer_h_ssse3_generic_uint8_16<unsigned short, true>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 721 → 18 taken 20 times.
✓ Branch 721 → 722 taken 4 times.
|
82 | for (int y = 0; y < height; y++) { |
| 1030 | 68 | const short* AVS_RESTRICT current_coeff_base = program->pixel_coefficient; | |
| 1031 | |||
| 1032 | // Process safe aligned pixels | ||
| 1033 |
6/6void resizer_h_ssse3_generic_uint8_16<unsigned char, true>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 230 → 17 taken 333 times.
✓ Branch 230 → 231 taken 24 times.
void resizer_h_ssse3_generic_uint8_16<unsigned short, false>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 254 → 19 taken 333 times.
✓ Branch 254 → 255 taken 24 times.
void resizer_h_ssse3_generic_uint8_16<unsigned short, true>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 232 → 19 taken 325 times.
✓ Branch 232 → 233 taken 20 times.
|
1059 | for (int x = 0; x < w_safe_mod8; x += 8) { |
| 1034 | process_eight_pixels_h_uint8_16<true, pixel_t, lessthan16bit>(src, x, current_coeff_base, filter_size, rounder128, shifttosigned_or_zero128, clamp_limit, dst, program); | ||
| 1035 | } | ||
| 1036 | |||
| 1037 | // Process up to the actual kernel size instead of the aligned filter_size to prevent overreading beyond the last source pixel. | ||
| 1038 | // We assume extra offset entries were added to the p->pixel_offset array (aligned to 8 during initialization). | ||
| 1039 | // This may store 1-7 false pixels, but they are ignored since Avisynth will not read beyond the width. | ||
| 1040 |
6/6void resizer_h_ssse3_generic_uint8_16<unsigned char, true>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 765 → 232 taken 59 times.
✓ Branch 765 → 766 taken 24 times.
void resizer_h_ssse3_generic_uint8_16<unsigned short, false>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 795 → 256 taken 59 times.
✓ Branch 795 → 796 taken 24 times.
void resizer_h_ssse3_generic_uint8_16<unsigned short, true>(unsigned char*, unsigned char const*, int, int, ResamplingProgram*, int, int, int):
✓ Branch 719 → 234 taken 55 times.
✓ Branch 719 → 720 taken 20 times.
|
241 | for (int x = w_safe_mod8; x < width; x += 8) { |
| 1041 | process_eight_pixels_h_uint8_16<false, pixel_t, lessthan16bit>(src, x, current_coeff_base, filter_size, rounder128, shifttosigned_or_zero128, clamp_limit, dst, program); | ||
| 1042 | } | ||
| 1043 | |||
| 1044 | 68 | dst += dst_pitch; | |
| 1045 | 68 | src += src_pitch; | |
| 1046 | } | ||
| 1047 | 14 | } | |
| 1048 | |||
| 1049 | // 16 bit Horizontal | ||
| 1050 | |||
| 1051 | template void resizer_h_ssse3_generic_uint8_16<uint8_t, true>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel); | ||
| 1052 | template void resizer_h_ssse3_generic_uint8_16<uint16_t, false>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel); | ||
| 1053 | template void resizer_h_ssse3_generic_uint8_16<uint16_t, true>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel); | ||
| 1054 | |||
| 1055 | template void resize_v_sse2_planar_uint16_t<false>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int target_height, int bits_per_pixel); | ||
| 1056 | template void resize_v_sse2_planar_uint16_t<true>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int target_height, int bits_per_pixel); | ||
| 1057 | |||
| 1058 | |||
| 1059 | // Safe partial load with SSE2 | ||
| 1060 | // Read exactly N pixels, avoiding | ||
| 1061 | // - reading beyond the end of the source buffer. | ||
| 1062 | // - avoid NaN contamination, since event with zero coefficients NaN * 0 = NaN | ||
| 1063 | template <int Nmod4> | ||
| 1064 | AVS_FORCEINLINE static __m128 load_partial_safe_sse2(const float* src_ptr_offsetted) { | ||
| 1065 | switch (Nmod4) { | ||
| 1066 | case 1: | ||
| 1067 | ✗ | return _mm_set_ps(0.0f, 0.0f, 0.0f, src_ptr_offsetted[0]); | |
| 1068 | // ideally: movss | ||
| 1069 | case 2: | ||
| 1070 | ✗ | return _mm_set_ps(0.0f, 0.0f, src_ptr_offsetted[1], src_ptr_offsetted[0]); | |
| 1071 | // ideally: movsd | ||
| 1072 | case 3: | ||
| 1073 | ✗ | return _mm_set_ps(0.0f, src_ptr_offsetted[2], src_ptr_offsetted[1], src_ptr_offsetted[0]); | |
| 1074 | // ideally: movss + movsd + shuffle or movsd + insert | ||
| 1075 | case 0: | ||
| 1076 | ✗ | return _mm_set_ps(src_ptr_offsetted[3], src_ptr_offsetted[2], src_ptr_offsetted[1], src_ptr_offsetted[0]); | |
| 1077 | // ideally: movups | ||
| 1078 | default: | ||
| 1079 | return _mm_setzero_ps(); // n/a cannot happen | ||
| 1080 | } | ||
| 1081 | } | ||
| 1082 | |||
| 1083 | // Processes a horizontal resampling kernel of up to four coefficients for float pixel types. | ||
| 1084 | // Supports BilinearResize, BicubicResize, or sinc with up to 2 taps (filter size <= 4). | ||
| 1085 | // SSE2 optimization loads and processes four float coefficients and pixels simultaneously. | ||
| 1086 | // The 'filtersizemod4' template parameter (0-3) helps optimize for different filter sizes modulo 4. | ||
| 1087 | // This SSE2 requires only filter_size_alignment of 4. | ||
| 1088 | template<int filtersizemod4> | ||
| 1089 | ✗ | void resize_h_planar_float_sse_transpose_vstripe_ks4(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel) { | |
| 1090 | assert(filtersizemod4 >= 0 && filtersizemod4 <= 3); | ||
| 1091 | |||
| 1092 | ✗ | const int filter_size = program->filter_size; // aligned, practically the coeff table stride | |
| 1093 | |||
| 1094 | ✗ | src_pitch /= sizeof(float); | |
| 1095 | ✗ | dst_pitch /= sizeof(float); | |
| 1096 | |||
| 1097 | ✗ | float* src = (float*)src8; | |
| 1098 | ✗ | float* dst = (float*)dst8; | |
| 1099 | |||
| 1100 | ✗ | const float* AVS_RESTRICT current_coeff = (const float* AVS_RESTRICT)program->pixel_coefficient_float; | |
| 1101 | |||
| 1102 | ✗ | constexpr int PIXELS_AT_A_TIME = 4; // Process four pixels in parallel using SSE2 | |
| 1103 | |||
| 1104 | // 'source_overread_beyond_targetx' indicates if the filter kernel can read beyond the target width. | ||
| 1105 | // Even if the filter alignment allows larger reads, our safety boundary for unaligned loads starts at 4 pixels back | ||
| 1106 | // from the target width, as we load 4 floats at once with '_mm_loadu_ps'. | ||
| 1107 | // In AVX2 we process two lanes, so any of the 8 offsets cannot be safely used, fallback to the unsafe case. | ||
| 1108 | // This is why then safelimit_4_pixels is used combined with safelimit_4 / PIXELS_AT_A_TIME * PIXELS_AT_A_TIME. | ||
| 1109 | ✗ | const int width_safe_mod = (program->safelimit_4_pixels.overread_possible ? program->safelimit_4_pixels.source_overread_beyond_targetx : width) / PIXELS_AT_A_TIME * PIXELS_AT_A_TIME; | |
| 1110 | |||
| 1111 | // Preconditions: | ||
| 1112 | ✗ | assert(program->filter_size_real <= 4); // We preload all relevant coefficients (up to 4) before the height loop. | |
| 1113 | |||
| 1114 | // 'target_size_alignment' ensures we can safely access coefficients using offsets like | ||
| 1115 | // 'filter_size * 3' when processing 4 H pixels at a time | ||
| 1116 | ✗ | assert(program->target_size_alignment >= 4); | |
| 1117 | |||
| 1118 | // Ensure that coefficient loading beyond the valid target size is safe for 4x4 float loads. | ||
| 1119 | ✗ | assert(program->filter_size_alignment >= 4); | |
| 1120 | |||
| 1121 | ✗ | int x = 0; | |
| 1122 | |||
| 1123 | // This 'auto' lambda construct replaces the need of templates | ||
| 1124 | ✗ | auto do_h_float_core = [&](auto partial_load) { | |
| 1125 | // Load up to 4 coefficients at once before the height loop. | ||
| 1126 | // Pre-loading and transposing coefficients keeps register usage efficient. | ||
| 1127 | // Assumes 'filter_size_aligned' is at least 4. | ||
| 1128 | ✗ | __m128 coeff_1 = _mm_load_ps(current_coeff + filter_size * 0); // Coefficients for the source pixel offset (for src_ptr + begin1 [0..3]) | |
| 1129 | ✗ | __m128 coeff_2 = _mm_load_ps(current_coeff + filter_size * 1); // for src_ptr + begin2 [0..3] | |
| 1130 | ✗ | __m128 coeff_3 = _mm_load_ps(current_coeff + filter_size * 2); // for src_ptr + begin3 [0..3] | |
| 1131 | ✗ | __m128 coeff_4 = _mm_load_ps(current_coeff + filter_size * 3); // for src_ptr + begin4 [0..3] | |
| 1132 | |||
| 1133 | ✗ | _MM_TRANSPOSE4_PS(coeff_1, coeff_2, coeff_3, coeff_4); | |
| 1134 | |||
| 1135 | ✗ | float* AVS_RESTRICT dst_ptr = dst + x; | |
| 1136 | ✗ | const float* src_ptr = src; | |
| 1137 | |||
| 1138 | // Pixel offsets for the current target x-positions. | ||
| 1139 | // Even for x >= width, these offsets are guaranteed to be within the allocated 'target_size_alignment'. | ||
| 1140 | ✗ | const int begin1 = program->pixel_offset[x + 0]; | |
| 1141 | ✗ | const int begin2 = program->pixel_offset[x + 1]; | |
| 1142 | ✗ | const int begin3 = program->pixel_offset[x + 2]; | |
| 1143 | ✗ | const int begin4 = program->pixel_offset[x + 3]; | |
| 1144 | |||
| 1145 | ✗ | for (int y = 0; y < height; y++) | |
| 1146 | { | ||
| 1147 | __m128 data_1; | ||
| 1148 | __m128 data_2; | ||
| 1149 | __m128 data_3; | ||
| 1150 | __m128 data_4; | ||
| 1151 | if constexpr (partial_load) { | ||
| 1152 | // In the potentially unsafe zone (near the right edge of the image), we use a safe loading function | ||
| 1153 | // to prevent reading beyond the allocated source scanline. This handles cases where loading 4 floats | ||
| 1154 | // starting from 'src_ptr + beginX' might exceed the source buffer. | ||
| 1155 | |||
| 1156 | // Example of the unsafe scenario: If target width is 320, a naive load at src_ptr + 317 | ||
| 1157 | // would attempt to read floats at indices 317, 318, 319, and 320, potentially going out of bounds. | ||
| 1158 | |||
| 1159 | // Two main issues in the unsafe zone: | ||
| 1160 | // 1.) Out-of-bounds memory access: Reading beyond the allocated memory for the source scanline can | ||
| 1161 | // lead to access violations and crashes. '_mm_loadu_ps' attempts to load 16 bytes, so even if | ||
| 1162 | // the starting address is within bounds, subsequent reads might not be. | ||
| 1163 | // 2.) Garbage or NaN values: Even if a read doesn't cause a crash, accessing uninitialized or | ||
| 1164 | // out-of-bounds memory (especially for float types) can result in garbage data, including NaN. | ||
| 1165 | // Multiplying by a valid coefficient and accumulating this NaN can contaminate the final result. | ||
| 1166 | |||
| 1167 | // 'load_partial_safe_sse2' safely loads up to 'filter_size_real' pixels and pads with zeros if needed, | ||
| 1168 | // preventing out-of-bounds reads and ensuring predictable results even near the image edges. | ||
| 1169 | |||
| 1170 | ✗ | data_1 = load_partial_safe_sse2<filtersizemod4>(src_ptr + begin1); | |
| 1171 | ✗ | data_2 = load_partial_safe_sse2<filtersizemod4>(src_ptr + begin2); | |
| 1172 | ✗ | data_3 = load_partial_safe_sse2<filtersizemod4>(src_ptr + begin3); | |
| 1173 | ✗ | data_4 = load_partial_safe_sse2<filtersizemod4>(src_ptr + begin4); | |
| 1174 | } | ||
| 1175 | else { | ||
| 1176 | // In the safe zone, we can directly load 4 pixels at a time using unaligned loads. | ||
| 1177 | ✗ | data_1 = _mm_loadu_ps(src_ptr + begin1); | |
| 1178 | ✗ | data_2 = _mm_loadu_ps(src_ptr + begin2); | |
| 1179 | ✗ | data_3 = _mm_loadu_ps(src_ptr + begin3); | |
| 1180 | ✗ | data_4 = _mm_loadu_ps(src_ptr + begin4); | |
| 1181 | } | ||
| 1182 | |||
| 1183 | ✗ | _MM_TRANSPOSE4_PS(data_1, data_2, data_3, data_4); | |
| 1184 | |||
| 1185 | ✗ | __m128 result = _mm_mul_ps(data_1, coeff_1); | |
| 1186 | ✗ | result = _mm_add_ps(_mm_mul_ps(data_2, coeff_2), result); | |
| 1187 | ✗ | result = _mm_add_ps(_mm_mul_ps(data_3, coeff_3), result); | |
| 1188 | ✗ | result = _mm_add_ps(_mm_mul_ps(data_4, coeff_4), result); | |
| 1189 | |||
| 1190 | _mm_store_ps(dst_ptr, result); | ||
| 1191 | |||
| 1192 | ✗ | dst_ptr += dst_pitch; | |
| 1193 | ✗ | src_ptr += src_pitch; | |
| 1194 | } // y | ||
| 1195 | ✗ | current_coeff += filter_size * 4; // Move to the next set of coefficients for the next 4 output pixels | |
| 1196 | }; // end of lambda | ||
| 1197 | |||
| 1198 | // Process the 'safe zone' where direct full unaligned loads are acceptable. | ||
| 1199 | ✗ | for (; x < width_safe_mod; x += PIXELS_AT_A_TIME) | |
| 1200 | { | ||
| 1201 | ✗ | do_h_float_core(std::false_type{}); // partial_load == false, use direct _mm_loadu_ps | |
| 1202 | } | ||
| 1203 | |||
| 1204 | // Process the potentially 'unsafe zone' near the image edge, using safe loading. | ||
| 1205 | ✗ | for (; x < width; x += PIXELS_AT_A_TIME) | |
| 1206 | { | ||
| 1207 | ✗ | do_h_float_core(std::true_type{}); // partial_load == true, use the safer 'load_partial_safe_sse2' | |
| 1208 | } | ||
| 1209 | ✗ | } | |
| 1210 | |||
| 1211 | // Instantiate them | ||
| 1212 | template void resize_h_planar_float_sse_transpose_vstripe_ks4<0>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel); | ||
| 1213 | template void resize_h_planar_float_sse_transpose_vstripe_ks4<1>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel); | ||
| 1214 | template void resize_h_planar_float_sse_transpose_vstripe_ks4<2>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel); | ||
| 1215 | template void resize_h_planar_float_sse_transpose_vstripe_ks4<3>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel); | ||
| 1216 |