filters/intel/layer_sse41.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // AviSynth+ Copyright 2026- AviSynth+ Project | ||
| 2 | // SPDX-License-Identifier: GPL-2.0-or-later | ||
| 3 | // | ||
| 4 | // SSE4.1 Layer add/subtract dispatcher + SSE2 invert_plane kernels. | ||
| 5 | // Named *_sse41.cpp so the CMake handle_arch_flags(SSE41) glob assigns | ||
| 6 | // per-file -msse4.1 (GCC/Clang) flags to this translation unit. | ||
| 7 | // That means layer.hpp C templates included here are auto-vectorized for SSE4.1, | ||
| 8 | // matching the AVX2 pattern in layer_avx2.cpp. | ||
| 9 | // invert_plane_sse2_* are marked __target__("sse2") on GCC/Clang so the compiler | ||
| 10 | // does not emit SSE4.1 instructions inside them despite the TU's -msse4.1 flag. | ||
| 11 | |||
| 12 | #include "../layer.h" // layer_yuv_add_c_t/f, PLACEMENT_*, pulls in blend_common.h | ||
| 13 | #include "layer_sse41.h" // declaration of get_layer_yuv_add_functions_sse41 | ||
| 14 | |||
| 15 | #if defined(_MSC_VER) | ||
| 16 | #include <intrin.h> // MSVC | ||
| 17 | #else | ||
| 18 | #include <x86intrin.h> // GCC/MinGW/Clang/LLVM | ||
| 19 | #endif | ||
| 20 | #include <smmintrin.h> // SSE4.1 | ||
| 21 | |||
| 22 | #include <cstdint> | ||
| 23 | #include <type_traits> | ||
| 24 | #include <vector> | ||
| 25 | |||
| 26 | #include "../core/internal.h" // DISABLE_WARNING_PUSH/POP — needed by layer.hpp | ||
| 27 | |||
| 28 | // masked_merge_sse41_impl<maskMode> — static, SSE4.1. | ||
| 29 | // Including here (after intrinsic headers) compiles it with SSE4.1 flags for this TU. | ||
| 30 | #include "../overlay/intel/masked_merge_sse41_impl.hpp" | ||
| 31 | #include "../overlay/intel/blend_common_sse.h" | ||
| 32 | |||
| 33 | // C template dispatcher: get_layer_yuv_add_masked_functions<is_subtract>. | ||
| 34 | // static functions — this TU's copy gets -msse4.1 auto-vectorization (GCC/Clang). | ||
| 35 | // Use SSE4.1 SIMD rowprep variants — masked_rowprep_sse41.hpp is already included above | ||
| 36 | // via masked_merge_sse41_impl.hpp. | ||
| 37 | #define LAYER_ROWPREP_FN prepare_effective_mask_for_row_sse41 | ||
| 38 | #include "../layer.hpp" | ||
| 39 | #undef LAYER_ROWPREP_FN | ||
| 40 | |||
| 41 | // --------------------------------------------------------------------------- | ||
| 42 | // Planar RGB add — SSE4.1 per-plane wrappers (mirrors AVX2 counterparts). | ||
| 43 | // All planar RGB planes are MASK444. maskp8 is the per-pixel weight. | ||
| 44 | // chroma=false and float fall back to C templates. | ||
| 45 | |||
| 46 | 5 | static void layer_planarrgb_add_sse41_3plane( | |
| 47 | BYTE** dstp8, const BYTE** ovrp8, const BYTE* maskp8, | ||
| 48 | int dst_pitch, int overlay_pitch, int mask_pitch, | ||
| 49 | int width, int height, int opacity_i, int bits_per_pixel) | ||
| 50 | { | ||
| 51 |
2/2✓ Branch 5 → 3 taken 15 times.
✓ Branch 5 → 6 taken 5 times.
|
20 | for (int i = 0; i < 3; i++) |
| 52 | 15 | masked_merge_sse41_impl<MASK444>( | |
| 53 | 15 | dstp8[i], ovrp8[i], maskp8, | |
| 54 | dst_pitch, overlay_pitch, mask_pitch, | ||
| 55 | width, height, opacity_i, bits_per_pixel); | ||
| 56 | 5 | } | |
| 57 | |||
| 58 | 5 | static void layer_planarrgb_add_sse41_4plane( | |
| 59 | BYTE** dstp8, const BYTE** ovrp8, const BYTE* maskp8, | ||
| 60 | int dst_pitch, int overlay_pitch, int mask_pitch, | ||
| 61 | int width, int height, int opacity_i, int bits_per_pixel) | ||
| 62 | { | ||
| 63 |
2/2✓ Branch 5 → 3 taken 20 times.
✓ Branch 5 → 6 taken 5 times.
|
25 | for (int i = 0; i < 4; i++) |
| 64 | 20 | masked_merge_sse41_impl<MASK444>( | |
| 65 | 20 | dstp8[i], ovrp8[i], maskp8, | |
| 66 | dst_pitch, overlay_pitch, mask_pitch, | ||
| 67 | width, height, opacity_i, bits_per_pixel); | ||
| 68 | 5 | } | |
| 69 | |||
| 70 | 1820 | void get_layer_planarrgb_add_functions_sse41( | |
| 71 | bool chroma, bool hasAlpha, bool blendAlpha, int bits_per_pixel, | ||
| 72 | layer_planarrgb_add_c_t** layer_fn, | ||
| 73 | layer_planarrgb_add_f_c_t** layer_f_fn) | ||
| 74 | { | ||
| 75 | // Integer + hasAlpha + chroma=true: dispatch per-plane to masked_merge_avx2_impl (MASK444). | ||
| 76 | // chroma=false (blend toward luma) has a different formula — keep C template. | ||
| 77 | // float: keep C template (float perf is usually fine; could add later). | ||
| 78 |
3/6✓ Branch 2 → 3 taken 1820 times.
✗ Branch 2 → 9 not taken.
✓ Branch 3 → 4 taken 1820 times.
✗ Branch 3 → 9 not taken.
✓ Branch 4 → 5 taken 1820 times.
✗ Branch 4 → 9 not taken.
|
1820 | if (hasAlpha && chroma && bits_per_pixel != 32) { |
| 79 |
2/2✓ Branch 5 → 6 taken 910 times.
✓ Branch 5 → 7 taken 910 times.
|
1820 | *layer_fn = blendAlpha ? layer_planarrgb_add_sse41_4plane : layer_planarrgb_add_sse41_3plane; |
| 80 | 1820 | return; | |
| 81 | } | ||
| 82 | |||
| 83 | // chroma is true: Layer can use the unified masked and weighted blend routines | ||
| 84 | // chroma is false: Layer-specific extension | ||
| 85 | // Integer + hasAlpha + chroma=true: dispatch per-plane to masked_merge_avx2_impl (MASK444). | ||
| 86 | // chroma=false (blend toward luma) has a different formula — keep C template. | ||
| 87 | // float: keep C template (float perf is usually fine; could add later). | ||
| 88 | ✗ | if (bits_per_pixel != 32 && chroma) { | |
| 89 | ✗ | if (hasAlpha) { | |
| 90 | // standard masked merge | ||
| 91 | ✗ | *layer_fn = blendAlpha ? layer_planarrgb_add_sse41_4plane : layer_planarrgb_add_sse41_3plane; | |
| 92 | ✗ | return; | |
| 93 | } | ||
| 94 | // no alpha: standard weighted merge, to be added later. | ||
| 95 | } | ||
| 96 | ✗ | get_layer_planarrgb_add_functions(chroma, hasAlpha, blendAlpha, bits_per_pixel, layer_fn, layer_f_fn); | |
| 97 | } | ||
| 98 | |||
| 99 | // --------------------------------------------------------------------------- | ||
| 100 | // Packed RGBA (RGB32) magic-div blend — SSE4.1, 8-bit only. | ||
| 101 | // | ||
| 102 | // Processes 4 BGRA pixels (16 bytes) per iteration. | ||
| 103 | // Arithmetic is identical to the AVX2 version; see that function's comment | ||
| 104 | // for derivations and overflow proofs. | ||
| 105 | // | ||
| 106 | // has_separate_mask=false → alpha from ovrp8[x*4+3] (Add) | ||
| 107 | // has_separate_mask=true → alpha from maskp8[x] (Subtract, overlay pre-inverted) | ||
| 108 | // --------------------------------------------------------------------------- | ||
| 109 | template<bool has_separate_mask> | ||
| 110 | #if defined(GCC) || defined(CLANG) | ||
| 111 | __attribute__((__target__("sse4.1"))) | ||
| 112 | #endif | ||
| 113 | 6 | static void masked_blend_packedrgba_sse41_u8( | |
| 114 | BYTE* dstp8, const BYTE* ovrp8, const BYTE* maskp8, | ||
| 115 | int dst_pitch, int ovr_pitch, int mask_pitch, | ||
| 116 | int width, int height, int opacity_i) | ||
| 117 | { | ||
| 118 | // Shuffle: replicate byte 3 (A) of each BGRA pixel to all 4 bytes. | ||
| 119 | // {3,3,3,3, 7,7,7,7, 11,11,11,11, 15,15,15,15} | ||
| 120 | 6 | const __m128i shuf_alpha_bgra = _mm_set_epi8( | |
| 121 | 15,15,15,15, 11,11,11,11, 7,7,7,7, 3,3,3,3); | ||
| 122 | // Shuffle for separate mask: 4 bytes m0..m3 → each byte broadcast to 4 positions. | ||
| 123 | // {0,0,0,0, 1,1,1,1, 2,2,2,2, 3,3,3,3} | ||
| 124 | 6 | const __m128i shuf_alpha_mask = _mm_set_epi8( | |
| 125 | 3,3,3,3, 2,2,2,2, 1,1,1,1, 0,0,0,0); | ||
| 126 | |||
| 127 | 12 | const __m128i v_opacity = _mm_set1_epi16((short)opacity_i); | |
| 128 | 6 | const __m128i v_half = _mm_set1_epi16(127); | |
| 129 | 6 | const __m128i v_max = _mm_set1_epi16(255); | |
| 130 | 6 | const __m128i v_magic = _mm_set1_epi16((short)0x8081u); // magic for ÷255 | |
| 131 | |||
| 132 | // magic ÷255: mulhi_epu16(x, 0x8081) >> 7 | ||
| 133 | 270 | auto div255 = [&](__m128i x) -> __m128i { | |
| 134 | 792 | return _mm_srli_epi16(_mm_mulhi_epu16(x, v_magic), 7); | |
| 135 | }; | ||
| 136 | |||
| 137 | 6 | const int mod4_width = width / 4 * 4; | |
| 138 | |||
| 139 |
4/4void masked_blend_packedrgba_sse41_u8<false>(unsigned char*, unsigned char const*, unsigned char const*, int, int, int, int, int, int):
✓ Branch 90 → 23 taken 13 times.
✓ Branch 90 → 91 taken 3 times.
void masked_blend_packedrgba_sse41_u8<true>(unsigned char*, unsigned char const*, unsigned char const*, int, int, int, int, int, int):
✓ Branch 94 → 23 taken 13 times.
✓ Branch 94 → 95 taken 3 times.
|
32 | for (int y = 0; y < height; ++y) { |
| 140 | 26 | int x = 0; | |
| 141 |
4/4void masked_blend_packedrgba_sse41_u8<false>(unsigned char*, unsigned char const*, unsigned char const*, int, int, int, int, int, int):
✓ Branch 78 → 24 taken 33 times.
✓ Branch 78 → 79 taken 13 times.
void masked_blend_packedrgba_sse41_u8<true>(unsigned char*, unsigned char const*, unsigned char const*, int, int, int, int, int, int):
✓ Branch 82 → 24 taken 33 times.
✓ Branch 82 → 83 taken 13 times.
|
92 | for (; x < mod4_width; x += 4) { |
| 142 | 66 | __m128i dst16 = _mm_loadu_si128((const __m128i*)(dstp8 + x * 4)); | |
| 143 | 99 | __m128i ovr16 = _mm_loadu_si128((const __m128i*)(ovrp8 + x * 4)); | |
| 144 | |||
| 145 | // Broadcast per-pixel alpha to all 4 channel bytes. | ||
| 146 | __m128i alpha_bcast; | ||
| 147 | if constexpr (has_separate_mask) { | ||
| 148 | // Load 4 mask bytes (one per pixel) and broadcast each to 4 bytes. | ||
| 149 | 66 | __m128i mask4 = _mm_cvtsi32_si128(*(const int*)(maskp8 + x)); | |
| 150 | 33 | alpha_bcast = _mm_shuffle_epi8(mask4, shuf_alpha_mask); | |
| 151 | } else { | ||
| 152 | 33 | alpha_bcast = _mm_shuffle_epi8(ovr16, shuf_alpha_bgra); | |
| 153 | } | ||
| 154 | |||
| 155 | // Expand to 16-bit in two halves (lo=px0-1, hi=px2-3). | ||
| 156 | 66 | __m128i dst_lo = _mm_cvtepu8_epi16(dst16); | |
| 157 | 132 | __m128i dst_hi = _mm_cvtepu8_epi16(_mm_srli_si128(dst16, 8)); | |
| 158 | 66 | __m128i ovr_lo = _mm_cvtepu8_epi16(ovr16); | |
| 159 | 132 | __m128i ovr_hi = _mm_cvtepu8_epi16(_mm_srli_si128(ovr16, 8)); | |
| 160 | 66 | __m128i a_lo = _mm_cvtepu8_epi16(alpha_bcast); | |
| 161 | 132 | __m128i a_hi = _mm_cvtepu8_epi16(_mm_srli_si128(alpha_bcast, 8)); | |
| 162 | |||
| 163 | // alpha_eff = (alpha * opacity_i + 127) / 255 | ||
| 164 | 132 | __m128i aeff_lo = div255(_mm_add_epi16(_mm_mullo_epi16(a_lo, v_opacity), v_half)); | |
| 165 | 132 | __m128i aeff_hi = div255(_mm_add_epi16(_mm_mullo_epi16(a_hi, v_opacity), v_half)); | |
| 166 | 66 | __m128i inv_lo = _mm_sub_epi16(v_max, aeff_lo); | |
| 167 | 66 | __m128i inv_hi = _mm_sub_epi16(v_max, aeff_hi); | |
| 168 | |||
| 169 | // b = ovr (overlay pre-inverted for Subtract; plain for Add) | ||
| 170 | 66 | const __m128i b_lo = ovr_lo; | |
| 171 | 66 | const __m128i b_hi = ovr_hi; | |
| 172 | |||
| 173 | // result = (dst * inv + b * aeff + 127) / 255 | ||
| 174 | 264 | __m128i res_lo = div255(_mm_add_epi16( | |
| 175 | _mm_add_epi16(_mm_mullo_epi16(dst_lo, inv_lo), | ||
| 176 | _mm_mullo_epi16(b_lo, aeff_lo)), v_half)); | ||
| 177 | 264 | __m128i res_hi = div255(_mm_add_epi16( | |
| 178 | _mm_add_epi16(_mm_mullo_epi16(dst_hi, inv_hi), | ||
| 179 | _mm_mullo_epi16(b_hi, aeff_hi)), v_half)); | ||
| 180 | |||
| 181 | // Pack 16→8-bit; _mm_packus_epi16 preserves pixel order (no permute needed). | ||
| 182 | 66 | _mm_storeu_si128((__m128i*)(dstp8 + x * 4), _mm_packus_epi16(res_lo, res_hi)); | |
| 183 | } | ||
| 184 | |||
| 185 | // Scalar tail. | ||
| 186 | 26 | constexpr MagicDiv magic8 = get_magic_div(8); | |
| 187 |
4/4void masked_blend_packedrgba_sse41_u8<false>(unsigned char*, unsigned char const*, unsigned char const*, int, int, int, int, int, int):
✓ Branch 88 → 80 taken 13 times.
✓ Branch 88 → 89 taken 13 times.
void masked_blend_packedrgba_sse41_u8<true>(unsigned char*, unsigned char const*, unsigned char const*, int, int, int, int, int, int):
✓ Branch 92 → 84 taken 13 times.
✓ Branch 92 → 93 taken 13 times.
|
52 | for (; x < width; ++x) { |
| 188 | 26 | const uint32_t alpha_src = has_separate_mask | |
| 189 | 13 | ? (uint32_t)maskp8[x] | |
| 190 | 13 | : (uint32_t)ovrp8[x * 4 + 3]; | |
| 191 | 52 | const uint32_t ae = (uint32_t)magic_div_rt<uint8_t>( | |
| 192 | 26 | alpha_src * (uint32_t)opacity_i + 127u, magic8); | |
| 193 | 26 | const uint32_t iv = 255u - ae; | |
| 194 |
4/4void masked_blend_packedrgba_sse41_u8<false>(unsigned char*, unsigned char const*, unsigned char const*, int, int, int, int, int, int):
✓ Branch 86 → 83 taken 52 times.
✓ Branch 86 → 87 taken 13 times.
void masked_blend_packedrgba_sse41_u8<true>(unsigned char*, unsigned char const*, unsigned char const*, int, int, int, int, int, int):
✓ Branch 90 → 87 taken 52 times.
✓ Branch 90 → 91 taken 13 times.
|
130 | for (int ch = 0; ch < 4; ++ch) { |
| 195 | 104 | dstp8[x * 4 + ch] = (BYTE)magic_div_rt<uint8_t>( | |
| 196 | 104 | (uint32_t)dstp8[x * 4 + ch] * iv + (uint32_t)ovrp8[x * 4 + ch] * ae + 127u, magic8); | |
| 197 | } | ||
| 198 | } | ||
| 199 | |||
| 200 | 26 | dstp8 += dst_pitch; | |
| 201 | 26 | ovrp8 += ovr_pitch; | |
| 202 | 13 | if constexpr (has_separate_mask) maskp8 += mask_pitch; | |
| 203 | } | ||
| 204 | 6 | } | |
| 205 | |||
| 206 | // Dispatcher: 8-bit only; 16-bit falls back to C reference. | ||
| 207 | // has_separate_mask=false → Add (alpha from ovrp8[x*4+3]) | ||
| 208 | // has_separate_mask=true → Subtract (alpha from maskp8[x], overlay pre-inverted) | ||
| 209 | 1092 | void get_layer_packedrgb_blend_functions_sse41( | |
| 210 | bool has_separate_mask, int bits_per_pixel, | ||
| 211 | layer_packedrgb_blend_c_t** fn) | ||
| 212 | { | ||
| 213 |
1/2✓ Branch 2 → 3 taken 1092 times.
✗ Branch 2 → 7 not taken.
|
1092 | if (bits_per_pixel == 8) { |
| 214 |
2/2✓ Branch 3 → 4 taken 546 times.
✓ Branch 3 → 5 taken 546 times.
|
1092 | *fn = has_separate_mask ? masked_blend_packedrgba_sse41_u8<true> |
| 215 | : masked_blend_packedrgba_sse41_u8<false>; | ||
| 216 | 1092 | return; | |
| 217 | } | ||
| 218 | ✗ | get_layer_packedrgb_blend_functions(has_separate_mask, bits_per_pixel, fn); | |
| 219 | } | ||
| 220 | |||
| 221 | // --------------------------------------------------------------------------- | ||
| 222 | // SSE4.1 Layer add dispatcher. | ||
| 223 | // --------------------------------------------------------------------------- | ||
| 224 | 5278 | void get_layer_yuv_masked_add_functions_sse41( | |
| 225 | bool is_chroma, | ||
| 226 | int placement, VideoInfo& vi, int bits_per_pixel, | ||
| 227 | /*out*/masked_merge_fn_t** layer_fn, | ||
| 228 | /*out*/masked_merge_float_fn_t** layer_f_fn) | ||
| 229 | { | ||
| 230 | // only masked merge is left here, simple weighted blend is already handled | ||
| 231 | |||
| 232 | // Use the unified (Layer,Overlay) masked merge functions | ||
| 233 | // Determine MaskMode from format and placement | ||
| 234 | 5278 | MaskMode maskMode = MASK444; | |
| 235 |
2/2✓ Branch 2 → 3 taken 4550 times.
✓ Branch 2 → 20 taken 728 times.
|
5278 | if (is_chroma) { |
| 236 |
1/2✗ Branch 4 → 5 not taken.
✓ Branch 4 → 6 taken 4550 times.
|
4550 | if (vi.IsYV411()) |
| 237 | ✗ | maskMode = MASK411; | |
| 238 |
2/2✓ Branch 7 → 8 taken 2366 times.
✓ Branch 7 → 13 taken 2184 times.
|
4550 | else if (vi.Is420()) |
| 239 |
4/4✓ Branch 8 → 9 taken 1638 times.
✓ Branch 8 → 12 taken 728 times.
✓ Branch 9 → 10 taken 728 times.
✓ Branch 9 → 11 taken 910 times.
|
2366 | maskMode = (placement == PLACEMENT_MPEG1) ? MASK420 : (placement == PLACEMENT_TOPLEFT) ? MASK420_TOPLEFT : MASK420_MPEG2; |
| 240 |
1/2✓ Branch 14 → 15 taken 2184 times.
✗ Branch 14 → 20 not taken.
|
2184 | else if (vi.Is422()) |
| 241 |
4/4✓ Branch 15 → 16 taken 1456 times.
✓ Branch 15 → 19 taken 728 times.
✓ Branch 16 → 17 taken 728 times.
✓ Branch 16 → 18 taken 728 times.
|
2184 | maskMode = (placement == PLACEMENT_MPEG1) ? MASK422 : (placement == PLACEMENT_TOPLEFT) ? MASK422_TOPLEFT : MASK422_MPEG2; |
| 242 | // Is444() / IsY(): stay MASK444 | ||
| 243 | } | ||
| 244 | // is_chroma=false (luma): always MASK444 | ||
| 245 | |||
| 246 | 5278 | *layer_fn = get_overlay_blend_masked_fn_sse41(is_chroma, maskMode); | |
| 247 | 5278 | *layer_f_fn = get_overlay_blend_masked_float_fn_sse41(is_chroma, maskMode); | |
| 248 | |||
| 249 | 5278 | } | |
| 250 |