GCC Code Coverage Report

Directory: avs_core/
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 0.0% 0 / 0 / 5058
Functions: 0.0% 0 / 0 / 276
Branches: 0.0% 0 / 0 / 45724
filters/exprfilter/exprfilter.cpp
Line Branch Exec Source
1 /*
2 *
3 * Avisynth+ Expression filter, ported from the VapourSynth project
4 * Copyright (c) 2012-2015 Fredrik Mellbin
5 *
6 * Additions and differences to VS r39 version:
7 * ------------------------------
8 * (similar features to the masktools mt_lut family syntax)
9 * Operator aliases:
10 * Caret (^) can be used like pow
11 * For equality check "==" can be used like "="
12 * & same as and
13 * | same as or
14 * New operator: != (not equal)
15 * Built-in constants
16 * ymin, ymax (ymin_a .. ymin_z for individual clips) - the usual luma limits (16..235 or scaled equivalents)
17 * cmin, cmax (cmin_a .. cmin_z) - chroma limits (16..240 or scaled equivalents)
18 * range_half (range_half_a .. range_half_z) - half of the range, (128 or scaled equivalents)
19 * range_size, range_half, range_min, range_max (range_size_a .. range_size_z , etc..)
20 * Autoscale helper functions (operand is treated as being a 8 bit constant unless i8..i16 or f32 is specified)
21 * scaleb (scale by bit shift - mul or div by 2, 4, 6, 8...)
22 * scalef (scale by stretch full scale - mul or div by source_max/target_max
23 * Keywords for modifying base bit depth for scaleb and scalef
24 * i8, i10, i12, i14, i16, f32
25 * Built-in math constant
26 * pi
27 * Alpha plane handling. When no separate expression is supplied for alpha, plane is copied instead of reusing last expression parameter
28 * Proper clamping when storing 10,12 or 14 bit outputs
29 * Faster storing of results for 8 and 10-16 bit outputs
30 * 16 pixels/cycle instead of 8 when avx2, with fallback to 8-pixel case on the right edge. Thus no need for 64 byte alignment for 32 bit float.
31 * (Load zeros for nonvisible pixels, when simd block size goes beyond image width, to prevent garbage input for simd calculation)
32 * Optimizations: x^0.5 is sqrt, ^1 +0 -0 *1 /1 to nothing, ^2, ^3, ^4 is done by faster and more precise multiplication
33 * spatial input variables in expr syntax:
34 * sx, sy (absolute x and y coordinates, 0 to width-1 and 0 to height-1)
35 * sxr, syr (relative x and y coordinates, from 0 to 1.0)
36 * Optimize: recognize constant plane expression: use fast memset instead of generic simd process. Approx. 3-4x (32 bits) to 10-12x (8 bits) speedup
37 * Optimize: Recognize single clip letter in expression: use fast plane copy (BitBlt)
38 * (e.g. for 8-16 bits: instead of load-convert_to_float-clamp-convert_to_int-store). Approx. 1.4x (32 bits), 3x (16 bits), 8-9x (8 bits) speedup
39 * Optimize: do not call GetFrame for input clips that are not referenced or plane-copied
40 * 20171211: Implement relative pixel indexing e.g. x[-1,-3], requires SSSE3
41 * Fix jitasm code generation (corrupt code when doing register reorders)
42 * 20171212: Variables: A..Z
43 * To store the current top value of the stack to a variable: A@ .. Z@
44 * To store the current top value and pop it from the top of the stack: A^.. Z^
45 * To use a stored variable: single uppercase letter. E.g. A
46 * 20171214: Trig.functions (C only): sin, cos, tan, asin, acos, atan
47 * '%' The implementation is fmod-like: x - trunc(x/d)*d.
48 * Note: SSE2 and up is using trunc for float->integer conversion, works for usual width/height magnitude.
49 * (A float can hold a 24 bit integer w/o losing precision)
50 * expr constants: 'width', 'height' for current plane width and height
51 * expr auto variable: 'frameno' holds the current frame number 0..total number of frames-1
52 * expr auto variable: 'time' relative time in clip, 0 <= time <= 1
53 * calculation: time = frameno/(total number of frames - 1)
54 * 20180614 new parameters: scale_inputs, clamp_float
55 * implement 'clip' three operand operator like in masktools2: x minvalue maxvalue clip -> max(min(x, maxvalue), minvalue)
56 * 20191120 yrange_min, yrange_half, yrange_max, clamp_float_UV param, allow "float_UV" for scale_inputs
57 * yscalef, yscaleb forcing non-chroma rules for scaling even when processing chroma planes
58 * 202107xx round, floor, ceil, trunc (acceleration from SSE4.1 and up)
59 * arbitrary variable names; instead of 'A' to 'Z', up to 256 of them can be used
60 * 20210924 frame property access clip.framePropName syntax after VSAkarin idea (no array access yet)
61 * sin and cos as SIMD (VSAkarin, port from VS)
62 * 20211116 neg
63 * 20211117 atan2 as SIMD
64 * 20211118 sgn
65 * 20211127 lutx, lutxy (lut=1 and 2)
66 * 20211128 allow f32 for 'scale_inputs' when "int", "intf", "all", "allf"
67 * 20250309 tan as SIMD
68 *
69 * Differences from masktools 2.2.15
70 * ---------------------------------
71 * Up to 26 clips are allowed (x,y,z,a,b,...w). Masktools handles only up to 4 clips with its mt_lut, my_lutxy, mt_lutxyz, mt_lutxyza
72 * Clips with different bit depths are allowed
73 * works with 32 bit floats instead of 64 bit double internally
74 * less functions (e.g. no bit shifts)
75 * logical 'false' is 0 instead of -1
76 * avs+: ymin, ymax, etc built-in constants can have a _X suffix, where X is the corresponding clip designator letter. E.g. cmax_z, range_half_x
77 * mt_lutspa-like functionality is available through "sx", "sy", "sxr", "syr"
78 */
79
80 #include <iostream>
81 #include <locale>
82 #include <sstream>
83 #include <vector>
84 #include <list>
85 #include <string>
86 #include <algorithm>
87 #include <stdexcept>
88 #include <memory>
89 #include <cmath>
90 #include <unordered_map>
91
92 #include <avisynth.h>
93
94 #ifdef AVS_WINDOWS
95 #include <avs/win.h>
96 #else
97 #include <avs/posix.h>
98 #endif
99
100 #include <stdlib.h>
101 #include "../../core/internal.h"
102 #include "../../convert/convert_planar.h" // fill_plane
103 #include "../../convert/convert_helper.h"
104 #include "avs/alignment.h"
105
106 #if defined(_MSC_VER) || (defined(__clang__) && defined(_MSC_VER)) || defined(__MINGW32__) || defined(__MINGW64__)
107 #include <malloc.h> // For _aligned_malloc and _aligned_free
108 #endif
109
110 #if (defined(_WIN64) && (defined(_M_AMD64) || defined(_M_X64))) || defined(__x86_64__)
111 #define JITASM64
112 #endif
113
114 #ifdef INTEL_INTRINSICS
115 #define VS_TARGET_CPU_X86
116 #endif
117 #ifdef AVS_WINDOWS
118 #define VS_TARGET_OS_WINDOWS
119 #endif
120 #include "exprfilter.h"
121
122 #ifdef VS_TARGET_CPU_X86
123 #ifndef NOMINMAX
124 #define NOMINMAX
125 #endif
126 #include "jitasm.h"
127 #endif
128
129 #ifndef VS_TARGET_OS_WINDOWS
130 #include <sys/mman.h>
131 #endif
132
133 #ifdef XP_TLS
134 #ifdef MSVC_PURE
135 // v141_xp workaround: disabling /O2 global optimizations reduces build time
136 // from 56 minutes to 48 seconds.
137 #pragma optimize("g", off) // disables global optimizations (e.g. /O2 max opt,speed)
138 #pragma optimize("t", on) // favor speed /Ot
139 #pragma auto_inline(on) // enable aggressive inlining, equivalent of /Ob2
140 #endif
141 #endif
142
143 #ifdef VS_TARGET_CPU_X86
144
145 //#define TEST_AVX2_CODEGEN_IN_AVX
146
147 #include <immintrin.h>
148
149 #if defined(GCC) || defined(CLANG)
150 #include <avxintrin.h>
151 #endif
152
153 // rounder constants
154 constexpr int FROUND_TO_NEAREST_INT = 0x00;
155 constexpr int FROUND_TO_NEG_INF = 0x01;
156 constexpr int FROUND_TO_POS_INF = 0x02;
157 constexpr int FROUND_TO_ZERO = 0x03;
158 constexpr int FROUND_NO_EXC = 0x08;
159
160 // normal versions work with two xmm or ymm registers (2*4 or 2*8 pixels per cycle)
161 // _Single suffixed versions work only one xmm or ymm registers at a time (1*4 or 1*8 pixels per cycle)
162
163 #define OneArgOp(instr) \
164 auto &t1 = stack.back(); \
165 instr(t1.first, t1.first); \
166 instr(t1.second, t1.second);
167
168 #define OneArgOp_Single(instr) \
169 auto &t1 = stack1.back(); \
170 instr(t1, t1);
171
172 #define TwoArgOp(instr) \
173 auto t1 = stack.back(); \
174 stack.pop_back(); \
175 auto &t2 = stack.back(); \
176 instr(t2.first, t1.first); \
177 instr(t2.second, t1.second);
178
179 #define TwoArgOp_Single(instr) \
180 auto t1 = stack1.back(); \
181 stack1.pop_back(); \
182 auto &t2 = stack1.back(); \
183 instr(t2, t1);
184
185 #define TwoArgOp_Avx(instr) \
186 auto t1 = stack.back(); \
187 stack.pop_back(); \
188 auto &t2 = stack.back(); \
189 instr(t2.first, t2.first, t1.first); \
190 instr(t2.second, t2.second, t1.second);
191
192 #define TwoArgOp_Single_Avx(instr) \
193 auto t1 = stack1.back(); \
194 stack1.pop_back(); \
195 auto &t2 = stack1.back(); \
196 instr(t2, t2, t1);
197
198 #define CmpOp(instr) \
199 auto t1 = stack.back(); \
200 stack.pop_back(); \
201 auto t2 = stack.back(); \
202 stack.pop_back(); \
203 instr(t1.first, t2.first); \
204 instr(t1.second, t2.second); \
205 andps(t1.first, CPTR(elfloat_one)); \
206 andps(t1.second, CPTR(elfloat_one)); \
207 stack.push_back(t1);
208
209 #define CmpOp_Single(instr) \
210 auto t1 = stack1.back(); \
211 stack1.pop_back(); \
212 auto t2 = stack1.back(); \
213 stack1.pop_back(); \
214 instr(t1, t2); \
215 andps(t1, CPTR(elfloat_one)); \
216 stack1.push_back(t1);
217
218 #define CmpOp_Avx(instr, op) \
219 auto t1 = stack.back(); \
220 stack.pop_back(); \
221 auto t2 = stack.back(); \
222 stack.pop_back(); \
223 instr(t1.first, t1.first, t2.first, op); \
224 instr(t1.second, t1.second, t2.second, op); \
225 vandps(t1.first, t1.first, CPTR_AVX(elfloat_one)); \
226 vandps(t1.second, t1.second, CPTR_AVX(elfloat_one)); \
227 stack.push_back(t1);
228
229 #define CmpOp_Single_Avx(instr, op) \
230 auto t1 = stack1.back(); \
231 stack1.pop_back(); \
232 auto t2 = stack1.back(); \
233 stack1.pop_back(); \
234 instr(t1, t1, t2, op); \
235 vandps(t1, t1, CPTR_AVX(elfloat_one)); \
236 stack1.push_back(t1);
237
238 #define LogicOp(instr) \
239 auto t1 = stack.back(); \
240 stack.pop_back(); \
241 auto t2 = stack.back(); \
242 stack.pop_back(); \
243 cmpnleps(t1.first, zero); \
244 cmpnleps(t1.second, zero); \
245 cmpnleps(t2.first, zero); \
246 cmpnleps(t2.second, zero); \
247 instr(t1.first, t2.first); \
248 instr(t1.second, t2.second); \
249 andps(t1.first, CPTR(elfloat_one)); \
250 andps(t1.second, CPTR(elfloat_one)); \
251 stack.push_back(t1);
252
253 #define LogicOp_Single(instr) \
254 auto t1 = stack1.back(); \
255 stack1.pop_back(); \
256 auto t2 = stack1.back(); \
257 stack1.pop_back(); \
258 cmpnleps(t1, zero); \
259 cmpnleps(t2, zero); \
260 instr(t1, t2); \
261 andps(t1, CPTR(elfloat_one)); \
262 stack1.push_back(t1);
263
264 #define LogicOp_Avx(instr) \
265 auto t1 = stack.back(); \
266 stack.pop_back(); \
267 auto t2 = stack.back(); \
268 stack.pop_back(); \
269 vcmpps(t1.first, t1.first, zero, _CMP_GT_OQ); \
270 vcmpps(t1.second, t1.second, zero, _CMP_GT_OQ); \
271 vcmpps(t2.first, t2.first, zero, _CMP_GT_OQ); \
272 vcmpps(t2.second, t2.second, zero, _CMP_GT_OQ); \
273 instr(t1.first, t1.first, t2.first); \
274 instr(t1.second, t1.second, t2.second); \
275 vandps(t1.first, t1.first, CPTR_AVX(elfloat_one)); \
276 vandps(t1.second, t1.second, CPTR_AVX(elfloat_one)); \
277 stack.push_back(t1);
278
279 #define LogicOp_Single_Avx(instr) \
280 auto t1 = stack1.back(); \
281 stack1.pop_back(); \
282 auto t2 = stack1.back(); \
283 stack1.pop_back(); \
284 vcmpps(t1, t1, zero, _CMP_GT_OQ); \
285 vcmpps(t2, t2, zero, _CMP_GT_OQ); \
286 instr(t1, t1, t2); \
287 vandps(t1, t1, CPTR_AVX(elfloat_one)); \
288 stack1.push_back(t1);
289
290 enum {
291 elabsmask, elc7F, elmin_norm_pos, elinv_mant_mask,
292 elfloat_one, elfloat_minusone, elfloat_half, elsignmask, elstore8, elstore10, elstore12, elstore14, elstore16,
293 spatialX, spatialX2,
294 loadmask1000, loadmask1100, loadmask1110,
295 elShuffleForRight0, elShuffleForRight1, elShuffleForRight2, elShuffleForRight3, elShuffleForRight4, elShuffleForRight5, elShuffleForRight6,
296 elShuffleForLeft0, elShuffleForLeft1, elShuffleForLeft2, elShuffleForLeft3, elShuffleForLeft4, elShuffleForLeft5, elShuffleForLeft6,
297 elexp_hi, elexp_lo, elcephes_LOG2EF,
298 elcephes_exp_C1, elcephes_log_q2 = elcephes_exp_C1, elcephes_exp_C2, elcephes_log_q1 = elcephes_exp_C2, elcephes_exp_p0, elcephes_exp_p1, elcephes_exp_p2, elcephes_exp_p3, elcephes_exp_p4, elcephes_exp_p5, elcephes_SQRTHF,
299 elcephes_log_p0, elcephes_log_p1, elcephes_log_p2, elcephes_log_p3, elcephes_log_p4, elcephes_log_p5, elcephes_log_p6, elcephes_log_p7, elcephes_log_p8,
300 float_invpi, float_rintf,
301 float_pi1, float_pi2, float_pi3, float_pi4,
302 float_sinC3, float_sinC5, float_sinC7, float_sinC9,
303 float_cosC2, float_cosC4, float_cosC6, float_cosC8,
304 float_atan2f_rmul, float_atan2f_radd, float_atan2f_tmul, float_atan2f_tadd, float_atan2f_halfpi, float_atan2f_pi,
305 float_tan_p0, float_tan_p2, float_tan_p4, float_tan_p6, float_tan_p8,
306 float_tan_q0, float_tan_q2, float_tan_q4, float_tan_q6, float_tan_q8,
307 float_tan_small_limit, float_tan_asympt_a1, float_tan_asympt_a2, float_tan_asympt_limit
308 };
309
310 // constants for xmm
311
312 #define XCONST(x) { x, x, x, x }
313 #define MAKEDWORD(ch0, ch1, ch2, ch3) \
314 ((uint32_t)(unsigned char)(ch0) | ((uint32_t)(unsigned char)(ch1) << 8) | \
315 ((uint32_t)(unsigned char)(ch2) << 16) | ((uint32_t)(unsigned char)(ch3) << 24 ))
316 #define XBYTECONST(a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) \
317 { (int)MAKEDWORD(a0,a1,a2,a3), (int)MAKEDWORD(a4,a5,a6,a7), (int)MAKEDWORD(a8,a9,a10,a11), (int)MAKEDWORD(a12,a13,a14,a15) }
318
319
320 static constexpr ExprUnion logexpconst alignas(16)[87][4] = {
321 XCONST(0x7FFFFFFF), // absmask
322 XCONST(0x7F), // c7F
323 XCONST(0x00800000), // min_norm_pos
324 XCONST(~0x7f800000), // inv_mant_mask
325 XCONST(1.0f), // float_one
326 XCONST(-1.0f), // float_minusone
327 XCONST(0.5f), // float_half
328 XCONST(0x80000000), // elsignmask
329 XCONST(255.0f), // store8
330 XCONST(1023.0f), // store10 (avs+)
331 XCONST(4095.0f), // store12 (avs+)
332 XCONST(16383.0f), // store14 (avs+)
333 XCONST(65535.0f), // store16
334 { 0.0f, 1.0f, 2.0f, 3.0f }, // spatialX
335 { 4.0f, 5.0f, 6.0f, 7.0f }, // spatialX2
336 { (int)0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000 }, // loadmask1000
337 { (int)0xFFFFFFFF, (int)0xFFFFFFFF, 0x00000000, 0x00000000 }, // loadmask1100
338 { (int)0xFFFFFFFF, (int)0xFFFFFFFF, (int)0xFFFFFFFF, 0x00000000 }, // loadmask1110
339 XBYTECONST(0,1,2,3,4,5,6,7, 8,9,10,11,12,13,12,13), // elShuffleForRight0
340 XBYTECONST(0,1,2,3,4,5,6,7, 8,9,10,11,10,11,10,11), // elShuffleForRight1
341 XBYTECONST(0,1,2,3,4,5,6,7, 8,9,8,9,8,9,8,9), // elShuffleForRight2
342 XBYTECONST(0,1,2,3,4,5,6,7, 6,7,6,7,6,7,6,7), // elShuffleForRight3
343 XBYTECONST(0,1,2,3,4,5,4,5, 4,5,4,5,4,5,4,5), // elShuffleForRight4
344 XBYTECONST(0,1,2,3,2,3,2,3, 2,3,2,3,2,3,2,3), // elShuffleForRight5
345 XBYTECONST(0,1,0,1,0,1,0,1, 0,1,0,1,0,1,0,1), // elShuffleForRight6
346 XBYTECONST(2,3,2,3,4,5,6,7, 8,9,10,11,12,13,14,15), // elShuffleForLeft0
347 XBYTECONST(4,5,4,5,4,5,6,7, 8,9,10,11,12,13,14,15), // elShuffleForLeft1
348 XBYTECONST(6,7,6,7,6,7,6,7, 8,9,10,11,12,13,14,15), // elShuffleForLeft2
349 XBYTECONST(8,9,8,9,8,9,8,9, 8,9,10,11,12,13,14,15), // elShuffleForLeft3
350 XBYTECONST(10,11,10,11,10,11,10,11, 10,11,10,11,12,13,14,15), // elShuffleForLeft4
351 XBYTECONST(12,13,12,13,12,13,12,13, 12,13,12,13,12,13,14,15), // elShuffleForLeft5
352 XBYTECONST(14,15,14,15,14,15,14,15, 14,15,14,15,14,15,14,15), // elShuffleForLeft6
353 XCONST(88.3762626647949f), // exp_hi
354 XCONST(-88.3762626647949f), // exp_lo
355 XCONST(1.44269504088896341f), // cephes_LOG2EF
356 XCONST(0.693359375f), // cephes_exp_C1
357 XCONST(-2.12194440e-4f), // cephes_exp_C2
358 XCONST(1.9875691500E-4f), // cephes_exp_p0
359 XCONST(1.3981999507E-3f), // cephes_exp_p1
360 XCONST(8.3334519073E-3f), // cephes_exp_p2
361 XCONST(4.1665795894E-2f), // cephes_exp_p3
362 XCONST(1.6666665459E-1f), // cephes_exp_p4
363 XCONST(5.0000001201E-1f), // cephes_exp_p5
364 XCONST(0.707106781186547524f), // cephes_SQRTHF
365 XCONST(7.0376836292E-2f), // cephes_log_p0
366 XCONST(-1.1514610310E-1f), // cephes_log_p1
367 XCONST(1.1676998740E-1f), // cephes_log_p2
368 XCONST(-1.2420140846E-1f), // cephes_log_p3
369 XCONST(+1.4249322787E-1f), // cephes_log_p4
370 XCONST(-1.6668057665E-1f), // cephes_log_p5
371 XCONST(+2.0000714765E-1f), // cephes_log_p6
372 XCONST(-2.4999993993E-1f), // cephes_log_p7
373 XCONST(+3.3333331174E-1f), // cephes_log_p8
374 XCONST(0x3ea2f983), // float_invpi, 1/pi = 0.31830988618379067154f
375 XCONST(0x4b400000), // float_rintf Used for rounding
376 XCONST(0x40490000), // float_pi1 High precision part of Pi
377 XCONST(0x3a7da000), // float_pi2 Second part of Pi for extended precision
378 XCONST(0x34222000), // float_pi3 Third part of Pi for extended precision
379 XCONST(0x2cb4611a), // float_pi4 Fourth part of Pi for extended precision
380 XCONST(0xbe2aaaa6), // float_sinC3
381 XCONST(0x3c08876a), // float_sinC5
382 XCONST(0xb94fb7ff), // float_sinC7
383 XCONST(0x362edef8), // float_sinC9
384 XCONST(static_cast<int32_t>(0xBEFFFFE2)), // float_cosC2
385 XCONST(0x3D2AA73C), // float_cosC4
386 XCONST(static_cast<int32_t>(0XBAB58D50)), // float_cosC6
387 XCONST(0x37C1AD76), // float_cosC8
388 XCONST(0x3ccb7dda), // float_atan2f_rmul 0.024840285f
389 XCONST(0x3e3f4c37), // float_atan2f_radd 0.18681418f
390 XCONST(0xbdc0b66d), // float_atan2f_tmul -0.094097948f
391 XCONST(0xbeaa0d0a), // float_atan2f_tadd -0.33213072f
392 XCONST(0x3fc90fdb), // float_atan2f_halfpi 1.57079637f
393 XCONST(0x40490fdb), // float_atan2f_pi 3.14159274f
394 // Tangent approximation coefficients up to p8 and helpers
395 XCONST(1.0f), // float_tan_p0 - Numerator constant term
396 XCONST(0.3333314036f), // float_tan_p2 - Numerator x^2 coefficient
397 XCONST(0.1333923995f), // float_tan_p4 - Numerator x^4 coefficient
398 XCONST(0.0533740603f), // float_tan_p6 - Numerator x^6 coefficient
399 XCONST(0.0245650893f), // float_tan_p8 - Numerator x^8 coefficient
400 XCONST(1.0f), // float_tan_q0 - Denominator constant term
401 XCONST(0.1333835001f), // float_tan_q2 - Denominator x^2 coefficient
402 XCONST(0.0089270802f), // float_tan_q4 - Denominator x^4 coefficient
403 XCONST(0.0005908960f), // float_tan_q6 - Denominator x^6 coefficient
404 XCONST(0.0000342237f), // float_tan_q8 - Denominator x^8 coefficient
405 XCONST(1e-4f), // float_tan_small_limit
406 XCONST(0.97f), // float_tan_asympt_a1
407 XCONST(0.35f), // float_tan_asympt_a2
408 XCONST(0.8f), // float_tan_asympt_limit
409 };
410
411
412 #define CPTR(x) (xmmword_ptr[constptr + (x) * 16])
413
414 // AVX2 stuff
415 // constants for ymm
416
417 #undef XCONST
418 #define XCONST(x) { x, x, x, x, x, x, x, x }
419
420 static constexpr ExprUnion logexpconst_avx alignas(32)[87][8] = {
421 XCONST(0x7FFFFFFF), // absmask
422 XCONST(0x7F), // c7F
423 XCONST(0x00800000), // min_norm_pos
424 XCONST(~0x7f800000), // inv_mant_mask
425 XCONST(1.0f), // float_one
426 XCONST(-1.0f), // float_minusone
427 XCONST(0.5f), // float_half
428 XCONST(0x80000000), // elsignmask
429 XCONST(255.0f), // store8
430 XCONST(1023.0f), // store10 (avs+)
431 XCONST(4095.0f), // store12 (avs+)
432 XCONST(16383.0f), // store14 (avs+)
433 XCONST(65535.0f), // store16
434 { 0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f }, // spatialX
435 { 8.0f, 9.0f, 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f }, // spatialX2
436 { (int)0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000, 0, 0, 0, 0 }, // loadmask1000 not used, avx supports blendps
437 { (int)0xFFFFFFFF, (int)0xFFFFFFFF, 0x00000000, 0x00000000, 0, 0, 0, 0 }, // loadmask1100 not used, avx supports blendps
438 { (int)0xFFFFFFFF, (int)0xFFFFFFFF, (int)0xFFFFFFFF, 0x00000000, 0, 0, 0, 0 }, // loadmask1110 not used, avx supports blendps
439 XCONST(0), // n/a elShuffleForRight0
440 XCONST(0), // n/a elShuffleForRight1
441 XCONST(0), // n/a elShuffleForRight2
442 XCONST(0), // n/a elShuffleForRight3
443 XCONST(0), // n/a elShuffleForRight4
444 XCONST(0), // n/a elShuffleForRight5
445 XCONST(0), // n/a elShuffleForRight6
446 XCONST(0), // n/a elShuffleForLeft0
447 XCONST(0), // n/a elShuffleForLeft1
448 XCONST(0), // n/a elShuffleForLeft2
449 XCONST(0), // n/a elShuffleForLeft3
450 XCONST(0), // n/a elShuffleForLeft4
451 XCONST(0), // n/a elShuffleForLeft5
452 XCONST(0), // n/a elShuffleForLeft6
453 XCONST(88.3762626647949f), // exp_hi
454 XCONST(-88.3762626647949f), // exp_lo
455 XCONST(1.44269504088896341f), // cephes_LOG2EF
456 XCONST(0.693359375f), // cephes_exp_C1
457 XCONST(-2.12194440e-4f), // cephes_exp_C2
458 XCONST(1.9875691500E-4f), // cephes_exp_p0
459 XCONST(1.3981999507E-3f), // cephes_exp_p1
460 XCONST(8.3334519073E-3f), // cephes_exp_p2
461 XCONST(4.1665795894E-2f), // cephes_exp_p3
462 XCONST(1.6666665459E-1f), // cephes_exp_p4
463 XCONST(5.0000001201E-1f), // cephes_exp_p5
464 XCONST(0.707106781186547524f), // cephes_SQRTHF
465 XCONST(7.0376836292E-2f), // cephes_log_p0
466 XCONST(-1.1514610310E-1f), // cephes_log_p1
467 XCONST(1.1676998740E-1f), // cephes_log_p2
468 XCONST(-1.2420140846E-1f), // cephes_log_p3
469 XCONST(+1.4249322787E-1f), // cephes_log_p4
470 XCONST(-1.6668057665E-1f), // cephes_log_p5
471 XCONST(+2.0000714765E-1f), // cephes_log_p6
472 XCONST(-2.4999993993E-1f), // cephes_log_p7
473 XCONST(+3.3333331174E-1f), // cephes_log_p8
474 XCONST(0x3ea2f983), // float_invpi, 1/pi = 0.31830988618379067154f
475 XCONST(0x4b400000), // float_rintf Used for rounding
476 XCONST(0x40490000), // float_pi1 High precision part of Pi
477 XCONST(0x3a7da000), // float_pi2 Second part of Pi for extended precision
478 XCONST(0x34222000), // float_pi3 Third part of Pi for extended precision
479 XCONST(0x2cb4611a), // float_pi4 Fourth part of Pi for extended precision
480 XCONST(0xbe2aaaa6), // float_sinC3
481 XCONST(0x3c08876a), // float_sinC5
482 XCONST(0xb94fb7ff), // float_sinC7
483 XCONST(0x362edef8), // float_sinC9
484 XCONST(static_cast<int32_t>(0xBEFFFFE2)), // float_cosC2
485 XCONST(0x3D2AA73C), // float_cosC4
486 XCONST(static_cast<int32_t>(0XBAB58D50)), // float_cosC6
487 XCONST(0x37C1AD76), // float_cosC8
488 XCONST(0x3ccb7dda), // float_atan2f_rmul 0.024840285f
489 XCONST(0x3e3f4c37), // float_atan2f_radd 0.18681418f
490 XCONST(0xbdc0b66d), // float_atan2f_tmul -0.094097948f
491 XCONST(0xbeaa0d0a), // float_atan2f_tadd -0.33213072f
492 XCONST(0x3fc90fdb), // float_atan2f_halfpi 1.57079637f
493 XCONST(0x40490fdb), // float_atan2f_pi 3.14159274f
494 // Tangent approximation coefficients up to p8 and helpers
495 XCONST(1.0f), // float_tan_p0 - Numerator constant term
496 XCONST(0.3333314036f), // float_tan_p2 - Numerator x^2 coefficient
497 XCONST(0.1333923995f), // float_tan_p4 - Numerator x^4 coefficient
498 XCONST(0.0533740603f), // float_tan_p6 - Numerator x^6 coefficient
499 XCONST(0.0245650893f), // float_tan_p8 - Numerator x^8 coefficient
500 XCONST(1.0f), // float_tan_q0 - Denominator constant term
501 XCONST(0.1333835001f), // float_tan_q2 - Denominator x^2 coefficient
502 XCONST(0.0089270802f), // float_tan_q4 - Denominator x^4 coefficient
503 XCONST(0.0005908960f), // float_tan_q6 - Denominator x^6 coefficient
504 XCONST(0.0000342237f), // float_tan_q8 - Denominator x^8 coefficient
505 XCONST(1e-4f), // float_tan_small_limit
506 XCONST(0.97f), // float_tan_asympt_a1
507 XCONST(0.35f), // float_tan_asympt_a2
508 XCONST(0.8f), // float_tan_asympt_limit
509 };
510 #undef XCONST
511
512 #define CPTR_AVX(x) (ymmword_ptr[constptr + (x) * 32])
513
514 #define EXP_PS(x) { \
515 XmmReg fx, emm0, etmp, y, mask, z; \
516 minps(x, CPTR(elexp_hi)); \
517 maxps(x, CPTR(elexp_lo)); \
518 movaps(fx, x); \
519 mulps(fx, CPTR(elcephes_LOG2EF)); \
520 addps(fx, CPTR(elfloat_half)); \
521 cvttps2dq(emm0, fx); \
522 cvtdq2ps(etmp, emm0); \
523 movaps(mask, etmp); \
524 cmpnleps(mask, fx); \
525 andps(mask, CPTR(elfloat_one)); \
526 movaps(fx, etmp); \
527 subps(fx, mask); \
528 movaps(etmp, fx); \
529 mulps(etmp, CPTR(elcephes_exp_C1)); \
530 movaps(z, fx); \
531 mulps(z, CPTR(elcephes_exp_C2)); \
532 subps(x, etmp); \
533 subps(x, z); \
534 movaps(z, x); \
535 mulps(z, z); \
536 movaps(y, CPTR(elcephes_exp_p0)); \
537 mulps(y, x); \
538 addps(y, CPTR(elcephes_exp_p1)); \
539 mulps(y, x); \
540 addps(y, CPTR(elcephes_exp_p2)); \
541 mulps(y, x); \
542 addps(y, CPTR(elcephes_exp_p3)); \
543 mulps(y, x); \
544 addps(y, CPTR(elcephes_exp_p4)); \
545 mulps(y, x); \
546 addps(y, CPTR(elcephes_exp_p5)); \
547 mulps(y, z); \
548 addps(y, x); \
549 addps(y, CPTR(elfloat_one)); \
550 cvttps2dq(emm0, fx); \
551 paddd(emm0, CPTR(elc7F)); \
552 pslld(emm0, 23); \
553 mulps(y, emm0); \
554 x = y; }
555
556 #define LOG_PS(x) { \
557 XmmReg emm0, invalid_mask, mask, y, etmp, z; \
558 xorps(invalid_mask, invalid_mask); \
559 cmpnleps(invalid_mask, x); \
560 maxps(x, CPTR(elmin_norm_pos)); \
561 movaps(emm0, x); \
562 psrld(emm0, 23); \
563 andps(x, CPTR(elinv_mant_mask)); \
564 orps(x, CPTR(elfloat_half)); \
565 psubd(emm0, CPTR(elc7F)); \
566 cvtdq2ps(emm0, emm0); \
567 addps(emm0, CPTR(elfloat_one)); \
568 movaps(mask, x); \
569 cmpltps(mask, CPTR(elcephes_SQRTHF)); \
570 movaps(etmp, x); \
571 andps(etmp, mask); \
572 subps(x, CPTR(elfloat_one)); \
573 andps(mask, CPTR(elfloat_one)); \
574 subps(emm0, mask); \
575 addps(x, etmp); \
576 movaps(z, x); \
577 mulps(z, z); \
578 movaps(y, CPTR(elcephes_log_p0)); \
579 mulps(y, x); \
580 addps(y, CPTR(elcephes_log_p1)); \
581 mulps(y, x); \
582 addps(y, CPTR(elcephes_log_p2)); \
583 mulps(y, x); \
584 addps(y, CPTR(elcephes_log_p3)); \
585 mulps(y, x); \
586 addps(y, CPTR(elcephes_log_p4)); \
587 mulps(y, x); \
588 addps(y, CPTR(elcephes_log_p5)); \
589 mulps(y, x); \
590 addps(y, CPTR(elcephes_log_p6)); \
591 mulps(y, x); \
592 addps(y, CPTR(elcephes_log_p7)); \
593 mulps(y, x); \
594 addps(y, CPTR(elcephes_log_p8)); \
595 mulps(y, x); \
596 mulps(y, z); \
597 movaps(etmp, emm0); \
598 mulps(etmp, CPTR(elcephes_log_q1)); \
599 addps(y, etmp); \
600 mulps(z, CPTR(elfloat_half)); \
601 subps(y, z); \
602 mulps(emm0, CPTR(elcephes_log_q2)); \
603 addps(x, y); \
604 addps(x, emm0); \
605 orps(x, invalid_mask); }
606
607 #define EXP_PS_AVX(x) { \
608 YmmReg fx, emm0, etmp, y, mask, z; \
609 vminps(x, x, CPTR_AVX(elexp_hi)); \
610 vmaxps(x, x, CPTR_AVX(elexp_lo)); \
611 vmulps(fx, x, CPTR_AVX(elcephes_LOG2EF)); \
612 vaddps(fx, fx, CPTR_AVX(elfloat_half)); \
613 vcvttps2dq(emm0, fx); \
614 vcvtdq2ps(etmp, emm0); \
615 vcmpps(mask, etmp, fx, _CMP_GT_OQ); /* cmpnleps */ \
616 vandps(mask, mask, CPTR_AVX(elfloat_one)); \
617 vsubps(fx, etmp, mask); \
618 vfnmadd231ps(x, fx, CPTR_AVX(elcephes_exp_C1)); \
619 vfnmadd231ps(x, fx, CPTR_AVX(elcephes_exp_C2)); \
620 vmulps(z, x, x); \
621 vmovaps(y, CPTR_AVX(elcephes_exp_p0)); \
622 vfmadd213ps(y, x, CPTR_AVX(elcephes_exp_p1)); \
623 vfmadd213ps(y, x, CPTR_AVX(elcephes_exp_p2)); \
624 vfmadd213ps(y, x, CPTR_AVX(elcephes_exp_p3)); \
625 vfmadd213ps(y, x, CPTR_AVX(elcephes_exp_p4)); \
626 vfmadd213ps(y, x, CPTR_AVX(elcephes_exp_p5)); \
627 vfmadd213ps(y, z, x); \
628 vaddps(y, y, CPTR_AVX(elfloat_one)); \
629 vcvttps2dq(emm0, fx); \
630 vpaddd(emm0, emm0, CPTR_AVX(elc7F)); \
631 vpslld(emm0, emm0, 23); \
632 vmulps(x, y, emm0); \
633 }
634
635 #define LOG_PS_AVX(x) { \
636 YmmReg emm0, invalid_mask, mask, y, etmp, z; \
637 vcmpps(invalid_mask, zero, x, _CMP_GT_OQ); /* cmpnleps. or signalling _CMP_NLE_US? */ \
638 vmaxps(x, x, CPTR_AVX(elmin_norm_pos)); \
639 vpsrld(emm0, x, 23); \
640 vandps(x, x, CPTR_AVX(elinv_mant_mask)); \
641 vorps(x, x, CPTR_AVX(elfloat_half)); \
642 vpsubd(emm0, emm0, CPTR_AVX(elc7F)); \
643 vcvtdq2ps(emm0, emm0); \
644 vaddps(emm0, emm0, CPTR_AVX(elfloat_one)); \
645 vcmpps(mask, x, CPTR_AVX(elcephes_SQRTHF), _CMP_LT_OQ); /* cmpltps. or signalling _CMP_LT_OS? */ \
646 vandps(etmp, x, mask); \
647 vsubps(x, x, CPTR_AVX(elfloat_one)); \
648 vandps(mask, mask, CPTR_AVX(elfloat_one)); \
649 vsubps(emm0, emm0, mask); \
650 vaddps(x, x, etmp); \
651 vmulps(z, x, x); \
652 vmovaps(y, CPTR_AVX(elcephes_log_p0)); \
653 vfmadd213ps(y, x, CPTR_AVX(elcephes_log_p1)); \
654 vfmadd213ps(y, x, CPTR_AVX(elcephes_log_p2)); \
655 vfmadd213ps(y, x, CPTR_AVX(elcephes_log_p3)); \
656 vfmadd213ps(y, x, CPTR_AVX(elcephes_log_p4)); \
657 vfmadd213ps(y, x, CPTR_AVX(elcephes_log_p5)); \
658 vfmadd213ps(y, x, CPTR_AVX(elcephes_log_p6)); \
659 vfmadd213ps(y, x, CPTR_AVX(elcephes_log_p7)); \
660 vfmadd213ps(y, x, CPTR_AVX(elcephes_log_p8)); \
661 vmulps(y, y, x); \
662 vmulps(y, y, z); \
663 vfmadd231ps(y, emm0, CPTR_AVX(elcephes_log_q1)); \
664 vfnmadd231ps(y, z, CPTR_AVX(elfloat_half)); \
665 vaddps(x, x, y); \
666 vfmadd231ps(x, emm0, CPTR_AVX(elcephes_log_q2)); \
667 vorps(x, x, invalid_mask); }
668
669 // Note: VS Expr changed a lot since ported to Avisynth,
670 // Here we are using their VEX macros for easy port of new sin and cos
671 // however we do not support VEX encoding or FMA3 in xmm register mode
672 // Note: VEX2 cmpltps -> vcmpltps does not work so we uncomment v##op parts as well (comparisons changed in vex)
673 #define VEX1(op, arg1, arg2) \
674 do { \
675 if constexpr(false /*cpuFlags & CPUF_AVX*/) \
676 /*v##op(arg1, arg2)*/; \
677 else \
678 op(arg1, arg2); \
679 } while (0)
680 #define VEX1IMM(op, arg1, arg2, imm) \
681 do { \
682 if constexpr(false /*cpuFlags & CPUF_AVX*/) { \
683 /*v##op(arg1, arg2, imm)*/; \
684 } else if (arg1 == arg2) { \
685 op(arg2, imm); \
686 } else { \
687 movdqa(arg1, arg2); \
688 op(arg1, imm); \
689 } \
690 } while (0)
691 #define VEX2(op, arg1, arg2, arg3) \
692 do { \
693 if constexpr(false /*cpuFlags & CPUF_AVX*/) { \
694 /*v##op(arg1, arg2, arg3)*/; \
695 } else if (arg1 == arg2) { \
696 op(arg2, arg3); \
697 } else if (arg1 != arg3) { \
698 movdqa(arg1, arg2); \
699 op(arg1, arg3); \
700 } else { \
701 XmmReg tmp; \
702 movdqa(tmp, arg2); \
703 op(tmp, arg3); \
704 movdqa(arg1, tmp); \
705 } \
706 } while (0)
707 #define VEX2IMM(op, arg1, arg2, arg3, imm) \
708 do { \
709 if constexpr(false/*cpuFlags & CPUF_AVX*/) { \
710 /*v##op(arg1, arg2, arg3, imm)*/; \
711 } else if (arg1 == arg2) { \
712 op(arg2, arg3, imm); \
713 } else if (arg1 != arg3) { \
714 movdqa(arg1, arg2); \
715 op(arg1, arg3, imm); \
716 } else { \
717 XmmReg tmp; \
718 movdqa(tmp, arg2); \
719 op(tmp, arg3, imm); \
720 movdqa(arg1, tmp); \
721 } \
722 } while (0)
723
724 #if 0
725 // Fast tangent approximation using rational function with 8th order terms
726 float fast_tanf(float x) {
727 // Constants for Pi approximation and range reduction
728 const float polyPI = 3.14159265358979f;
729 const float halfPI = polyPI * 0.5f;
730 // Reduce to [-polyPI, polyPI] range
731 float y = fmodf(x, polyPI);
732 if (y > halfPI) y -= polyPI;
733 else if (y < -halfPI) y += polyPI;
734 // At this point y is in [-polyPI/2, polyPI/2]
735 // For very small angles, return the angle itself
736 // LLVM would add a small epsilon: sign(y)*2^-25 (0x1p-25f)
737 float abs_y = fabsf(y);
738 const float small_limit = 1e-4f;
739 if (abs_y < small_limit) {
740 return y;
741 }
742 // Check proximity to asymptotes
743 const float asympt_limit = 0.8f;
744 float distToAsymptote = halfPI - abs_y;
745 // If very close to ±polyPI/2, use asymptotic approximation
746 if (distToAsymptote < asympt_limit) {
747 // The tangent function approaches 1/distToAsymptote as y approaches +/-polyPI/2
748 // Improved coefficients based on curve fitting to better match std::tan
749 // not needed a 3rd term float asymptotic = 1.0f / (distToAsymptote * (0.9963f + distToAsymptote * (0.3642f + distToAsymptote * 0.0173f)));
750 float asymptotic = 1.0f / (distToAsymptote * (0.97f + distToAsymptote * 0.35f));
751 // Preserve sign
752 return (y < 0) ? -asymptotic : asymptotic;
753 }
754 float y2 = y * y;
755 // Optimized coefficients with terms up to 8th order
756 // Numerator coefficients
757 const float p0 = 1.0f;
758 const float p2 = 0.3333314036f;
759 const float p4 = 0.1333923995f;
760 const float p6 = 0.0533740603f;
761 const float p8 = 0.0245650893f;
762 // Denominator coefficients
763 const float q0 = 1.0f;
764 const float q2 = 0.1333835001f;
765 const float q4 = 0.0089270802f;
766 const float q6 = 0.0005908960f;
767 const float q8 = 0.0000342237f;
768 // Calculate approximation using Horner's method for efficiency
769 float num = y * (p0 + y2 * (p2 + y2 * (p4 + y2 * (p6 + y2 * p8))));
770 float den = q0 + y2 * (q2 + y2 * (q4 + y2 * (q6 + y2 * q8)));
771 return num / den;
772 }
773 // We are good with this, but LLVM does differently:
774 // https://github.com/llvm/llvm-project/blob/main/libc/src/math/generic/tanf.cpp
775 #endif
776
777 // Fast tangent approximation for non-AVX version
778 #define TAN_PS(x0) { \
779 XmmReg x1, x2, x3, x4, x5, x6, x7, x8, x9, x10; \
780 /* Normalize to [-pi, pi] using multiplication and subtraction */ \
781 VEX1(movaps, x1, CPTR(float_invpi)); /* 1/pi */ \
782 VEX2(mulps, x2, x0, x1); /* x / pi */ \
783 /* round to nearest integer */ \
784 VEX1(movaps, x3, CPTR(float_rintf)); /* round to int helper */ \
785 VEX2(addps, x4, x3, x2); /* x/pi + 2^23 */ \
786 VEX2(subps, x4, x4, x3); /* round(x/pi) */ \
787 VEX1(movaps, x5, CPTR(float_pi1)); /* Load pi1 (highest precision part) */ \
788 VEX2(mulps, x6, x4, x5); /* round(x/pi) * pi1 */ \
789 VEX2(subps, x7, x0, x6); /* Remainder after subtracting largest part */ \
790 /* Subtract remaining parts for higher precision */ \
791 VEX1(movaps, x3, CPTR(float_pi2)); \
792 VEX2(mulps, x3, x4, x3); /* round(x/pi) * pi2 */ \
793 VEX2(subps, x7, x7, x3); /* Further refine remainder */ \
794 VEX1(movaps, x3, CPTR(float_pi3)); \
795 VEX2(mulps, x3, x4, x3); /* round(x/pi) * pi3 */ \
796 VEX2(subps, x7, x7, x3); /* Further refine remainder */ \
797 VEX1(movaps, x3, CPTR(float_pi4)); \
798 VEX2(mulps, x3, x4, x3); /* round(x/pi) * pi4 */ \
799 VEX2(subps, x7, x7, x3); /* Final remainder, now x is in range [-pi, pi] */ \
800 /* Check range for symmetry, normalize to [-pi/2, pi/2] */ \
801 VEX2(mulps, x6, x5, CPTR(elfloat_half)); /* halfPI = pi/2 */ \
802 VEX2(cmpltps, x3, x6, x7); /* halfPI < y (cmpgt -> cmplt)*/ \
803 VEX2(subps, x4, x5, x7); /* pi - y */ \
804 /* blend: if y > halfPI, use pi - y */ \
805 VEX1(movaps, x2, x3); \
806 VEX2(andps, x2, x2, x4); \
807 VEX2(andnps, x3, x3, x7); \
808 VEX2(orps, x7, x3, x2); /* now y is <= pi/2 */ \
809 /* Check for y < -halfPI */ \
810 VEX1(movaps, x3, x6); \
811 VEX2(xorps, x3, x3, CPTR(elsignmask)); /* -halfPI */ \
812 VEX2(cmpltps, x2, x7, x3); /* y < -halfPI */ \
813 VEX2(xorps, x4, x5, CPTR(elsignmask)); /* -pi */ \
814 VEX2(subps, x4, x4, x7); /* -pi - y */ \
815 /* blend: if y < -halfPI, use -pi - y */ \
816 VEX1(movaps, x3, x2); \
817 VEX2(andps, x3, x3, x4); \
818 VEX2(andnps, x2, x2, x7); \
819 VEX2(orps, x7, x2, x3); /* now y is in [-pi/2, pi/2] */ \
820 /* Range reduction end */ \
821 \
822 /* Check for small values */ \
823 VEX1(movaps, x2, x7); \
824 VEX2(andps, x2, x2, CPTR(elabsmask)); /* abs_y */ \
825 VEX2(cmpltps, x3, x2, CPTR(float_tan_small_limit)); /* abs_y < small_limit */ \
826 VEX2(andps, x3, x3, x7); /* y * (abs_y < small_limit) */ \
827 \
828 /* Check for asymptotic proximity */ \
829 VEX1(movaps, x4, x6); /* Load halfPI */ \
830 VEX2(subps, x4, x4, x2); /* distToAsymptote = halfPI - abs_y */ \
831 VEX2(cmpltps, x5, x4, CPTR(float_tan_asympt_limit)); /* distToAsymptote < asympt_limit */ \
832 \
833 /* Calculate asymptotic approximation: 1.0f / (distToAsymptote * (0.97f + distToAsymptote * 0.35f)) */ \
834 VEX2(mulps, x8, x4, CPTR(float_tan_asympt_a2)); /* distToAsymptote * 0.35f */ \
835 VEX2(addps, x8, x8, CPTR(float_tan_asympt_a1)); /* 0.97f + distToAsymptote * 0.35f */ \
836 VEX2(mulps, x8, x8, x4); /* distToAsymptote * (0.97f + distToAsymptote * 0.35f) */ \
837 VEX1(movaps, x4, CPTR(elfloat_one)); /* 1.0f */ \
838 VEX2(divps, x9, x4, x8); /* 1.0f / (distToAsymptote * (0.97f + distToAsymptote * 0.35f)) */ \
839 \
840 /* Adjust sign for asymptotic approximation */ \
841 VEX1(movaps, x8, x7); /* original y */ \
842 VEX2(andps, x8, x8, CPTR(elsignmask)); /* sign of y */ \
843 VEX2(xorps, x9, x9, x8); /* apply sign to asymptotic result */ \
844 \
845 /* Calculate rational approximation for regular values */ \
846 VEX1(movaps, x2, x7); \
847 VEX2(mulps, x2, x2, x7); /* y^2 */ \
848 \
849 /* Compute numerator with higher order terms */ \
850 VEX1(movaps, x10, CPTR(float_tan_p8)); /* p8 */ \
851 VEX2(mulps, x10, x10, x2); /* p8*y^2 */ \
852 VEX2(addps, x10, x10, CPTR(float_tan_p6)); /* p6 + p8*y^2 */ \
853 VEX2(mulps, x10, x10, x2); /* y^2 * (p6 + p8*y^2) */ \
854 VEX2(addps, x10, x10, CPTR(float_tan_p4)); /* p4 + y^2 * (p6 + p8*y^2) */ \
855 VEX2(mulps, x10, x10, x2); /* y^2 * (p4 + y^2 * (p6 + p8*y^2)) */ \
856 VEX2(addps, x10, x10, CPTR(float_tan_p2)); /* p2 + y^2 * (p4 + y^2 * (p6 + p8*y^2)) */ \
857 VEX2(mulps, x10, x10, x2); /* y^2 * (p2 + y^2 * (p4 + y^2 * (p6 + p8*y^2))) */ \
858 VEX2(addps, x10, x10, CPTR(float_tan_p0)); /* p0 + y^2 * (p2 + y^2 * (p4 + y^2 * (p6 + p8*y^2))) */ \
859 VEX2(mulps, x10, x10, x7); /* y * (p0 + y^2 * (p2 + y^2 * (p4 + y^2 * (p6 + p8*y^2)))) */ \
860 \
861 /* Compute denominator with higher order terms */ \
862 VEX1(movaps, x8, CPTR(float_tan_q8)); /* q8 */ \
863 VEX2(mulps, x8, x8, x2); /* q8*y^2 */ \
864 VEX2(addps, x8, x8, CPTR(float_tan_q6)); /* q6 + q8*y^2 */ \
865 VEX2(mulps, x8, x8, x2); /* y^2 * (q6 + q8*y^2) */ \
866 VEX2(addps, x8, x8, CPTR(float_tan_q4)); /* q4 + y^2 * (q6 + q8*y^2) */ \
867 VEX2(mulps, x8, x8, x2); /* y^2 * (q4 + y^2 * (q6 + q8*y^2)) */ \
868 VEX2(addps, x8, x8, CPTR(float_tan_q2)); /* q2 + y^2 * (q4 + y^2 * (q6 + q8*y^2)) */ \
869 VEX2(mulps, x8, x8, x2); /* y^2 * (q2 + y^2 * (q4 + y^2 * (q6 + q8*y^2))) */ \
870 VEX2(addps, x8, x8, CPTR(float_tan_q0)); /* q0 + y^2 * (q2 + y^2 * (q4 + y^2 * (q6 + q8*y^2))) */ \
871 \
872 /* Division: numerator/denominator for regular case */ \
873 VEX2(divps, x1, x10, x8); /* Final result: tan(y) = numerator/denominator */ \
874 \
875 /* Select the appropriate result based on conditions */ \
876 /* If close to asymptote, use asymptotic approximation, else use rational approximation */ \
877 VEX1(movaps, x2, x5); /* Load asymptote condition */ \
878 VEX2(andps, x2, x2, x9); /* asymptotic * (distToAsymptote < asympt_limit) */ \
879 VEX2(andnps, x5, x5, x1); /* rational * !(distToAsymptote < asympt_limit) */ \
880 VEX2(orps, x1, x5, x2); /* Blend asymptotic and rational results */ \
881 \
882 /* If very small value, use y itself, otherwise use computed result */ \
883 VEX2(andnps, x5, x3, x1); /* result * !(abs_y < small_limit) */ \
884 VEX2(orps, x0, x3, x5); /* Final result with small value handling */ \
885 }
886
887 // Fast tangent approximation for AVX version
888 #define TAN_PS_AVX(x0) { \
889 YmmReg x1, x2, x3, x4, x5, x6, x7, x8, x9, x10; \
890 /* Normalize to [-pi, pi] using multiplication and subtraction */ \
891 vmovaps(x1, CPTR_AVX(float_invpi)); /* 1/pi */ \
892 vmulps(x2, x0, x1); /* x / pi */ \
893 vroundps(x3, x2, FROUND_TO_NEAREST_INT); /* round(x / pi) */ \
894 vmovaps(x4, CPTR_AVX(float_pi1)); /* Load pi1 (highest precision part) */ \
895 vmulps(x5, x3, x4); /* round(x / pi) * pi1 */ \
896 vsubps(x6, x0, x5); /* Remainder after subtracting largest part */ \
897 /* Subtract remaining parts for higher precision */ \
898 vmovaps(x7, CPTR_AVX(float_pi2)); \
899 vmulps(x7, x3, x7); /* round(x / pi) * pi2 */ \
900 vsubps(x6, x6, x7); /* Further refine remainder */ \
901 vmovaps(x7, CPTR_AVX(float_pi3)); \
902 vmulps(x7, x3, x7); /* round(x / pi) * pi3 */ \
903 vsubps(x6, x6, x7); /* Further refine remainder */ \
904 vmovaps(x7, CPTR_AVX(float_pi4)); \
905 vmulps(x7, x3, x7); /* round(x / pi) * pi4 */ \
906 vsubps(x6, x6, x7); /* Final remainder, now x is in range [-pi, pi] */ \
907 /* Check range for symmetry, normalize to [-pi/2, pi/2] */ \
908 vmulps(x5, x4, CPTR_AVX(elfloat_half)); /* halfPI = pi/2 */ \
909 vcmpps(x7, x6, x5, _CMP_GT_OQ); /* y > halfPI */ \
910 vsubps(x8, x4, x6); /* pi - y */ \
911 vblendvps(x6, x6, x8, x7); /* if (y > halfPI) y = pi - y; */ \
912 /* Check for y < -halfPI */ \
913 vxorps(x7, x5, CPTR_AVX(elsignmask)); /* -halfPI */ \
914 vcmpps(x8, x6, x7, _CMP_LT_OQ); /* y < -halfPI */ \
915 vxorps(x2, x4, CPTR_AVX(elsignmask)); /* -pi */ \
916 vsubps(x2, x2, x6); /* -pi - y */ \
917 vblendvps(x6, x6, x2, x8); /* if (y < -halfPI) y = -pi - y; */ \
918 /* now y is in [-pi/2, pi/2] */ \
919 /* Range reduction end */ \
920 \
921 /* Check for small values */ \
922 vandps(x2, x6, CPTR_AVX(elabsmask)); /* abs_y */ \
923 vcmpps(x3, x2, CPTR_AVX(float_tan_small_limit), _CMP_LT_OQ); /* abs_y < small_limit */ \
924 /* Save small value result for later blending */ \
925 vandps(x7, x6, x3); /* y * (abs_y < small_limit) */ \
926 \
927 /* Check for asymptotic proximity */ \
928 vmovaps(x4, x5); /* Load halfPI */ \
929 vsubps(x4, x4, x2); /* distToAsymptote = halfPI - abs_y */ \
930 vcmpps(x5, x4, CPTR_AVX(float_tan_asympt_limit), _CMP_LT_OQ); /* distToAsymptote < asympt_limit */ \
931 \
932 /* Calculate asymptotic approximation */ \
933 vmulps(x8, x4, CPTR_AVX(float_tan_asympt_a2)); /* distToAsymptote * 0.35f */ \
934 vaddps(x8, x8, CPTR_AVX(float_tan_asympt_a1)); /* 0.97f + distToAsymptote * 0.35f */ \
935 vmulps(x8, x8, x4); /* distToAsymptote * (0.97f + distToAsymptote * 0.35f) */ \
936 vmovaps(x4, CPTR_AVX(elfloat_one)); /* Load 1.0f */ \
937 vdivps(x9, x4, x8); /* 1.0f / (distToAsymptote * (0.97f + distToAsymptote * 0.35f)) */ \
938 \
939 /* Adjust sign for asymptotic approximation */ \
940 vmovaps(x8, x6); /* original y */ \
941 vandps(x8, x8, CPTR_AVX(elsignmask)); /* sign of y */ \
942 vxorps(x9, x9, x8); /* apply sign to asymptotic result */ \
943 \
944 /* Calculate rational approximation for regular values */ \
945 vmulps(x2, x6, x6); /* y^2 */ \
946 \
947 /* Compute numerator with higher order terms using FMA instructions */ \
948 vmovaps(x10, CPTR_AVX(float_tan_p8)); /* p8 */ \
949 vfmadd213ps(x10, x2, CPTR_AVX(float_tan_p6)); /* p6 + p8*y^2 */ \
950 vfmadd213ps(x10, x2, CPTR_AVX(float_tan_p4)); /* p4 + y^2 * (p6 + p8*y^2) */ \
951 vfmadd213ps(x10, x2, CPTR_AVX(float_tan_p2)); /* p2 + y^2 * (p4 + y^2 * (p6 + p8*y^2)) */ \
952 vfmadd213ps(x10, x2, CPTR_AVX(float_tan_p0)); /* p0 + y^2 * (p2 + y^2 * (p4 + y^2 * (p6 + p8*y^2))) */ \
953 vmulps(x10, x10, x6); /* y * (p0 + y^2 * (p2 + y^2 * (p4 + y^2 * (p6 + p8*y^2)))) */ \
954 \
955 /* Compute denominator with higher order terms using FMA instructions */ \
956 vmovaps(x8, CPTR_AVX(float_tan_q8)); /* q8 */ \
957 vfmadd213ps(x8, x2, CPTR_AVX(float_tan_q6)); /* q6 + q8*y^2 */ \
958 vfmadd213ps(x8, x2, CPTR_AVX(float_tan_q4)); /* q4 + y^2 * (q6 + q8*y^2) */ \
959 vfmadd213ps(x8, x2, CPTR_AVX(float_tan_q2)); /* q2 + y^2 * (q4 + y^2 * (q6 + q8*y^2)) */ \
960 vfmadd213ps(x8, x2, CPTR_AVX(float_tan_q0)); /* q0 + y^2 * (q2 + y^2 * (q4 + y^2 * (q6 + q8*y^2))) */ \
961 \
962 /* Division: numerator/denominator */ \
963 vdivps(x1, x10, x8); /* tan(y) = numerator/denominator */ \
964 \
965 /* Select appropriate result based on conditions */ \
966 /* If close to asymptote, use asymptotic approximation, else use rational approximation */ \
967 vblendvps(x1, x1, x9, x5); /* Blend asymptotic and rational results */ \
968 \
969 /* If very small value, use y itself, otherwise use computed result */ \
970 vblendvps(x0, x1, x7, x3); /* Final result with small value handling */ \
971 }
972
973 // atan2: based on https://stackoverflow.com/questions/46210708/atan2-approximation-with-11bits-in-mantissa-on-x86with-sse2-and-armwith-vfpv4?noredirect=1&lq=1
974 #if 0
975 float fast_atan2f(float y, float x)
976 {
977 // max rel err = 3.53486939e-5
978 const float atan2f_rmul = 0.024840285f;
979 const float atan2f_radd = 0.18681418f;
980 const float atan2f_tmul = -0.094097948f;
981 const float atan2f_tadd = -0.33213072f;
982 const float atan2f_halfpi = 1.57079637f;
983 const float atan2f_pi = 3.14159274f;
984
985 float a, r, s, t, c, q, ax, ay, mx, mn;
986 ax = fabsf(x);
987 ay = fabsf(y);
988
989 mx = fmaxf(ay, ax);
990 mn = fminf(ay, ax);
991 a = mn / mx;
992 // Minimax polynomial approximation to atan(a) on [0,1]
993 s = a * a;
994 c = s * a;
995 q = s * s;
996 r = atan2f_rmul * q + atan2f_radd;
997 t = atan2f_tmul * q + atan2f_tadd;
998 r = r * s + t;
999 r = r * c + a;
1000 // Map to full circle
1001 if (ay > ax) r = atan2f_halfpi - r;
1002 if (x < 0) r = atan2f_pi - r;
1003 if (y < 0) r = -r;
1004 return r;
1005 }
1006 #endif
1007
1008 // atan2(0, 0) = 0
1009 // ~speed: "y x atan2" C/SSE2/AVX2:52/480/1000 fps
1010 #define ATAN2_PS(x0 /*y*/, /*x*/x1) { \
1011 XmmReg x2, x3, x4, x5, x6, x7, x8; \
1012 /*VEX1(movaps, x1, x);*/ \
1013 /*VEX1(movaps, x0, y);*/ \
1014 /* Remove sign */ \
1015 VEX1(movaps, x3, CPTR(elabsmask)); \
1016 VEX1(movaps, x2, x1); \
1017 VEX2(andps, x2, x2, x3); /* ax = fabsf (x); */ \
1018 VEX2(andps, x3, x3, x0); /* ay = fabsf (y); */ \
1019 VEX1(movaps, x4, x3); \
1020 VEX2(maxps, x4, x4, x2); /* mx = fmaxf (ay, ax); */ \
1021 VEX1(movaps, x5, x3); \
1022 VEX2(minps, x5, x5, x2); /* fminf (ay, ax); */ \
1023 VEX2(divps, x5, x5, x4); /* a = mn / mx; */ \
1024 VEX1(movaps, x4, x5); \
1025 VEX2(mulps, x4, x4, x5); /* s = a * a; */ \
1026 VEX1(movaps, x6, x4); \
1027 VEX2(mulps, x6, x6, x4); /* q = s * s; */ \
1028 VEX1(movaps, x7, CPTR(float_atan2f_rmul)); \
1029 VEX2(mulps, x7, x7, x6); /* r = atan2f_rmul * q (+ atan2f_radd) */ \
1030 VEX2(addps, x7, x7, CPTR(float_atan2f_radd)); /* r = (atan2f_rmul * q) + atan2f_radd; */ \
1031 VEX2(mulps, x7, x7, x4); /* r = r * s (+ t) */ \
1032 VEX2(mulps, x4, x4, x5); /* c = s * a; */ \
1033 VEX2(mulps, x6, x6, CPTR(float_atan2f_tmul)); /* t = atan2f_tmul * q (+ atan2f_tadd) */ \
1034 VEX2(addps, x6, x6, CPTR(float_atan2f_tadd)); /* t = (atan2f_tmul * q) + atan2f_tadd; */ \
1035 VEX2(addps, x7, x7, x6); /* r = (r * s) + t; */ \
1036 VEX2(mulps, x7, x7, x4); /* r = r * c (+ a) */ \
1037 VEX2(addps, x7, x7, x5); /* r = (r * c) + a */ \
1038 /* Map to full circle */ \
1039 /* if (ay > ax) r = atan2f_halfpi - r; */ \
1040 /* if (x < 0) r = atan2f_pi - r; */ \
1041 /* if (y < 0) r = -r; */ \
1042 /* r = atan2f_halfpi - r */ \
1043 VEX1(movaps, x4, CPTR(float_atan2f_halfpi)); \
1044 VEX2(subps, x4, x4, x7); /* r = atan2f_halfpi - r */ \
1045 VEX2(cmpltps, x2, x2, x3); /* if (ay > ax) */ \
1046 /* blend */ \
1047 VEX1(movaps, x3, x2); \
1048 VEX2(andnps, x3, x3, x7); \
1049 VEX2(andps, x2, x2, x4); \
1050 VEX2(orps, x2, x2, x3); \
1051 /* r = atan2f_pi - r; */ \
1052 VEX1(movaps, x3, CPTR(float_atan2f_pi)); \
1053 VEX2(subps, x3, x3, x2); /* r = atan2f_pi - r */ \
1054 /* if (x < 0) */ \
1055 VEX2(xorps, x4, x4, x4); /* zero */ \
1056 VEX2(cmpltps, x1, x1, x4); /* if (x < 0) */ \
1057 /* blend */ \
1058 VEX1(movaps, x5, x1); \
1059 VEX2(andnps, x5, x5, x2); \
1060 VEX2(andps, x1, x1, x3); \
1061 VEX2(orps, x1, x1, x5); \
1062 /* r = -r; */ \
1063 VEX1(movaps, x2, CPTR(elsignmask)); \
1064 VEX2(subps, x2, x2, x1); /* r = -r */ \
1065 /* if (y < 0) */ \
1066 VEX2(cmpltps, x0, x0, x4); /* if (y < 0) */ \
1067 /* blend */ \
1068 VEX1(movaps, x3, x0); \
1069 VEX2(andnps, x3, x3, x1); \
1070 VEX2(andps, x0, x0, x2); \
1071 VEX2(orps, x0, x0, x3); \
1072 /* extra check when 0,0 given -> convert NaN to 0 */ \
1073 VEX1(movaps, x3, x0); \
1074 VEX2(cmpordps, x3, x3, x3);/* find NaNs. 0: NaN in either. FFFF: both non-Nan */ \
1075 /* mask NaN to zero */ \
1076 VEX2(andps, x0, x0, x3); \
1077 /* return value in y */ \
1078 /* input was "x0 for y */ \
1079 }
1080
1081 #define ATAN2_PS_AVX(x0 /*y*/, x1 /*x*/) { \
1082 YmmReg x2, x3, x4, x5, x6, x7, x8; \
1083 /* Remove sign */ \
1084 /* vmovaps(x1, x); */ \
1085 /* vmovaps(x0, y); */ \
1086 vmovaps(x2, CPTR_AVX(elabsmask)); \
1087 vandps(x8, x1, x2); /* ax = fabsf (x); */ \
1088 vandps(x2, x0, x2); /* ay = fabsf (y); */ \
1089 vmaxps(x4, x8, x2); /* mx = fmaxf (ay, ax); */ \
1090 vminps(x5, x8, x2); /* fminf (ay, ax); */ \
1091 vdivps(x4, x5, x4); /* a = mn / mx; */ \
1092 vmulps(x5, x4, x4); /* s = a * a; */ \
1093 vmulps(x6, x5, x5); /* q = s * s; */ \
1094 vmovaps(x7, CPTR_AVX(float_atan2f_rmul)); \
1095 vmovaps(x3, CPTR_AVX(float_atan2f_radd)); \
1096 vfmadd213ps(x7, x6, CPTR_AVX(float_atan2f_radd)); /* r = atan2f_rmul * q + atan2f_radd; */ \
1097 vmovaps(x3, CPTR_AVX(float_atan2f_tmul)); \
1098 vfmadd213ps(x3, x6, CPTR_AVX(float_atan2f_tadd)); /* t = atan2f_tmul * q + atan2f_tadd */ \
1099 vmulps(x6, x5, x4); \
1100 vfmadd231ps(x3, x5, x7); /* r = r * s + t; */ \
1101 vfmadd213ps(x3, x6, x4); /* r = (r * c) + a */ \
1102 /* Map to full circle */ \
1103 /* if (ay > ax) r = atan2f_halfpi - r; */ \
1104 /* if (x < 0) r = atan2f_pi - r; */ \
1105 /* if (y < 0) r = -r; */ \
1106 /* r = atan2f_pi - r */ \
1107 vmovaps(x4, CPTR_AVX(float_atan2f_halfpi)); \
1108 vsubps(x4, x4, x3); /* r = atan2f_halfpi - r */ \
1109 vcmpps(x2, x8, x2, _CMP_LT_OQ); /*vcmpltps(x2, x8, x2);*/ /* if (ay > ax) */ \
1110 vblendvps(x2, x3, x4, x2); \
1111 vmovaps(x3, CPTR_AVX(float_atan2f_pi)); \
1112 vsubps(x3, x3, x2); /* r = atan2f_pi - r */ \
1113 vxorps(x4, x4, x4); \
1114 vcmpps(x1, x1, x4, _CMP_LT_OQ); /* vcmpltps(x1, x1, x4); */ /* if (x < 0) */ \
1115 vblendvps(x1, x2, x3, x1); \
1116 vmovaps(x2, CPTR_AVX(elsignmask)); \
1117 vxorps(x2, x1, x2); /* r = -r */ \
1118 vcmpps(x0, x0, x4, _CMP_LT_OQ); /* vcmpltps(x0, x0, x4); */ /* if (y < 0) */ \
1119 vblendvps(x0, x1, x2, x0); \
1120 /* extra check when 0,0 given -> convert NaN to 0 */ \
1121 vcmpps(x3, x0, x0, _CMP_ORD_Q); /* vcmpordps, x3, x3, x3);*/ /* find NaNs. 0: NaN in either. FFFF: both non-Nan */ \
1122 /* mask NaN to zero */ \
1123 vandps(x0, x0, x3); \
1124 /* return value in y */ \
1125 /* no need. input was "x0" for y */ \
1126 }
1127
1128 #define SINCOS_PS(issin, y, x) { \
1129 XmmReg t1, sign, t2, t3, t4; \
1130 /* // Remove sign */ \
1131 VEX1(movaps, t1, CPTR(elabsmask)); \
1132 if (issin) { \
1133 VEX1(movaps, sign, t1); \
1134 VEX2(andnps, sign, sign, x); \
1135 } \
1136 else { \
1137 VEX2(pxor, sign, sign, sign); \
1138 } \
1139 VEX2(andps, t1, t1, x); \
1140 /*// Range reduction*/ \
1141 VEX1(movaps, t3, CPTR(float_rintf)); \
1142 VEX2(mulps, t2, t1, CPTR(float_invpi)); \
1143 VEX2(addps, t2, t2, t3); \
1144 VEX1IMM(pslld, t4, t2, 31); \
1145 VEX2(xorps, sign, sign, t4); \
1146 VEX2(subps, t2, t2, t3); \
1147 if constexpr(false /*cpuFlags & CPUF_FMA3*/) { \
1148 vfnmadd231ps(t1, t2, CPTR(float_pi1)); \
1149 vfnmadd231ps(t1, t2, CPTR(float_pi2)); \
1150 vfnmadd231ps(t1, t2, CPTR(float_pi3)); \
1151 vfnmadd231ps(t1, t2, CPTR(float_pi4)); \
1152 } \
1153 else { \
1154 VEX2(mulps, t4, t2, CPTR(float_pi1)); \
1155 VEX2(subps, t1, t1, t4); \
1156 VEX2(mulps, t4, t2, CPTR(float_pi2)); \
1157 VEX2(subps, t1, t1, t4); \
1158 VEX2(mulps, t4, t2, CPTR(float_pi3)); \
1159 VEX2(subps, t1, t1, t4); \
1160 VEX2(mulps, t4, t2, CPTR(float_pi4)); \
1161 VEX2(subps, t1, t1, t4); \
1162 } \
1163 if (issin) { \
1164 /* // Evaluate minimax polynomial for sin(x) in [-pi/2, pi/2] interval */ \
1165 /* // Y <- X + X * X^2 * (C3 + X^2 * (C5 + X^2 * (C7 + X^2 * C9))) */ \
1166 VEX2(mulps, t2, t1, t1); \
1167 if constexpr(false /*cpuFlags & CPUF_FMA3*/) { \
1168 vmovaps(t3, CPTR(float_sinC7)); \
1169 vfmadd231ps(t3, t2, CPTR(float_sinC9)); \
1170 vfmadd213ps(t3, t2, CPTR(float_sinC5)); \
1171 vfmadd213ps(t3, t2, CPTR(float_sinC3)); \
1172 VEX2(mulps, t3, t3, t2); \
1173 vfmadd231ps(t1, t1, t3); \
1174 } \
1175 else { \
1176 VEX2(mulps, t3, t2, CPTR(float_sinC9)); \
1177 VEX2(addps, t3, t3, CPTR(float_sinC7)); \
1178 VEX2(mulps, t3, t3, t2); \
1179 VEX2(addps, t3, t3, CPTR(float_sinC5)); \
1180 VEX2(mulps, t3, t3, t2); \
1181 VEX2(addps, t3, t3, CPTR(float_sinC3)); \
1182 VEX2(mulps, t3, t3, t2); \
1183 VEX2(mulps, t3, t3, t1); \
1184 VEX2(addps, t1, t1, t3); \
1185 } \
1186 } \
1187 else { \
1188 /* // Evaluate minimax polynomial for cos(x) in [-pi/2, pi/2] interval */ \
1189 /* // Y <- 1 + X^2 * (C2 + X^2 * (C4 + X^2 * (C6 + X^2 * C8))) */ \
1190 VEX2(mulps, t2, t1, t1); \
1191 if constexpr(false /*cpuFlags & CPUF_FMA3*/) { \
1192 vmovaps(t1, CPTR(float_cosC6)); \
1193 vfmadd231ps(t1, t2, CPTR(float_cosC8)); \
1194 vfmadd213ps(t1, t2, CPTR(float_cosC4)); \
1195 vfmadd213ps(t1, t2, CPTR(float_cosC2)); \
1196 vfmadd213ps(t1, t2, CPTR(elfloat_one)); \
1197 } \
1198 else { \
1199 VEX2(mulps, t1, t2, CPTR(float_cosC8)); \
1200 VEX2(addps, t1, t1, CPTR(float_cosC6)); \
1201 VEX2(mulps, t1, t1, t2); \
1202 VEX2(addps, t1, t1, CPTR(float_cosC4)); \
1203 VEX2(mulps, t1, t1, t2); \
1204 VEX2(addps, t1, t1, CPTR(float_cosC2)); \
1205 VEX2(mulps, t1, t1, t2); \
1206 VEX2(addps, t1, t1, CPTR(elfloat_one)); \
1207 } \
1208 } \
1209 /*// Apply sign */ \
1210 VEX2(xorps, y, t1, sign); \
1211 }
1212
1213 // y dst x src
1214 #define SINCOS_PS_AVX(issin, y, x) { \
1215 YmmReg t1, sign, t2, t3, t4; \
1216 /* // Remove sign */ \
1217 vmovaps(t1, CPTR_AVX(elabsmask)); \
1218 if (issin) { \
1219 vmovaps(sign, t1); \
1220 vandnps(sign, sign, x); \
1221 } \
1222 else { \
1223 vxorps(sign, sign, sign); \
1224 } \
1225 vandps(t1, t1, x); \
1226 /*// Range reduction*/ \
1227 vmovaps(t3, CPTR_AVX(float_rintf)); \
1228 vmulps(t2, t1, CPTR_AVX(float_invpi)); \
1229 vaddps(t2, t2, t3); \
1230 vpslld(t4, t2, 31); \
1231 vxorps(sign, sign, t4); \
1232 vsubps(t2, t2, t3); \
1233 vfnmadd231ps(t1, t2, CPTR_AVX(float_pi1)); \
1234 vfnmadd231ps(t1, t2, CPTR_AVX(float_pi2)); \
1235 vfnmadd231ps(t1, t2, CPTR_AVX(float_pi3)); \
1236 vfnmadd231ps(t1, t2, CPTR_AVX(float_pi4)); \
1237 if (issin) { \
1238 /* // Evaluate minimax polynomial for sin(x) in [-pi/2, pi/2] interval */ \
1239 /* // Y <- X + X * X^2 * (C3 + X^2 * (C5 + X^2 * (C7 + X^2 * C9))) */ \
1240 vmulps(t2, t1, t1); \
1241 vmovaps(t3, CPTR_AVX(float_sinC7)); \
1242 vfmadd231ps(t3, t2, CPTR_AVX(float_sinC9)); \
1243 vfmadd213ps(t3, t2, CPTR_AVX(float_sinC5)); \
1244 vfmadd213ps(t3, t2, CPTR_AVX(float_sinC3)); \
1245 vmulps(t3, t3, t2); \
1246 vfmadd231ps(t1, t1, t3); \
1247 } \
1248 else { \
1249 /* // Evaluate minimax polynomial for cos(x) in [-pi/2, pi/2] interval */ \
1250 /* // Y <- 1 + X^2 * (C2 + X^2 * (C4 + X^2 * (C6 + X^2 * C8))) */ \
1251 vmulps(t2, t1, t1); \
1252 vmovaps(t1, CPTR_AVX(float_cosC6)); \
1253 vfmadd231ps(t1, t2, CPTR_AVX(float_cosC8)); \
1254 vfmadd213ps(t1, t2, CPTR_AVX(float_cosC4)); \
1255 vfmadd213ps(t1, t2, CPTR_AVX(float_cosC2)); \
1256 vfmadd213ps(t1, t2, CPTR_AVX(elfloat_one)); \
1257 } \
1258 /*// Apply sign */ \
1259 vxorps(y, t1, sign); \
1260 }
1261
1262 // return (x - std::round(x / d)*d);
1263 #define FMOD_PS(x, d) { \
1264 XmmReg aTmp; \
1265 movaps(aTmp, x); \
1266 divps(aTmp, d); \
1267 cvttps2dq(aTmp,aTmp); \
1268 cvtdq2ps(aTmp,aTmp); \
1269 mulps(aTmp, d); \
1270 subps(x, aTmp); }
1271
1272 #define FMOD_PS_AVX(x, d) { \
1273 YmmReg aTmp; \
1274 vdivps(aTmp, x, d); \
1275 vcvttps2dq(aTmp,aTmp); \
1276 vcvtdq2ps(aTmp,aTmp); \
1277 vmulps(aTmp, aTmp, d); \
1278 vsubps(x, x, aTmp); }
1279
1280 struct ExprEval : public jitasm::function<void, ExprEval, uint8_t *, const intptr_t *, intptr_t, intptr_t> {
1281
1282 std::vector<ExprOp> ops;
1283 int numInputs;
1284 int cpuFlags;
1285 int planeheight;
1286 int planewidth;
1287 bool singleMode;
1288 int labelCount; // to have unique label strings
1289
1290 ✗ std::string getLabelCount()
1291 {
1292 ✗ return std::to_string(++labelCount);
1293 }
1294
1295 ✗ ExprEval(std::vector<ExprOp> &ops, int numInputs, int cpuFlags, int planewidth, int planeheight, bool singleMode) : ops(ops), numInputs(numInputs), cpuFlags(cpuFlags),
1296 ✗ planeheight(planeheight), planewidth(planewidth), singleMode(singleMode), labelCount(0) {}
1297
1298 AVS_FORCEINLINE void doMask(XmmReg &r, Reg &constptr, int _planewidth)
1299 {
1300 ✗ switch (_planewidth & 3) {
1301 ✗ case 1: andps(r, CPTR(loadmask1000)); break;
1302 ✗ case 2: andps(r, CPTR(loadmask1100)); break;
1303 ✗ case 3: andps(r, CPTR(loadmask1110)); break;
1304 }
1305 ✗ }
1306
1307 template<bool processSingle, bool maskUnused>
1308 AVS_FORCEINLINE void processingLoop(Reg &regptrs, XmmReg &zero, Reg &constptr, Reg &SpatialY)
1309 {
1310 ✗ std::list<std::pair<XmmReg, XmmReg>> stack;
1311 ✗ std::list<XmmReg> stack1;
1312
1313 ✗ const int pixels_per_cycle = processSingle ? 4 : 8;
1314
1315 ✗ const bool maskIt = (maskUnused && ((planewidth & 3) != 0));
1316
1317 ✗ for (const auto &iter : ops) {
1318 ✗ if (iter.op == opLoadSpatialX) {
1319 if (processSingle) {
1320 ✗ XmmReg r1;
1321 ✗ movd(r1, dword_ptr[regptrs + sizeof(void *) * (RWPTR_START_OF_XCOUNTER)]);
1322 ✗ shufps(r1, r1, 0);
1323 ✗ cvtdq2ps(r1, r1);
1324 ✗ addps(r1, CPTR(spatialX));
1325 ✗ stack1.push_back(r1);
1326 }
1327 else {
1328 ✗ XmmReg r1, r2;
1329 ✗ movd(r1, dword_ptr[regptrs + sizeof(void *) * (RWPTR_START_OF_XCOUNTER)]);
1330 ✗ shufps(r1, r1, 0);
1331 ✗ cvtdq2ps(r1, r1);
1332 ✗ movaps(r2, r1);
1333 ✗ addps(r1, CPTR(spatialX));
1334 ✗ addps(r2, CPTR(spatialX2));
1335 ✗ stack.push_back(std::make_pair(r1, r2));
1336 }
1337 }
1338 ✗ else if (iter.op == opLoadSpatialY) {
1339 if (processSingle) {
1340 ✗ XmmReg r1;
1341 ✗ movd(r1, SpatialY);
1342 ✗ shufps(r1, r1, 0);
1343 ✗ cvtdq2ps(r1, r1);
1344 ✗ stack1.push_back(r1);
1345 }
1346 else {
1347 ✗ XmmReg r1, r2;
1348 ✗ movd(r1, SpatialY);
1349 ✗ shufps(r1, r1, 0);
1350 ✗ cvtdq2ps(r1, r1);
1351 ✗ movaps(r2, r1);
1352 ✗ stack.push_back(std::make_pair(r1, r2));
1353 }
1354 }
1355 ✗ else if (iter.op == opLoadInternalVar) {
1356 if (processSingle) {
1357 ✗ XmmReg r1;
1358 ✗ movd(r1, dword_ptr[regptrs + sizeof(void *) * (iter.e.ival + RWPTR_START_OF_INTERNAL_VARIABLES)]);
1359 ✗ shufps(r1, r1, 0);
1360 ✗ stack1.push_back(r1);
1361 }
1362 else {
1363 ✗ XmmReg r1, r2;
1364 ✗ movd(r1, dword_ptr[regptrs + sizeof(void *) * (iter.e.ival + RWPTR_START_OF_INTERNAL_VARIABLES)]);
1365 ✗ shufps(r1, r1, 0);
1366 ✗ movaps(r2, r1);
1367 ✗ stack.push_back(std::make_pair(r1, r2));
1368 }
1369 }
1370 ✗ else if (iter.op == opLoadFramePropVar) {
1371 if (processSingle) {
1372 ✗ XmmReg r1;
1373 ✗ movd(r1, dword_ptr[regptrs + sizeof(void*) * (iter.e.ival + RWPTR_START_OF_INTERNAL_FRAMEPROP_VARIABLES)]);
1374 ✗ shufps(r1, r1, 0);
1375 ✗ stack1.push_back(r1);
1376 }
1377 else {
1378 ✗ XmmReg r1, r2;
1379 ✗ movd(r1, dword_ptr[regptrs + sizeof(void*) * (iter.e.ival + RWPTR_START_OF_INTERNAL_FRAMEPROP_VARIABLES)]);
1380 ✗ shufps(r1, r1, 0);
1381 ✗ movaps(r2, r1);
1382 ✗ stack.push_back(std::make_pair(r1, r2));
1383 }
1384 }
1385 ✗ else if (iter.op == opLoadRelSrc8 || iter.op == opLoadRelSrc16 || iter.op == opLoadRelSrcF32) {
1386 // either dx or dy is nonzero
1387 // common part follows for single 4 pixels/cycle and dual 8 pixels/cycle
1388 ✗ Reg newx;
1389 ✗ if (iter.dx != 0) {
1390 ✗ mov(newx, ptr[regptrs + sizeof(void *) * (RWPTR_START_OF_XCOUNTER)]); // original base
1391 ✗ add(newx, iter.dx); // new base
1392 }
1393
1394 ✗ Reg a;
1395 ✗ mov(a, ptr[regptrs + sizeof(void *) * (iter.e.ival + RWPTR_START_OF_INPUTS)]); // current pixel group of current line
1396 // adjust read pointer vertically for nonzero dy, keep 0..height-1 limits
1397 ✗ if (iter.dy < 0) {
1398 // Read from above
1399 ✗ Reg dy, sy;
1400 ✗ mov(sy, SpatialY);
1401 ✗ mov(dy, -iter.dy); // dy = -dy;
1402 ✗ cmp(dy, sy);
1403 ✗ cmovg(dy, sy); // mov if greater: if (dy > SpatialY) dy = SpatialY;
1404 #ifdef JITASM64
1405 ✗ imul(dy, qword_ptr[regptrs + sizeof(void *) * (iter.e.ival + RWPTR_START_OF_STRIDES)]); // dy * stride
1406 #else
1407 imul(dy, dword_ptr[regptrs + sizeof(void *) * (iter.e.ival + RWPTR_START_OF_STRIDES)]); // dy * stride
1408 #endif
1409 ✗ sub(a, dy); // a -= dy * stride
1410 }
1411 ✗ else if (iter.dy > 0) {
1412 // Read from bottom
1413 ✗ Reg dy, sy;
1414 ✗ mov(sy, planeheight - 1);
1415 ✗ sub(sy, SpatialY);
1416 ✗ mov(dy, iter.dy);
1417 ✗ cmp(dy, sy);
1418 ✗ cmovg(dy, sy); // mov if greater: if (dy > (planeheight - 1) - SpatialY) dy = SpatialY;
1419 #ifdef JITASM64
1420 ✗ imul(dy, qword_ptr[regptrs + sizeof(void *) * (iter.e.ival + RWPTR_START_OF_STRIDES)]); // dy * stride
1421 #else
1422 imul(dy, dword_ptr[regptrs + sizeof(void *) * (iter.e.ival + RWPTR_START_OF_STRIDES)]); // dy * stride
1423 #endif
1424 ✗ add(a, dy); // a += dy * stride
1425 }
1426
1427 // dy shift is done already. newx holds xcounter + dx
1428 // Cases:
1429 // ReadBefore: xcounter + dx < 0 (only when dx < 0):
1430 // FullReadBefore: dx <= pixels_per_cycle: clone leftmost pixel to each pixel posision in the group
1431 // PartialReadBefore: pixels_per_cycle < dx < 0: close leftmost pixel to -dx positions
1432 // NormalRead: 0 <= xcounter + dx < planewidth - (pixels_per_cycle - 1) (can read whole pixel group)
1433 // OverRead:
1434 // PartialOverRead when pixel at (planewidth-1) is current read position
1435 // PartialOverRead when pixel at (planewidth-1) is after current read position
1436 // FullOverRead: clone pixel at (planewidth-1) to each pixel posision in the group
1437 if (processSingle) {
1438 // LoadRel8/16/32, single register mode 1x4 pixels
1439
1440 // Use getLabelCount: names should be unique across multiple calls to processingLoop
1441 ✗ std::string LabelNeg = "neg" + getLabelCount();
1442 ✗ std::string LabelOver = "over" + getLabelCount();
1443 ✗ std::string LabelEnd = "end" + getLabelCount();
1444
1445 ✗ XmmReg r1;
1446
1447 ✗ if (iter.dx < 0) { // Optim: read from left is possible only for dx<0 case
1448 ✗ cmp(newx, 0);
1449 ✗ jl(LabelNeg); // newx < 0, read (partially or fully) from before the leftmost pixel
1450 }
1451 ✗ if (iter.dx != 0) { // Optim: read after rightmost pixel is possible only for dx>0 case
1452 // Also check for dx<0, because of possible memory overread
1453 // e.g.: planewidth = 64, dx = -1, 16 bit pixels, 16 bytes/cycle, reading from offsets -1(0), 15, 31, 47, then 63
1454 // When we read 16 bytes from offset 63, we are overaddressing the 64 byte scanline,
1455 // which may give access violation when pointer is in the most bottom line.
1456 ✗ cmp(newx, planewidth - (pixels_per_cycle - 1)); // read (partially of fully) after the rightmost pixel
1457 ✗ jge(LabelOver);
1458 }
1459
1460 // It's safe to read the whole pixel group
1461 int offset;
1462 ✗ if (iter.op == opLoadRelSrc8)
1463 ✗ offset = iter.dx;
1464 ✗ else if (iter.op == opLoadRelSrc16)
1465 ✗ offset = iter.dx * sizeof(uint16_t);
1466 ✗ else if (iter.op == opLoadRelSrcF32)
1467 ✗ offset = iter.dx * sizeof(float);
1468
1469 ✗ if (iter.op == opLoadRelSrc8) {
1470 ✗ movd(r1, dword_ptr[a + offset]); // 4 pixels, 4 bytes
1471 ✗ punpcklbw(r1, zero);
1472 ✗ punpcklwd(r1, zero);
1473 ✗ cvtdq2ps(r1, r1);
1474 }
1475 ✗ else if (iter.op == opLoadRelSrc16) {
1476 ✗ movq(r1, mmword_ptr[a + offset]); // 4 pixels, 8 bytes
1477 ✗ punpcklwd(r1, zero);
1478 ✗ cvtdq2ps(r1, r1);
1479 }
1480 ✗ else if (iter.op == opLoadRelSrcF32) {
1481 ✗ if (iter.dx % 4 == 0)
1482 ✗ movdqa(r1, xmmword_ptr[a + offset]); // 4 pixels, 16 bytes aligned
1483 else
1484 ✗ movdqu(r1, xmmword_ptr[a + offset]); // 4 pixels, 16 bytes unaligned
1485 }
1486 ✗ if (iter.dx != 0) {
1487 ✗ jmp(LabelEnd); // generate jump only when over/negative branches exist
1488 }
1489
1490 ✗ if (iter.dx != 0) {
1491 ✗ L(LabelOver);
1492 ✗ std::string PartialOverread = "PartialOverread" + getLabelCount();
1493 ✗ std::string NoFullOverReadFromNewX = "NoFullOverReadFromNewX" + getLabelCount();
1494 ✗ std::string labelDoOver = "DoOver" + getLabelCount();
1495
1496 ✗ if (iter.dx > 0) { // FullOverRead possible only when dx>0
1497 ✗ cmp(newx, planewidth);
1498 ✗ jl(PartialOverread); // if newx < planewidth ->
1499
1500 // case: FullOver
1501 // even the first pixel to read is beyond the end of line
1502 // We have to clone the rightmost pixel from (planewidth-1)
1503 ✗ if (iter.op == opLoadRelSrc8) {
1504 ✗ sub(a, ptr[regptrs + sizeof(void *) * (RWPTR_START_OF_XCOUNTER)]);
1505 ✗ add(a, planewidth - 1);
1506 // reuse newx
1507 ✗ movzx(newx, byte_ptr[a]);
1508 ✗ movd(r1, newx);
1509 ✗ punpcklbw(r1, zero); // words
1510 ✗ pshufb(r1, CPTR(elShuffleForRight6)); // duplicate last word to all
1511
1512 ✗ punpcklwd(r1, zero);
1513 ✗ cvtdq2ps(r1, r1);
1514 }
1515 ✗ else if (iter.op == opLoadRelSrc16) {
1516 ✗ Reg tmp;
1517 ✗ mov(tmp, ptr[regptrs + sizeof(void *) * (RWPTR_START_OF_XCOUNTER)]);
1518 ✗ shl(tmp, 1); // for 16 bit 2*xcounter
1519 ✗ sub(a, tmp);
1520 ✗ add(a, (planewidth - 1) * 2);
1521 // reuse newx
1522 ✗ movzx(newx, word_ptr[a]);
1523 ✗ movd(r1, newx);
1524 ✗ pshufb(r1, CPTR(elShuffleForRight6)); // duplicate last word to all
1525
1526 ✗ punpcklwd(r1, zero);
1527 ✗ cvtdq2ps(r1, r1);
1528 }
1529 ✗ else if (iter.op == opLoadRelSrcF32) {
1530 ✗ Reg tmp;
1531 ✗ mov(tmp, ptr[regptrs + sizeof(void *) * (RWPTR_START_OF_XCOUNTER)]);
1532 ✗ shl(tmp, 2); // for 32 bit 4*xcounter
1533 ✗ sub(a, tmp);
1534 ✗ add(a, (planewidth - 1) * 4);
1535 ✗ movd(r1, dword_ptr[a]);
1536 ✗ pshufd(r1, r1, (0 << 0) | (0 << 2) | (0 << 4) | (0 << 6));
1537 }
1538 ✗ jmp(LabelEnd);
1539 } // full OverRead path, needed when iter.dx>0
1540
1541 // case: Partial overread
1542 // read the block, then clone the last valid pixel from position (planewidth-1)
1543 // problem: newx is not aligned
1544 ✗ L(PartialOverread);
1545
1546 // planewidth == 14 dx=7 newx = 0+7=7, newx>=planewidth-7, then not newx>=planewidth => newx = 13 => planewidth-newx = 1
1547 // sample 1: newx is in different segment than planewidth-1
1548 // [xcounter]
1549 // [a] [newx] [pw-1]
1550 // V V V
1551 // 0 1 2 3 4 5 6 7 8 9 A B C D e f g h i j k l
1552 // P Q R S T U V w need this
1553 // 0 1 2 3 4 5 6 7 we can read this
1554 // P Q R S T U V w last pixel is beyond
1555 // P Q R S T U V V need this
1556 // sample 2: newx is in the same segment than planewidth-1
1557 // planewidth == 13 dx=3
1558 // [xcounter] [newx]
1559 // [a][newx][pw-1]
1560 // V V V
1561 // 0 1 2 3 4 5 6 7 8 9 A B C d e f g h i j
1562 // P Q x x x x x x need this
1563 // P Q Q Q Q Q Q Q duplicated the last valid pixel
1564 // 0 1 2 3 4 5 6 7 we can read this
1565 // when newx and (planewidth-1) are in different segments then we read from newx
1566 ✗ Reg tmp;
1567 ✗ mov(tmp, newx);
1568 ✗ and_(tmp, ~(pixels_per_cycle - 1));
1569 ✗ cmp(tmp, (planewidth & ~(pixels_per_cycle - 1)));
1570 ✗ jle(NoFullOverReadFromNewX); // jump if (newx and ~0x07) < (planewidth & ~0x07) (in another segment)
1571
1572 // read from current (last) pointer,
1573 ✗ if (iter.op == opLoadRelSrc8 || iter.op == opLoadRelSrc16) {
1574 ✗ if (iter.op == opLoadRelSrc8) {
1575 ✗ movd(r1, dword_ptr[a]); // 4 pixels, 4 bytes
1576 ✗ punpcklbw(r1, zero); // words
1577 }
1578 else { // opLoadRel16
1579 ✗ movq(r1, mmword_ptr[a]); // 8 pixels, 16 bytes, here still aligned
1580 }
1581 /*
1582 psrldq(r1, ((planewidth - 1) & (pixels_per_cycle - 1)) * sizeof(uint16_t)); // Shift right by (planewidth - 1) & 7 to lose low words
1583 sub(newx, planewidth - (pixels_per_cycle - 1)); // find out shuffle pointer -1, ... -7 -> 6 ... 0
1584 shl(newx, 4); // *16 for shuffle table
1585 // LabelDoOver copied here
1586 // reuse a : Reg shuffleTable;
1587 lea(a, CPTR(elShuffleForRight0)); // ptr for word shuffle
1588 pshufb(r1, xmmword_ptr[a + newx]);
1589 */
1590 ✗ punpcklwd(r1, zero);
1591 ✗ cvtdq2ps(r1, r1);
1592 //jmp(LabelEnd);
1593 //jmp(labelDoOver);
1594 }
1595 ✗ else if (iter.op == opLoadRelSrcF32) {
1596 // omg it's complicated
1597 ✗ movdqa(r1, xmmword_ptr[a]); // 4 pixels, 16 bytes, here still aligned
1598 }
1599 ✗ int bytes_to_shift = ((planewidth - 1) & (pixels_per_cycle - 1)) * sizeof(float);
1600 ✗ if (bytes_to_shift > 0) {
1601 ✗ psrldq(r1, bytes_to_shift);
1602 ✗ switch (bytes_to_shift) { // 4, 8, 12
1603 ✗ case 4:
1604 ✗ pshufd(r1, r1, (0 << 0) | (1 << 2) | (2 << 4) | (2 << 6));
1605 ✗ break;
1606 ✗ case 8:
1607 ✗ pshufd(r1, r1, (0 << 0) | (1 << 2) | (1 << 4) | (1 << 6));
1608 ✗ break;
1609 ✗ case 12:
1610 ✗ pshufd(r1, r1, (0 << 0) | (0 << 2) | (0 << 4) | (0 << 6));
1611 ✗ break;
1612 }
1613 }
1614 ✗ jmp(LabelEnd);
1615 //}
1616
1617 ✗ L(NoFullOverReadFromNewX);
1618 // read from newx
1619 ✗ if (iter.op == opLoadRelSrc8) {
1620 ✗ sub(a, ptr[regptrs + sizeof(void *) * (RWPTR_START_OF_XCOUNTER)]); // back x counter bytes to the beginning
1621 ✗ add(a, newx); // new position
1622 ✗ movd(r1, dword_ptr[a]); // 4 pixels, 4 bytes
1623 ✗ punpcklbw(r1, zero); // words
1624 /*
1625 // no shift here, just duplicate appropriate pixel into the high ones
1626 mov(newx, (6 - ((planewidth - iter.dx - 1) & (pixels_per_cycle - 1))) << 4); // find out shuffle pointer
1627 // shuffle by the pattern, table offset in newx, and finalizes
1628 // here r1 contains words
1629 // todo direct load
1630 Reg shuffleTable;
1631 lea(shuffleTable, CPTR(elShuffleForRight4)); // for dual: elShuffleForRight0
1632 add(shuffleTable, newx);
1633 pshufb(r1, xmmword_ptr[shuffleTable]);
1634 */
1635 ✗ punpcklwd(r1, zero);
1636 ✗ cvtdq2ps(r1, r1);
1637 }
1638 ✗ else if (iter.op == opLoadRelSrc16) {
1639 //a = a - 2 * xcounter + 2 * newx;
1640 ✗ sub(newx, ptr[regptrs + sizeof(void *) * (RWPTR_START_OF_XCOUNTER)]);
1641 ✗ shl(newx, 1);
1642 ✗ add(a, newx);
1643 ✗ movq(r1, mmword_ptr[a]); // 4 pixels, 8 bytes
1644 // no shift here, just duplicate appropriate pixel into the high ones
1645 /*
1646 // reuse newx
1647 mov(newx, (6 - ((planewidth - iter.dx - 1) & (pixels_per_cycle - 1))) << 4); // find out shuffle pointer
1648 // ((planewidth - iter.dx - 1) & (pixels_per_cycle - 1)) keep
1649 // 0 ShuffleForRight6 keep word #0, spread it to 1..7
1650 // 1 ShuffleForRight5 keep word #0..1, spread it to 2..7
1651 //
1652 // 6 ShuffleForRight0 keep word #0..6, spread it to 7..7
1653 // continues on labelDoOver
1654 // todo direct load
1655 Reg shuffleTable;
1656 lea(shuffleTable, CPTR(elShuffleForRight4)); // for dual: elShuffleForRight0
1657 add(shuffleTable, newx);
1658 pshufb(r1, xmmword_ptr[shuffleTable]);
1659 */
1660 ✗ punpcklwd(r1, zero);
1661 ✗ cvtdq2ps(r1, r1);
1662 }
1663 ✗ else if (iter.op == opLoadRelSrcF32) {
1664 //a = a - 4 * xcounter + 4 * newx;
1665 ✗ sub(newx, ptr[regptrs + sizeof(void *) * (RWPTR_START_OF_XCOUNTER)]);
1666 ✗ shl(newx, 2);
1667 ✗ add(a, newx);
1668
1669
1670 // no real shift here, just duplicate appropriate pixel into the high ones. But we have two registers
1671 ✗ movdqu(r1, xmmword_ptr[a]); // 4 pixels, 16 bytes, no need for upper 4 pixels
1672 }
1673 // we have 4 floats here in r1, common part
1674 ✗ int what = ((planewidth - iter.dx - 1) & (pixels_per_cycle - 1));
1675 // what
1676 // 0 ShuffleForRight2_32 keep r1 dword #0 , spread it to 1..3, then spread r1.3 to r2 (ShuffleForRight2_32(r2,r1)
1677 // 1 ShuffleForRight1_32 keep r1 dword #0..1, spread it to 2..3, then spread r1.3 to r2
1678 // 2 ShuffleForRight0_32 keep r1 dword #0..2, spread it to 3 , then spread r1.3 to r2
1679 // 3 keep r1 dword #0..3, , then spread r1.3 to r2
1680 // 4 keep r1 dword #0..3, , keep r2 dword #0 , spread it to 1..3
1681 // 5 keep r1 dword #0..3, , keep r2 dword #0..1, spread it to 2..3
1682 // 6 keep r1 dword #0..3, , keep r2 dword #0..2, spread it to 3
1683
1684 ✗ switch (what) {
1685 ✗ case 0:
1686 ✗ pshufd(r1, r1, (0 << 0) | (0 << 2) | (0 << 4) | (0 << 6)); // fill 3 upper dwords of r1 from r1.0
1687 ✗ break;
1688 ✗ case 1:
1689 ✗ pshufd(r1, r1, (0 << 0) | (1 << 2) | (1 << 4) | (1 << 6)); // fill 2 upper dwords of r1 from r1.1
1690 ✗ break;
1691 ✗ case 2:
1692 ✗ pshufd(r1, r1, (0 << 0) | (1 << 2) | (2 << 4) | (2 << 6)); // fill 1 upper dwords of r1 from r1.2
1693 ✗ break;
1694 }
1695 // continues on labelEnd
1696 //}
1697 ✗ if (iter.dx < 0)
1698 ✗ jmp(LabelEnd);
1699 ✗ } // over: iter.dx != 0
1700 ✗ if (iter.dx < 0) {
1701 ✗ L(LabelNeg);
1702 // read from negative area on the left side, read exactly from 0th, then shift
1703 // When reading from negative x coordinates we read exactly from 0th, then shift and duplicate
1704 // For extreme minus offsets we duplicate 0th (leftmost) pixel to each position
1705 // example: dx = -1
1706 // -1 0 1 2 3 4 5 6 7
1707 // A A B C D E F G we need this
1708 // A B C D E F G H read [0]
1709 // 0 A B C D E F G H shift
1710 // A A B C D E F G H duplicate by shuffle
1711 ✗ if (iter.op == opLoadRelSrc8 || iter.op == opLoadRelSrc16) {
1712 ✗ if (iter.op == opLoadRelSrc8) {
1713 ✗ sub(a, ptr[regptrs + sizeof(void *) * (RWPTR_START_OF_XCOUNTER)]); // go back to the beginning
1714 ✗ movd(r1, dword_ptr[a]); // 8 pixels, 8 bytes
1715 ✗ punpcklbw(r1, zero); // bytes to words
1716 }
1717 ✗ else if (iter.op == opLoadRelSrc16) {
1718 // go back to the beginning, in 16 bit, *2
1719 ✗ Reg tmp;
1720 ✗ mov(tmp, ptr[regptrs + sizeof(void *) * (RWPTR_START_OF_XCOUNTER)]);
1721 ✗ shl(tmp, 1); // for 16 bit 2*xcounter
1722 ✗ sub(a, tmp);
1723 ✗ movq(r1, mmword_ptr[a]); // 8 pixels, 16 bytes
1724 }
1725 ✗ punpcklwd(r1, zero);
1726 ✗ cvtdq2ps(r1, r1);
1727 ✗ }
1728 ✗ else if (iter.op == opLoadRelSrcF32) {
1729 ✗ Reg tmp;
1730 ✗ mov(tmp, ptr[regptrs + sizeof(void *) * (RWPTR_START_OF_XCOUNTER)]);
1731 ✗ shl(tmp, 2); // *4
1732 ✗ sub(a, tmp);
1733 ✗ movdqa(r1, xmmword_ptr[a]); // 4 pixels, 16 bytes aligned
1734 }
1735 ✗ std::string PartialReadBefore = "PartialReadBefore" + getLabelCount();
1736
1737 ✗ cmp(newx, -pixels_per_cycle);
1738 ✗ jg(PartialReadBefore);
1739 // FullReadBefore: newx <= -pixels_per_cycle, clone 0th (leftmost) pixel to all
1740 ✗ pshufd(r1, r1, (0 << 0) | (0 << 2) | (0 << 4) | (0 << 6));
1741 ✗ jmp(LabelEnd);
1742
1743 ✗ L(PartialReadBefore);
1744 // -pixels_per_cycle < newx < 0
1745 ✗ int bytes_to_shift = sizeof(float) * min(pixels_per_cycle - 1, (-iter.dx) & (pixels_per_cycle - 1));
1746 // shift bytes
1747 // 4 r1 << 4 shuffle r1.1 to r1.0-0
1748 // 8 r1 << 8 shuffle r1.2 to r1.0-1
1749 // 12 r1 << 12 shuffle r1.3 to r1.0-2
1750 ✗ pslldq(r1, bytes_to_shift); // todo: shift + shuffle = single shuffle
1751
1752 ✗ switch (bytes_to_shift) { // 4, 8, 12
1753 ✗ case 4:
1754 ✗ pshufd(r1, r1, (1 << 0) | (1 << 2) | (2 << 4) | (3 << 6)); // elShuffleForLeft0_32 // shuffle r1.1 to r1.0-0
1755 ✗ break;
1756 ✗ case 8:
1757 ✗ pshufd(r1, r1, (2 << 0) | (2 << 2) | (2 << 4) | (3 << 6)); // elShuffleForLeft1_32 // shuffle r1.2 to r1.0-1
1758 ✗ break;
1759 ✗ case 12:
1760 ✗ pshufd(r1, r1, (3 << 0) | (3 << 2) | (3 << 4) | (3 << 6)); // elShuffleForLeft2_32 // shuffle r1.3 to r1.0-2
1761 ✗ break;
1762 }
1763 ✗ } // negative
1764 ✗ L(LabelEnd);
1765 ✗ stack1.push_back(r1);
1766 ✗ } // end of single Relative Mode
1767 else {
1768 // LoadRel8/16/32, dual register mode 2x4 pixels
1769
1770 // Use getLabelCount: names should be unique across multiple calls to processingLoop
1771 ✗ std::string LabelNeg = "neg" + getLabelCount();
1772 ✗ std::string LabelOver = "over" + getLabelCount();
1773 ✗ std::string LabelEnd = "end" + getLabelCount();
1774
1775 ✗ XmmReg r1, r2;
1776
1777 // damn, when the order of the two comparisons is exchanged, bad code is generated for dx=-1 (expr=x[-1]).
1778 // jitasm cannot guess the proper register for 'a', it uses register 'xcounter' instead
1779 // maybe the jump order has to match the label order?
1780 // good: LabelOver, LabelNeg. Bad: LabelNeg, LabelOver
1781 // But it's not true. Now x[-2] fails for 32 bit clip
1782 ✗ if (iter.dx < 0) { // Optim: read from left is possible only for dx<0 case
1783 ✗ cmp(newx, 0);
1784 ✗ jl(LabelNeg); // newx < 0, read (partially or fully) from before the leftmost pixel
1785 }
1786 ✗ if (iter.dx != 0) {
1787 // Also check for dx<0, because of possible memory overread
1788 // e.g.: planewidth = 64, dx = -1, 16 bit pixels, 16 bytes/cycle, reading from offsets -1(0), 15, 31, 47, then 63
1789 // When we read 16 bytes from offset 63, we are overaddressing the 64 byte scanline,
1790 // which may give access violation when pointer is in the most bottom line.
1791 ✗ cmp(newx, planewidth - (pixels_per_cycle - 1)); // read (partially of fully) after the rightmost pixel
1792 ✗ jge(LabelOver);
1793 }
1794 /*
1795 if (iter.dx < 0) { // Optim: read from left is possible only for dx<0 case
1796 cmp(newx, 0);
1797 jl(LabelNeg); // newx < 0, read (partially or fully) from before the leftmost pixel
1798 }
1799 */
1800
1801 // It's safe to read the whole pixel group
1802 int offset;
1803 ✗ if (iter.op == opLoadRelSrc8)
1804 ✗ offset = iter.dx;
1805 ✗ else if (iter.op == opLoadRelSrc16)
1806 ✗ offset = iter.dx * sizeof(uint16_t);
1807 ✗ else if (iter.op == opLoadRelSrcF32)
1808 ✗ offset = iter.dx * sizeof(float);
1809
1810 ✗ if (iter.op == opLoadRelSrc8) {
1811 ✗ movq(r1, mmword_ptr[a + offset]); // 8 pixels, 8 bytes
1812 ✗ punpcklbw(r1, zero);
1813 ✗ movdqa(r2, r1);
1814 ✗ punpcklwd(r1, zero);
1815 ✗ punpckhwd(r2, zero);
1816 ✗ cvtdq2ps(r1, r1);
1817 ✗ cvtdq2ps(r2, r2);
1818 }
1819 ✗ else if (iter.op == opLoadRelSrc16) {
1820 ✗ if (iter.dx % 8 == 0)
1821 ✗ movdqa(r1, xmmword_ptr[a + offset]); // 8 pixels 16 byte boundary, aligned
1822 else
1823 ✗ movdqu(r1, xmmword_ptr[a + offset]);
1824 ✗ movdqa(r2, r1);
1825 ✗ punpcklwd(r1, zero);
1826 ✗ punpckhwd(r2, zero);
1827 ✗ cvtdq2ps(r1, r1);
1828 ✗ cvtdq2ps(r2, r2);
1829 }
1830 ✗ else if (iter.op == opLoadRelSrcF32) {
1831 ✗ if (iter.dx % 4 == 0) {
1832 ✗ movdqa(r1, xmmword_ptr[a + offset]); // // 4 pixels 16 byte boundary, aligned
1833 ✗ movdqa(r2, xmmword_ptr[a + offset + 16]);
1834 }
1835 else {
1836 ✗ movdqu(r1, xmmword_ptr[a + offset]); // unaligned
1837 ✗ movdqu(r2, xmmword_ptr[a + offset + 16]);
1838 }
1839 }
1840 ✗ if (iter.dx != 0) {
1841 ✗ jmp(LabelEnd); // Optim: generate jump only when over/negative branches exist
1842 }
1843
1844 ✗ if (iter.dx != 0) {
1845 ✗ L(LabelOver);
1846
1847 // x dx newx
1848 // 8 1 8+1=9
1849 // planewidth == 16
1850 // 0 1 2 3 4 5 6 7 8 9 A B C D E F g h i j
1851 // P Q R S T U V x need this
1852 // P Q R S T U V V duplicated the last valid pixel
1853 // 0 1 2 3 4 5 6 7 we can read this
1854 // 1 2 3 4 5 6 7 - Shift right by dx to lose low bytes
1855 // 1 2 3 4 5 6 7 7 have to make this one from it. Only the first planewidth-newx (7) bytes valid
1856 // x dx newx
1857 // 8 3 8+3=11
1858 // planewidth == 15
1859 // 0 1 2 3 4 5 6 7 8 9 A B C D E f g h i j
1860 // P Q R S T x x x need this
1861 // P Q R S S S S S duplicated the last valid pixel
1862 // 0 1 2 3 4 5 6 7 we can read this
1863 // 3 4 5 6 7 - - - Shift right by dx to lose low bytes
1864 // 3 4 5 6 6 6 6 6 have to make this one from it. Only the first planewidth-newx (4) bytes valid
1865 // planewidth == 14
1866 // 0 1 2 3 4 5 6 7 8 9 A B C D e f g h i j
1867 // P Q R x x x x x need this
1868 // P Q R R R R R R duplicated the last valid pixel
1869 // 0 1 2 3 4 5 6 7 we can read this
1870 // 3 4 5 6 7 - - - Shift right by dx to lose low bytes
1871 // 3 4 5 5 5 5 5 5 have to make this one from it. Only the first planewidth-newx (3) bytes valid
1872 // special case: full read from beyond line
1873 // planewidth == 14 dx=6 newx = 8+6=14, newx>=planewidth => newx = 13 => planewidth-newx = 1
1874 // 0 1 2 3 4 5 6 7 8 9 A B C D e f g h i j k l
1875 // ? ? ? ? ? ? ? ? need this, but overread
1876 // ? ? ? ? ? ? ? ? duplicated the last valid pixel
1877 // 0 1 2 3 4 5 6 7 we can read this
1878 // 5 6 7 - - - - - Shift right by not dx but (planewidth-1)&7, it's 5 in this example, to lose low bytes
1879 // 5 5 5 5 5 5 5 5 case(1): duplicate very first
1880 // planewidth == 14 dx=7 newx = 0+7=7, newx>=planewidth-7, then not newx>=planewidth => newx = 13 => planewidth-newx = 1
1881 // 0 1 2 3 4 5 6 7 8 9 A B C D e f g h i j k l
1882 // P Q R S T U V w need this
1883 // P Q R S T U V V duplicated the last valid pixel
1884 // 0 1 2 3 4 5 6 7 we can read this
1885 // 5 6 7 - - - - - Shift right by not dx but (planewidth-1)&7, it's 5 in this example, to lose low bytes
1886 // 5 5 5 5 5 5 5 5 case(1): duplicate very first
1887 // planewidth == 14 dx=7 newx = 8+7=15, newx>=planewidth => newx = 13 => planewidth-newx = 1
1888 // ? ? ? ? ? ? ? ? need this, but overread
1889 // 0 1 2 3 4 5 6 7 we can read this
1890 // 5 - - - - - - - Shift right to have the last pixel
1891 // 5 5 5 5 5 5 5 5 case(1): duplicate very first
1892 // planewidth == 13
1893 // 0 1 2 3 4 5 6 7 8 9 A B C d e f g h i j
1894 // P Q x x x x x x need this
1895 // P Q Q Q Q Q Q Q duplicated the last valid pixel
1896 // 0 1 2 3 4 5 6 7 we can read this
1897 // 3 4 5 6 7 - - - Shift right by dx to lose low bytes
1898 // 3 4 4 4 4 4 4 4 have to make this one from it. Only the first planewidth-newx (2) bytes valid
1899 // planewidth == 12
1900 // 0 1 2 3 4 5 6 7 8 9 A B c d e f g h i j
1901 // P x x x x x x x need this
1902 // P P P P P P P P duplicated the last valid pixel
1903 // 0 1 2 3 4 5 6 7 we can read this
1904 // 3 4 5 6 7 - - - Shift right by dx to lose low bytes
1905 // 3 3 3 3 3 3 3 3 have to make this one from it. Only the first planewidth-newx (1) bytes valid
1906
1907 // duplicate highest, make a shuffle table by planewidth-newx (1..7)
1908 // planewidth - newx newx-pw newx-pw+7 shuffle
1909 // newx-(pw-7)
1910 // 1 -1 6 0->0 0->1 0->2 0->3 0->4 0->5 0->6 0->7 elSuffleForRight6 (lowest to everywhere)
1911 // 2 -2 5 0->0 1->1 1->2 1->3 1->4 1->5 1->6 1->7 elSuffleForRight5 (lowest two remains then second duplicates)
1912 // 3 -3 4 0->0 1->1 2->2 2->3 2->4 2->5 2->6 2->7 elSuffleForRight4 (lowest three remains then third duplicates)
1913 // 4 -4 3 0->0 1->1 2->2 3->3 3->4 3->5 3->6 3->7 elSuffleForRight3
1914 // 5 -5 2 0->0 1->1 2->2 3->3 4->4 4->5 4->6 4->7 elSuffleForRight2
1915 // 6 -6 1 0->0 1->1 2->2 3->3 4->4 5->5 5->6 5->7 elSuffleForRight1
1916 // 7 -7 0 0->0 1->1 2->2 3->3 4->4 5->5 6->6 6->7 elSuffleForRight0 (lowest seven remains then seventh duplicates)
1917 // in extreme case (read all beyond last pixel): newx >= planewidth: ==> case of elSuffleForRight6
1918
1919 // shuffleTable = elShuffleForRight0 + 16*(newx-(planewidth-7))
1920 ✗ std::string PartialOverread = "PartialOverread" + getLabelCount();
1921 ✗ std::string NoFullOverReadFromNewX = "NoFullOverReadFromNewX" + getLabelCount();
1922 ✗ std::string labelDoOver = "DoOver" + getLabelCount();
1923
1924 ✗ if (iter.dx > 0) { // FullOverRead possible only when dx>0
1925 ✗ cmp(newx, planewidth);
1926 ✗ jl(PartialOverread); // if newx < planewidth ->
1927
1928 // case: FullOver
1929 // even the first pixel to read is beyond the end of line
1930 // We have to clone the rightmost pixel from (planewidth-1)
1931 ✗ if (iter.op == opLoadRelSrc8) {
1932 ✗ sub(a, ptr[regptrs + sizeof(void *) * (RWPTR_START_OF_XCOUNTER)]);
1933 ✗ add(a, planewidth - 1);
1934 // reuse newx
1935 ✗ movzx(newx, byte_ptr[a]);
1936 ✗ movd(r1, newx);
1937 ✗ punpcklbw(r1, zero); // words
1938 ✗ pshufb(r1, CPTR(elShuffleForRight6)); // duplicate last word to all
1939
1940 ✗ movdqa(r2, r1);
1941 ✗ punpcklwd(r1, zero);
1942 ✗ punpckhwd(r2, zero);
1943 ✗ cvtdq2ps(r1, r1);
1944 ✗ cvtdq2ps(r2, r2);
1945 }
1946 ✗ else if (iter.op == opLoadRelSrc16) {
1947 ✗ Reg tmp;
1948 ✗ mov(tmp, ptr[regptrs + sizeof(void *) * (RWPTR_START_OF_XCOUNTER)]);
1949 ✗ shl(tmp, 1); // for 16 bit 2*xcounter
1950 ✗ sub(a, tmp);
1951 ✗ add(a, (planewidth - 1) * 2);
1952 // reuse newx
1953 ✗ movzx(newx, word_ptr[a]);
1954 ✗ movd(r1, newx);
1955 ✗ pshufb(r1, CPTR(elShuffleForRight6)); // duplicate last word to all
1956
1957 ✗ movdqa(r2, r1);
1958 ✗ punpcklwd(r1, zero);
1959 ✗ punpckhwd(r2, zero);
1960 ✗ cvtdq2ps(r1, r1);
1961 ✗ cvtdq2ps(r2, r2);
1962 }
1963 ✗ else if (iter.op == opLoadRelSrcF32) {
1964 ✗ Reg tmp;
1965 ✗ mov(tmp, ptr[regptrs + sizeof(void *) * (RWPTR_START_OF_XCOUNTER)]);
1966 ✗ shl(tmp, 2); // for 32 bit 4*xcounter
1967 ✗ sub(a, tmp);
1968 ✗ add(a, (planewidth - 1) * 4);
1969 ✗ movd(r1, dword_ptr[a]);
1970 ✗ pshufd(r1, r1, (0 << 0) | (0 << 2) | (0 << 4) | (0 << 6));
1971 ✗ movdqa(r2, r1);
1972 }
1973 ✗ jmp(LabelEnd);
1974 } // full OverRead path, needed when iter.dx>0
1975
1976 // case: Partial overread
1977 // read the block, then clone the last valid pixel from position (planewidth-1)
1978 // problem: newx is not aligned
1979 ✗ L(PartialOverread);
1980 // planewidth == 14 dx=7 newx = 0+7=7, newx>=planewidth-7, then not newx>=planewidth => newx = 13 => planewidth-newx = 1
1981 // sample 1: newx is in different segment than planewidth-1
1982 // [xcounter]
1983 // [a] [newx] [pw-1]
1984 // V V V
1985 // 0 1 2 3 4 5 6 7 8 9 A B C D e f g h i j k l
1986 // P Q R S T U V w need this
1987 // 0 1 2 3 4 5 6 7 we can read this
1988 // P Q R S T U V w last pixel is beyond
1989 // P Q R S T U V V need this
1990 // sample 2: newx is in the same segment than planewidth-1
1991 // planewidth == 13 dx=3
1992 // [xcounter] [newx]
1993 // [a][newx][pw-1]
1994 // V V V
1995 // 0 1 2 3 4 5 6 7 8 9 A B C d e f g h i j
1996 // P Q x x x x x x need this
1997 // P Q Q Q Q Q Q Q duplicated the last valid pixel
1998 // 0 1 2 3 4 5 6 7 we can read this
1999 // when newx and (planewidth-1) are in different segments then we read from newx
2000 ✗ Reg tmp;
2001 ✗ mov(tmp, newx);
2002 ✗ and_(tmp, ~(pixels_per_cycle - 1));
2003 ✗ cmp(tmp, (planewidth & ~(pixels_per_cycle - 1)));
2004 ✗ jle(NoFullOverReadFromNewX); // jump if (newx and ~0x07) < (planewidth & ~0x07) (in another segment)
2005
2006 // read from current (last) pointer,
2007 ✗ if (iter.op == opLoadRelSrc8 || iter.op == opLoadRelSrc16) {
2008 ✗ if (iter.op == opLoadRelSrc8) {
2009 ✗ movq(r1, mmword_ptr[a]); // 8 pixels, 8 bytes
2010 ✗ punpcklbw(r1, zero); // words
2011 }
2012 else { // opLoadRel16
2013 ✗ movdqa(r1, xmmword_ptr[a]); // 8 pixels, 16 bytes, here still aligned
2014 }
2015 ✗ psrldq(r1, ((planewidth - 1) & (pixels_per_cycle - 1)) * sizeof(uint16_t)); // Shift right by (planewidth - 1) & 7 to lose low words
2016 ✗ sub(newx, planewidth - (pixels_per_cycle - 1)); // find out shuffle pointer -1, ... -7 -> 6 ... 0
2017 ✗ shl(newx, 4); // *16 for shuffle table
2018 // LabelDoOver copied here
2019 // reuse a : Reg shuffleTable;
2020 ✗ lea(a/*shuffleTable*/, CPTR(elShuffleForRight0)); // ptr for word shuffle
2021 //add(a/*shuffleTable*/, newx);
2022 ✗ pshufb(r1, xmmword_ptr[a/*shuffleTable*/ + newx]);
2023
2024 ✗ movdqa(r2, r1);
2025 ✗ punpcklwd(r1, zero);
2026 ✗ punpckhwd(r2, zero);
2027 ✗ cvtdq2ps(r1, r1);
2028 ✗ cvtdq2ps(r2, r2);
2029 ✗ jmp(LabelEnd);
2030 }
2031 ✗ else if (iter.op == opLoadRelSrcF32) {
2032 // omg it's complicated
2033
2034 // palignr memo
2035 // temp1[255:0] ((DEST[127:0] << 128) OR SRC[127:0]) >> (imm8*8);
2036 // DEST[127:0] temp1[127:0]
2037
2038 ✗ int bytes_to_shift = ((planewidth - 1) & (pixels_per_cycle - 1)) * sizeof(float);
2039 ✗ if (bytes_to_shift < 16) {
2040 // src dst
2041 // r2 r1
2042 // 15 14 13.... 0 15 14 13 ... 0
2043 // 15 14 13 1 0 15 14.... 1 palignr(dst, src, 1)
2044 // 15 14 13 ..... 15 palignr(dst, src, 15)
2045 ✗ movdqa(r1, xmmword_ptr[a]); // 4 pixels, 16 bytes, here still aligned
2046 ✗ movdqa(r2, xmmword_ptr[a + 16]); // 4 pixels, 16 bytes
2047 ✗ if (bytes_to_shift > 0) {
2048 ✗ palignr(r1, r2, bytes_to_shift); // shift right dualreg. r1 is ready.
2049 ✗ psrldq(r2, bytes_to_shift); // Shift right upper part
2050 ✗ switch (bytes_to_shift) { // 4, 8, 12
2051 ✗ case 4:
2052 ✗ pshufd(r2, r2, (0 << 0) | (1 << 2) | (2 << 4) | (2 << 6)); // elShuffleForRight0_32
2053 ✗ break;
2054 ✗ case 8:
2055 ✗ pshufd(r2, r2, (0 << 0) | (1 << 2) | (1 << 4) | (1 << 6));
2056 ✗ break;
2057 ✗ case 12:
2058 ✗ pshufd(r2, r2, (0 << 0) | (0 << 2) | (0 << 4) | (0 << 6)); // elShuffleForRight2_32
2059 ✗ break;
2060 }
2061 }
2062 }
2063 ✗ else if (bytes_to_shift == 16) {
2064 // src dst
2065 // r2 r1
2066 // 15 14 13.... 0 15 14 13 ... 0 --> 16 bytes: r1 = r2
2067 ✗ movdqa(r1, xmmword_ptr[a + 16]); // 4 pixels, 16 bytes, no need [a + 0], here still aligned
2068 ✗ pshufd(r2, r1, (3 << 0) | (3 << 2) | (3 << 4) | (3 << 6)); // fill r2 with highest dword of r1
2069 }
2070 else {
2071 // bytes to shift > 16 (20, 24, 28), ignore lower 4 pixels, move and shift and spread from upper 4 pixels
2072 ✗ movdqa(r1, xmmword_ptr[a + 16]); // 4 pixels, 16 bytes, no need [a + 0], here still aligned
2073 ✗ psrldq(r1, bytes_to_shift - 16); // Shift right upper part
2074 ✗ switch (bytes_to_shift) { // 0, 4, 8, 12
2075 ✗ case 20:
2076 ✗ pshufd(r1, r1, (0 << 0) | (1 << 2) | (2 << 4) | (2 << 6)); // elShuffleForRight0_32
2077 ✗ break;
2078 ✗ case 24:
2079 ✗ pshufd(r1, r1, (0 << 0) | (1 << 2) | (1 << 4) | (1 << 6));
2080 ✗ break;
2081 ✗ case 28:
2082 ✗ pshufd(r1, r1, (0 << 0) | (0 << 2) | (0 << 4) | (0 << 6)); // elShuffleForRight2_32
2083 ✗ break;
2084 }
2085 ✗ pshufd(r2, r1, (3 << 0) | (3 << 2) | (3 << 4) | (3 << 6)); // fill r2 with highest dword of r1
2086 }
2087 ✗ jmp(LabelEnd);
2088 }
2089
2090 ✗ L(NoFullOverReadFromNewX);
2091 // read from newx
2092 //a = a - 1,2,4 * xcounter + 1,2,4 * newx;
2093 ✗ sub(newx, ptr[regptrs + sizeof(void *) * (RWPTR_START_OF_XCOUNTER)]);
2094 ✗ if (iter.op == opLoadRelSrc8 || iter.op == opLoadRelSrc16) {
2095 ✗ if (iter.op == opLoadRelSrc8)
2096 {
2097 ✗ add(a, newx); // new position
2098 ✗ movq(r1, mmword_ptr[a]); // 8 pixels, 8 bytes
2099 ✗ punpcklbw(r1, zero); // words
2100 }
2101 else {
2102 ✗ shl(newx, 1); //a = a - 2 * xcounter + 2 * newx;
2103 ✗ add(a, newx);
2104 ✗ movdqu(r1, xmmword_ptr[a]); // 8 pixels, 16 bytes
2105 }
2106 // no shift here, just duplicate appropriate pixel into the high ones
2107 ✗ int what = ((planewidth - iter.dx - 1) & (pixels_per_cycle - 1));
2108 ✗ switch (what) {
2109 ✗ case 0:
2110 ✗ pshufb(r1, CPTR(elShuffleForRight6));
2111 ✗ break;
2112 ✗ case 1:
2113 ✗ pshufb(r1, CPTR(elShuffleForRight5));
2114 ✗ break;
2115 ✗ case 2:
2116 ✗ pshufb(r1, CPTR(elShuffleForRight4));
2117 ✗ break;
2118 ✗ case 3:
2119 ✗ pshufb(r1, CPTR(elShuffleForRight3));
2120 ✗ break;
2121 ✗ case 4:
2122 ✗ pshufb(r1, CPTR(elShuffleForRight2));
2123 ✗ break;
2124 ✗ case 5:
2125 ✗ pshufb(r1, CPTR(elShuffleForRight1));
2126 ✗ break;
2127 ✗ case 6:
2128 ✗ pshufb(r1, CPTR(elShuffleForRight0));
2129 ✗ break;
2130 }
2131 ✗ movdqa(r2, r1);
2132 ✗ punpcklwd(r1, zero);
2133 ✗ punpckhwd(r2, zero);
2134 ✗ cvtdq2ps(r1, r1);
2135 ✗ cvtdq2ps(r2, r2);
2136 //jmp(LabelEnd);
2137 ✗ }
2138 ✗ else if (iter.op == opLoadRelSrcF32) {
2139 ✗ shl(newx, 2); // float: *4
2140 ✗ add(a, newx);
2141
2142 ✗ int what = ((planewidth - iter.dx - 1) & (pixels_per_cycle - 1));
2143 // what
2144 // 0 ShuffleForRight2_32 keep r1 dword #0 , spread it to 1..3, then spread r1.3 to r2 (ShuffleForRight2_32(r2,r1)
2145 // 1 ShuffleForRight1_32 keep r1 dword #0..1, spread it to 2..3, then spread r1.3 to r2
2146 // 2 ShuffleForRight0_32 keep r1 dword #0..2, spread it to 3 , then spread r1.3 to r2
2147 // 3 keep r1 dword #0..3, , then spread r1.3 to r2
2148 // 4 keep r1 dword #0..3, , keep r2 dword #0 , spread it to 1..3
2149 // 5 keep r1 dword #0..3, , keep r2 dword #0..1, spread it to 2..3
2150 // 6 keep r1 dword #0..3, , keep r2 dword #0..2, spread it to 3
2151
2152 // no real shift here, just duplicate appropriate pixel into the high ones. But we have two registers
2153 ✗ if (what <= 3) {
2154 ✗ movdqu(r1, xmmword_ptr[a]); // 4 pixels, 16 bytes, no need for upper 4 pixels
2155 ✗ switch (what) {
2156 ✗ case 0:
2157 ✗ pshufd(r1, r1, (0 << 0) | (0 << 2) | (0 << 4) | (0 << 6)); // elShuffleForRight2_32 // fill 3 upper dwords of r1 from r1.0
2158 ✗ break;
2159 ✗ case 1:
2160 ✗ pshufd(r1, r1, (0 << 0) | (1 << 2) | (1 << 4) | (1 << 6)); // fill 2 upper dwords of r1 from r1.1
2161 ✗ break;
2162 ✗ case 2:
2163 ✗ pshufd(r1, r1, (0 << 0) | (1 << 2) | (2 << 4) | (2 << 6)); // elShuffleForRight0_32 // fill 1 upper dwords of r1 from r1.2
2164 ✗ break;
2165 }
2166 ✗ pshufd(r2, r1, (3 << 0) | (3 << 2) | (3 << 4) | (3 << 6)); // fill all dwords of r2 from r1.3
2167 }
2168 else {
2169 ✗ movdqu(r1, xmmword_ptr[a]); // 4 pixels, 16 bytes, low 4 pixels keep them as is
2170 ✗ movdqu(r2, xmmword_ptr[a + 16]); // 4 pixels, 16 bytes
2171 ✗ switch (what) {
2172 ✗ case 4:
2173 ✗ pshufd(r2, r2, (0 << 0) | (0 << 2) | (0 << 4) | (0 << 6)); // elShuffleForRight2_32 // fill 3 upper dwords of r1 from r2.0
2174 ✗ break;
2175 ✗ case 5:
2176 ✗ pshufd(r2, r2, (0 << 0) | (1 << 2) | (1 << 4) | (1 << 6)); // fill 2 upper dwords of r1 from r2.1
2177 ✗ break;
2178 ✗ case 6:
2179 ✗ pshufd(r2, r2, (0 << 0) | (1 << 2) | (2 << 4) | (2 << 6)); // elShuffleForRight0_32 // fill 1 upper dwords of r1 from r2.2
2180 ✗ break;
2181 }
2182 }
2183 // continues on labelEnd
2184 }
2185 ✗ if (iter.dx < 0)
2186 ✗ jmp(LabelEnd);
2187 ✗ } // over: iter.dx != 0
2188 ✗ if (iter.dx < 0) {
2189 ✗ L(LabelNeg);
2190 // When reading from negative x coordinates we read exactly from 0th, then shift and duplicate
2191 // For extreme minus offsets we duplicate 0th (leftmost) pixel to each position
2192 // example: dx = -1
2193 // -1 0 1 2 3 4 5 6 7
2194 // A A B C D E F G we need this
2195 // A B C D E F G H read [0]
2196 // 0 A B C D E F G H shift
2197 // A A B C D E F G H duplicate by shuffle
2198 ✗ if (iter.op == opLoadRelSrc8 || iter.op == opLoadRelSrc16) {
2199 ✗ if (iter.op == opLoadRelSrc8) {
2200 ✗ sub(a, ptr[regptrs + sizeof(void *) * (RWPTR_START_OF_XCOUNTER)]); // go back to the beginning
2201 ✗ movq(r1, mmword_ptr[a]); // 8 pixels, 8 bytes
2202 ✗ punpcklbw(r1, zero); // bytes to words
2203 }
2204 ✗ else if (iter.op == opLoadRelSrc16) {
2205 // go back to the beginning, in 16 bit, *2
2206 ✗ Reg tmp;
2207 ✗ mov(tmp, ptr[regptrs + sizeof(void *) * (RWPTR_START_OF_XCOUNTER)]);
2208 ✗ shl(tmp, 1); // for 16 bit 2*xcounter
2209 ✗ sub(a, tmp);
2210 ✗ movdqa(r1, xmmword_ptr[a]); // 8 pixels, 16 bytes
2211 }
2212
2213 ✗ std::string PartialReadBefore = "PartialReadBefore" + getLabelCount();
2214 ✗ std::string Finalize = "Finalize" + getLabelCount();
2215 ✗ cmp(newx, -pixels_per_cycle); // pixels_per_cycle words
2216 ✗ jg(PartialReadBefore);
2217 // FullReadBefore: newx <= -pixels_per_cycle, clone 0th (leftmost) pixel to all
2218 ✗ pshufb(r1, CPTR(elShuffleForRight6)); // lowest word to all
2219 ✗ jmp(Finalize);
2220 ✗ L(PartialReadBefore);
2221 // -pixels_per_cycle < newx < 0
2222 ✗ int toShift = min(pixels_per_cycle - 1, (-iter.dx) & (pixels_per_cycle - 1));
2223 ✗ pslldq(r1, toShift * 2); // shift in word domain
2224 ✗ switch (toShift) {
2225 ✗ case 1: pshufb(r1, CPTR(elShuffleForLeft0)); break;
2226 ✗ case 2: pshufb(r1, CPTR(elShuffleForLeft1)); break;
2227 ✗ case 3: pshufb(r1, CPTR(elShuffleForLeft2)); break;
2228 ✗ case 4: pshufb(r1, CPTR(elShuffleForLeft3)); break;
2229 ✗ case 5: pshufb(r1, CPTR(elShuffleForLeft4)); break;
2230 ✗ case 6: pshufb(r1, CPTR(elShuffleForLeft5)); break;
2231 ✗ case 7: pshufb(r1, CPTR(elShuffleForLeft6)); break;
2232 }
2233 ✗ L(Finalize);
2234
2235 ✗ movdqa(r2, r1);
2236 ✗ punpcklwd(r1, zero);
2237 ✗ punpckhwd(r2, zero);
2238 ✗ cvtdq2ps(r1, r1);
2239 ✗ cvtdq2ps(r2, r2);
2240 ✗ }
2241 ✗ else if (iter.op == opLoadRelSrcF32) {
2242 // negative
2243 // go back to the beginning, in 16 bit, *2
2244 ✗ Reg tmp;
2245 ✗ mov(tmp, ptr[regptrs + sizeof(void *) * (RWPTR_START_OF_XCOUNTER)]);
2246 ✗ shl(tmp, 2); // go back to the beginning, in 32 bit, *4
2247 ✗ sub(a, tmp);
2248
2249 ✗ std::string PartialReadBefore = "PartialReadBefore" + getLabelCount();
2250
2251 ✗ cmp(newx, -pixels_per_cycle);
2252 ✗ jg(PartialReadBefore);
2253 // FullReadBefore: newx <= -pixels_per_cycle, clone 0th (leftmost) pixel to all
2254 ✗ movdqa(r1, xmmword_ptr[a]);
2255 ✗ pshufd(r1, r1, (0 << 0) | (0 << 2) | (0 << 4) | (0 << 6));
2256 ✗ movdqa(r2, r1);
2257 ✗ jmp(LabelEnd);
2258
2259 ✗ L(PartialReadBefore);
2260 // -pixels_per_cycle < newx < 0
2261 ✗ int bytes_to_shift = sizeof(float) * min(pixels_per_cycle - 1, (-iter.dx) & (pixels_per_cycle - 1));
2262 // shift bytes
2263 // 4 r2r1 << 4 shuffle r1.1 to r1.0-0
2264 // 8 r2r1 << 8 shuffle r1.2 to r1.0-1
2265 // 12 r2r1 << 12 shuffle r1.3 to r1.0-2
2266 // 16 r2r1 << 16 -> r2 = r1, , shuffle r2.0 to all r1
2267 // 20 r2r1 << 20 -> r2 = r1, r2 << (20-4), shuffle r2.1 to r2.0-0, shuffle r2.0 to all r1
2268 // 24 r2r1 << 24 -> r2 = r1, r2 << (24-4), shuffle r2.2 to r2.0-1, shuffle r2.0 to all r1
2269 // 28 r2r1 << 28 -> r2 = r1, r2 << (28-4), shuffle r2.3 to r2.0-2, shuffle r2.0 to all r1
2270 ✗ if (bytes_to_shift < 16) {
2271 ✗ movdqa(r1, xmmword_ptr[a]); // 4 pixels, 16 bytes
2272 ✗ movdqa(r2, xmmword_ptr[a + 16]); // 4 pixels, 16 bytes
2273 // 4 floats
2274 // r2 r1
2275 // H3 H2 H1 H0 L3 L2 L1 L0 << 1*4 byte
2276 // H2 H1 H0 L3 L2 L1 L0 00
2277 // H2 H1 H0 00 or
2278 // 00 00 00 L3 L2 L1 L0 00
2279 ✗ psrldq(r1, 16 - bytes_to_shift);
2280 ✗ pslldq(r2, bytes_to_shift);
2281 ✗ por(r2, r1);
2282 ✗ movdqa(r1, xmmword_ptr[a]); // load again
2283 ✗ pslldq(r1, bytes_to_shift); // todo: shift + shuffle = single shuffle
2284
2285 ✗ switch (bytes_to_shift) { // 4, 8, 12
2286 ✗ case 4:
2287 ✗ pshufd(r1, r1, (1 << 0) | (1 << 2) | (2 << 4) | (3 << 6)); // elShuffleForLeft0_32 // shuffle r1.1 to r1.0-0
2288 ✗ break;
2289 ✗ case 8:
2290 ✗ pshufd(r1, r1, (2 << 0) | (2 << 2) | (2 << 4) | (3 << 6)); // elShuffleForLeft1_32 // shuffle r1.2 to r1.0-1
2291 ✗ break;
2292 ✗ case 12:
2293 ✗ pshufd(r1, r1, (3 << 0) | (3 << 2) | (3 << 4) | (3 << 6)); // elShuffleForLeft2_32 // shuffle r1.3 to r1.0-2
2294 ✗ break;
2295 }
2296 }
2297 else {
2298 // toShift >= 16
2299 //movdqa(r1, xmmword_ptr[a]); // no need for 15..31
2300 ✗ movdqa(r2, xmmword_ptr[a]); // 4 pixels, 16 bytes
2301 ✗ if (bytes_to_shift > 16)
2302 ✗ pslldq(r2, bytes_to_shift - 16);
2303
2304 ✗ switch (bytes_to_shift) { // 20, 24, 28
2305 ✗ case 20:
2306 ✗ pshufd(r2, r2, (1 << 0) | (1 << 2) | (2 << 4) | (3 << 6)); // elShuffleForLeft0_32 // shuffle r2.1 to r2.0-0
2307 ✗ break;
2308 ✗ case 24:
2309 ✗ pshufd(r2, r2, (2 << 0) | (2 << 2) | (2 << 4) | (3 << 6)); // elShuffleForLeft1_32 // shuffle r2.2 to r2.0-1
2310 ✗ break;
2311 ✗ case 28:
2312 ✗ pshufd(r2, r2, (3 << 0) | (3 << 2) | (3 << 4) | (3 << 6)); // elShuffleForLeft2_32 // shuffle r2.3 to r2.0-2
2313 ✗ break;
2314 }
2315 ✗ pshufd(r1, r2, (0 << 0) | (0 << 2) | (0 << 4) | (0 << 6)); // shuffle r2.0 to all r1
2316 }
2317 ✗ }
2318 }
2319 ✗ L(LabelEnd);
2320 ✗ stack.push_back(std::make_pair(r1, r2));
2321 ✗ }
2322 ✗ } // oploadRel8/16/32
2323 ✗ else if (iter.op == opLoadSrc8) {
2324 if (processSingle) {
2325 ✗ XmmReg r1;
2326 ✗ Reg a;
2327 ✗ mov(a, ptr[regptrs + sizeof(void *) * (iter.e.ival + RWPTR_START_OF_INPUTS)]);
2328 ✗ movd(r1, dword_ptr[a]); // 4 pixels, 4 bytes
2329 ✗ punpcklbw(r1, zero);
2330 ✗ punpcklwd(r1, zero);
2331 ✗ cvtdq2ps(r1, r1);
2332 ✗ if (maskIt)
2333 ✗ doMask(r1, constptr, planewidth);
2334 ✗ stack1.push_back(r1);
2335 }
2336 else {
2337 ✗ XmmReg r1, r2;
2338 ✗ Reg a;
2339 ✗ mov(a, ptr[regptrs + sizeof(void *) * (iter.e.ival + RWPTR_START_OF_INPUTS)]);
2340 ✗ movq(r1, mmword_ptr[a]);
2341 ✗ punpcklbw(r1, zero);
2342 ✗ movdqa(r2, r1);
2343 ✗ punpcklwd(r1, zero);
2344 ✗ punpckhwd(r2, zero);
2345 ✗ cvtdq2ps(r1, r1);
2346 ✗ cvtdq2ps(r2, r2);
2347 ✗ if (maskIt)
2348 ✗ doMask(r2, constptr, planewidth);
2349 ✗ stack.push_back(std::make_pair(r1, r2));
2350 }
2351 }
2352 ✗ else if (iter.op == opLoadSrc16) {
2353 if (processSingle) {
2354 ✗ XmmReg r1;
2355 ✗ Reg a;
2356 ✗ mov(a, ptr[regptrs + sizeof(void *) * (iter.e.ival + RWPTR_START_OF_INPUTS)]);
2357 ✗ movq(r1, mmword_ptr[a]); // 4 pixels, 8 bytes
2358 ✗ punpcklwd(r1, zero);
2359 ✗ cvtdq2ps(r1, r1);
2360 ✗ if (maskIt)
2361 ✗ doMask(r1, constptr, planewidth);
2362 ✗ stack1.push_back(r1);
2363 }
2364 else {
2365 ✗ XmmReg r1, r2;
2366 ✗ Reg a;
2367 ✗ mov(a, ptr[regptrs + sizeof(void *) * (iter.e.ival + RWPTR_START_OF_INPUTS)]);
2368 ✗ movdqa(r1, xmmword_ptr[a]);
2369 ✗ movdqa(r2, r1);
2370 ✗ punpcklwd(r1, zero);
2371 ✗ punpckhwd(r2, zero);
2372 ✗ cvtdq2ps(r1, r1);
2373 ✗ cvtdq2ps(r2, r2);
2374 ✗ if (maskIt)
2375 ✗ doMask(r2, constptr, planewidth);
2376 ✗ stack.push_back(std::make_pair(r1, r2));
2377 }
2378 }
2379 ✗ else if (iter.op == opLoadSrcF32) {
2380 if (processSingle) {
2381 ✗ XmmReg r1;
2382 ✗ Reg a;
2383 ✗ mov(a, ptr[regptrs + sizeof(void *) * (iter.e.ival + RWPTR_START_OF_INPUTS)]);
2384 ✗ movdqa(r1, xmmword_ptr[a]);
2385 ✗ if (maskIt)
2386 ✗ doMask(r1, constptr, planewidth);
2387 ✗ stack1.push_back(r1);
2388 }
2389 else {
2390 ✗ XmmReg r1, r2;
2391 ✗ Reg a;
2392 ✗ mov(a, ptr[regptrs + sizeof(void *) * (iter.e.ival + RWPTR_START_OF_INPUTS)]);
2393 ✗ movdqa(r1, xmmword_ptr[a]);
2394 ✗ movdqa(r2, xmmword_ptr[a + 16]);
2395 ✗ if (maskIt)
2396 ✗ doMask(r2, constptr, planewidth);
2397 ✗ stack.push_back(std::make_pair(r1, r2));
2398 }
2399 }
2400 ✗ else if (iter.op == opLoadSrcF16) { // not supported in avs+
2401 if (processSingle) {
2402 ✗ XmmReg r1;
2403 ✗ Reg a;
2404 ✗ mov(a, ptr[regptrs + sizeof(void *) * (iter.e.ival + RWPTR_START_OF_INPUTS)]);
2405 ✗ vcvtph2ps(r1, qword_ptr[a]);
2406 ✗ if (maskIt)
2407 ✗ doMask(r1, constptr, planewidth);
2408 ✗ stack1.push_back(r1);
2409 }
2410 else {
2411 ✗ XmmReg r1, r2;
2412 ✗ Reg a;
2413 ✗ mov(a, ptr[regptrs + sizeof(void *) * (iter.e.ival + RWPTR_START_OF_INPUTS)]);
2414 ✗ vcvtph2ps(r1, qword_ptr[a]);
2415 ✗ vcvtph2ps(r2, qword_ptr[a + 8]);
2416 ✗ if (maskIt)
2417 ✗ doMask(r2, constptr, planewidth);
2418 ✗ stack.push_back(std::make_pair(r1, r2));
2419 }
2420 }
2421 ✗ else if (iter.op == opLoadVar) {
2422 if (processSingle) {
2423 ✗ XmmReg r1;
2424 // 16 bytes/variable
2425 ✗ int offset = sizeof(void *) * RWPTR_START_OF_USERVARIABLES + 16 * iter.e.ival;
2426 ✗ movdqa(r1, xmmword_ptr[regptrs + offset]);
2427 ✗ if (maskIt)
2428 ✗ doMask(r1, constptr, planewidth);
2429 ✗ stack1.push_back(r1);
2430 }
2431 else {
2432 ✗ XmmReg r1, r2;
2433 // 32 bytes/variable
2434 ✗ int offset = sizeof(void *) * RWPTR_START_OF_USERVARIABLES + 32 * iter.e.ival;
2435 ✗ movdqa(r1, xmmword_ptr[regptrs + offset]);
2436 ✗ movdqa(r2, xmmword_ptr[regptrs + offset + 16]);
2437 ✗ if (maskIt)
2438 ✗ doMask(r2, constptr, planewidth);
2439 ✗ stack.push_back(std::make_pair(r1, r2));
2440 }
2441 }
2442 ✗ else if (iter.op == opLoadConst) {
2443 if (processSingle) {
2444 ✗ XmmReg r1;
2445 ✗ Reg a;
2446 ✗ mov(a, iter.e.ival);
2447 ✗ movd(r1, a);
2448 ✗ shufps(r1, r1, 0);
2449 ✗ if (maskIt)
2450 ✗ doMask(r1, constptr, planewidth);
2451 ✗ stack1.push_back(r1);
2452 }
2453 else {
2454 ✗ XmmReg r1, r2;
2455 ✗ Reg a;
2456 ✗ mov(a, iter.e.ival);
2457 ✗ movd(r1, a);
2458 ✗ shufps(r1, r1, 0);
2459 ✗ movaps(r2, r1);
2460 ✗ if (maskIt)
2461 ✗ doMask(r2, constptr, planewidth);
2462 ✗ stack.push_back(std::make_pair(r1, r2));
2463 }
2464 }
2465 ✗ else if (iter.op == opDup) {
2466 if (processSingle) {
2467 ✗ auto p = std::next(stack1.rbegin(), iter.e.ival);
2468 ✗ XmmReg r1;
2469 ✗ movaps(r1, *p);
2470 ✗ stack1.push_back(r1);
2471 }
2472 else {
2473 ✗ auto p = std::next(stack.rbegin(), iter.e.ival);
2474 ✗ XmmReg r1, r2;
2475 ✗ movaps(r1, p->first);
2476 ✗ movaps(r2, p->second);
2477 ✗ stack.push_back(std::make_pair(r1, r2));
2478 }
2479 }
2480 ✗ else if (iter.op == opSwap) {
2481 if (processSingle) {
2482 ✗ std::swap(stack1.back(), *std::next(stack1.rbegin(), iter.e.ival));
2483 }
2484 else {
2485 ✗ std::swap(stack.back(), *std::next(stack.rbegin(), iter.e.ival));
2486 }
2487 }
2488 ✗ else if (iter.op == opAdd) {
2489 if (processSingle) {
2490 ✗ TwoArgOp_Single(addps)
2491 }
2492 else {
2493 ✗ TwoArgOp(addps)
2494 }
2495 }
2496 ✗ else if (iter.op == opSub) {
2497 if (processSingle) {
2498 ✗ TwoArgOp_Single(subps)
2499 }
2500 else {
2501 ✗ TwoArgOp(subps)
2502 }
2503 }
2504 ✗ else if (iter.op == opMul) {
2505 if (processSingle) {
2506 ✗ TwoArgOp_Single(mulps)
2507 }
2508 else {
2509 ✗ TwoArgOp(mulps)
2510 }
2511 }
2512 ✗ else if (iter.op == opDiv) {
2513 if (processSingle) {
2514 ✗ TwoArgOp_Single(divps)
2515 }
2516 else {
2517 ✗ TwoArgOp(divps)
2518 }
2519 }
2520 ✗ else if (iter.op == opFmod) {
2521 if (processSingle) {
2522 ✗ auto t1 = stack1.back(); // despite the Intel compiler warnings, this is intentionally a copy and not a reference (valid for all such messages as well)
2523 ✗ stack1.pop_back();
2524 ✗ auto &t2 = stack1.back();
2525 ✗ FMOD_PS(t2, t1)
2526 }
2527 else {
2528 ✗ auto t1 = stack.back();
2529 ✗ stack.pop_back();
2530 ✗ auto &t2 = stack.back();
2531 ✗ FMOD_PS(t2.first, t1.first)
2532 ✗ FMOD_PS(t2.second, t1.second)
2533 }
2534 }
2535 ✗ else if (iter.op == opMax) {
2536 if (processSingle) {
2537 ✗ TwoArgOp_Single(maxps)
2538 }
2539 else {
2540 ✗ TwoArgOp(maxps)
2541 }
2542 }
2543 ✗ else if (iter.op == opMin) {
2544 if (processSingle) {
2545 ✗ TwoArgOp_Single(minps)
2546 }
2547 else {
2548 ✗ TwoArgOp(minps)
2549 }
2550 }
2551 ✗ else if (iter.op == opSqrt) {
2552 if (processSingle) {
2553 ✗ auto &t1 = stack1.back();
2554 ✗ maxps(t1, zero);
2555 ✗ sqrtps(t1, t1);
2556 }
2557 else {
2558 ✗ auto &t1 = stack.back();
2559 ✗ maxps(t1.first, zero);
2560 ✗ maxps(t1.second, zero);
2561 ✗ sqrtps(t1.first, t1.first);
2562 ✗ sqrtps(t1.second, t1.second);
2563 }
2564 }
2565 // Integer store operations: Why sometimes C version differs from SSE: convert to int from .5 intermediates.
2566 // Simd version of float -> int32 (cvtps2dq) is using the SSE rounding mode "round to nearest"
2567 // C version is using: (uint8_t)(f + 0.5f) which turnes into cvttps, typecast to int uses trunc
2568 // Even for positive numbers they are not the same, when converting occurs exactly from the halfway
2569 // SSE is using Banker's rounding, which rounds to the nearest _even_ integer value. https://en.wikipedia.org/wiki/IEEE_754#Roundings_to_nearest
2570 // C SSE
2571 // 0.5 1 0
2572 // 1.5 2 2
2573 // 2.5 3 2
2574 // 3.5 4 4
2575 // 3.7.1test27: no more banker's rounding. Using cvttps and +0.5f
2576 ✗ else if (iter.op == opStore8) {
2577 if (processSingle) {
2578 ✗ auto t1 = stack1.back();
2579 ✗ stack1.pop_back();
2580 ✗ XmmReg r1;
2581 ✗ Reg a;
2582 ✗ addps(t1, CPTR(elfloat_half)); // rounder for truncate! no banker's rounding
2583 ✗ maxps(t1, zero);
2584 ✗ minps(t1, CPTR(elstore8));
2585 ✗ mov(a, ptr[regptrs]);
2586 ✗ cvttps2dq(t1, t1); // 00 w3 00 w2 00 w1 00 w0 -- min/max clamp ensures that high words are zero
2587 ✗ packssdw(t1, zero); // _mm_packs_epi32: w7 w6 w5 w4 w3 w2 w1 w0
2588 ✗ packuswb(t1, zero); // _mm_packus_epi16: 0 0 0 0 0 0 0 0 b7 b6 b5 b4 b3 b2 b1 b0
2589 ✗ movd(dword_ptr[a], t1);
2590 }
2591 else {
2592 ✗ auto t1 = stack.back();
2593 ✗ stack.pop_back();
2594 ✗ XmmReg r1, r2;
2595 ✗ Reg a;
2596 ✗ addps(t1.first, CPTR(elfloat_half)); // rounder for truncate! no banker's rounding
2597 ✗ maxps(t1.first, zero);
2598 ✗ addps(t1.second, CPTR(elfloat_half)); // rounder for truncate! no banker's rounding
2599 ✗ maxps(t1.second, zero);
2600 ✗ minps(t1.first, CPTR(elstore8));
2601 ✗ minps(t1.second, CPTR(elstore8));
2602 ✗ mov(a, ptr[regptrs]);
2603 ✗ cvttps2dq(t1.first, t1.first); // 00 w3 00 w2 00 w1 00 w0 -- min/max clamp ensures that high words are zero
2604 ✗ cvttps2dq(t1.second, t1.second); // 00 w7 00 w6 00 w5 00 w4
2605 // t1.second is the lo
2606 ✗ packssdw(t1.first, t1.second); // _mm_packs_epi32: w7 w6 w5 w4 w3 w2 w1 w0
2607 ✗ packuswb(t1.first, zero); // _mm_packus_epi16: 0 0 0 0 0 0 0 0 b7 b6 b5 b4 b3 b2 b1 b0
2608 ✗ movq(mmword_ptr[a], t1.first);
2609 }
2610 }
2611 ✗ else if (iter.op == opStore10 // avs+
2612 ✗ || iter.op == opStore12 // avs+
2613 ✗ || iter.op == opStore14 // avs+
2614 ✗ || iter.op == opStore16
2615 ) {
2616 if (processSingle) {
2617 ✗ auto t1 = stack1.back();
2618 ✗ stack1.pop_back();
2619 ✗ XmmReg r1;
2620 ✗ Reg a;
2621 ✗ addps(t1, CPTR(elfloat_half)); // rounder for truncate! no banker's rounding
2622 ✗ maxps(t1, zero);
2623 ✗ switch (iter.op) {
2624 ✗ case opStore10:
2625 ✗ minps(t1, CPTR(elstore10));
2626 ✗ break;
2627 ✗ case opStore12:
2628 ✗ minps(t1, CPTR(elstore12));
2629 ✗ break;
2630 ✗ case opStore14:
2631 ✗ minps(t1, CPTR(elstore14));
2632 ✗ break;
2633 ✗ case opStore16:
2634 ✗ minps(t1, CPTR(elstore16));
2635 ✗ break;
2636 }
2637 ✗ mov(a, ptr[regptrs]);
2638 ✗ cvttps2dq(t1, t1); // no cvtps, but cvttps
2639 // new
2640 ✗ switch (iter.op) {
2641 ✗ case opStore10:
2642 case opStore12:
2643 case opStore14:
2644 ✗ packssdw(t1, zero); // _mm_packs_epi32: w7 w6 w5 w4 w3 w2 w1 w0
2645 ✗ break;
2646 ✗ case opStore16:
2647 ✗ if (cpuFlags & CPUF_SSE4_1) {
2648 ✗ packusdw(t1, zero); // _mm_packus_epi32: w7 w6 w5 w4 w3 w2 w1 w0
2649 }
2650 else {
2651 // old, sse2
2652 ✗ movdqa(r1, t1); // 00 w3 00 w2 00 w1 00 w0 -- min/max clamp ensures that high words are zero
2653 ✗ psrldq(t1, 6);
2654 ✗ por(t1, r1);
2655 ✗ pshuflw(t1, t1, 0b11011000);
2656 ✗ punpcklqdq(t1, zero);
2657 }
2658 ✗ break;
2659 }
2660 ✗ movq(mmword_ptr[a], t1);
2661 }
2662 else {
2663 ✗ auto t1 = stack.back();
2664 ✗ stack.pop_back();
2665 ✗ XmmReg r1, r2;
2666 ✗ Reg a;
2667 ✗ addps(t1.first, CPTR(elfloat_half)); // rounder for truncate! no banker's rounding
2668 ✗ maxps(t1.first, zero);
2669 ✗ addps(t1.second, CPTR(elfloat_half)); // rounder for truncate! no banker's rounding
2670 ✗ maxps(t1.second, zero);
2671 ✗ switch (iter.op) {
2672 ✗ case opStore10:
2673 ✗ minps(t1.first, CPTR(elstore10));
2674 ✗ minps(t1.second, CPTR(elstore10));
2675 ✗ break;
2676 ✗ case opStore12:
2677 ✗ minps(t1.first, CPTR(elstore12));
2678 ✗ minps(t1.second, CPTR(elstore12));
2679 ✗ break;
2680 ✗ case opStore14:
2681 ✗ minps(t1.first, CPTR(elstore14));
2682 ✗ minps(t1.second, CPTR(elstore14));
2683 ✗ break;
2684 ✗ case opStore16:
2685 ✗ minps(t1.first, CPTR(elstore16));
2686 ✗ minps(t1.second, CPTR(elstore16));
2687 ✗ break;
2688 }
2689 ✗ mov(a, ptr[regptrs]);
2690 ✗ cvttps2dq(t1.first, t1.first); // no bankers rounding
2691 ✗ cvttps2dq(t1.second, t1.second);
2692 // new
2693 ✗ switch (iter.op) {
2694 ✗ case opStore10:
2695 case opStore12:
2696 case opStore14:
2697 ✗ packssdw(t1.first, t1.second); // _mm_packs_epi32: w7 w6 w5 w4 w3 w2 w1 w0
2698 ✗ break;
2699 ✗ case opStore16:
2700 ✗ if (cpuFlags & CPUF_SSE4_1) {
2701 ✗ packusdw(t1.first, t1.second); // _mm_packus_epi32: w7 w6 w5 w4 w3 w2 w1 w0
2702 }
2703 else {
2704 // old, sse2
2705 ✗ movdqa(r1, t1.first); // 00 w3 00 w2 00 w1 00 w0 -- min/max clamp ensures that high words are zero
2706 ✗ movdqa(r2, t1.second); // 00 w7 00 w6 00 w5 00 w4
2707 ✗ psrldq(t1.first, 6);
2708 ✗ psrldq(t1.second, 6);
2709 ✗ por(t1.first, r1);
2710 ✗ por(t1.second, r2);
2711 ✗ pshuflw(t1.first, t1.first, 0b11011000);
2712 ✗ pshuflw(t1.second, t1.second, 0b11011000);
2713 ✗ punpcklqdq(t1.first, t1.second);
2714 }
2715 ✗ break;
2716 }
2717 ✗ movdqa(xmmword_ptr[a], t1.first);
2718 }
2719 ✗ }
2720 ✗ else if (iter.op == opStoreF32) {
2721 if (processSingle) {
2722 ✗ auto t1 = stack1.back();
2723 ✗ stack1.pop_back();
2724 ✗ Reg a;
2725 ✗ mov(a, ptr[regptrs]);
2726 ✗ movaps(xmmword_ptr[a], t1);
2727 }
2728 else {
2729 ✗ auto t1 = stack.back();
2730 ✗ stack.pop_back();
2731 ✗ Reg a;
2732 ✗ mov(a, ptr[regptrs]);
2733 ✗ movaps(xmmword_ptr[a], t1.first);
2734 ✗ movaps(xmmword_ptr[a + 16], t1.second);
2735 }
2736 }
2737 ✗ else if (iter.op == opStoreF16) { // not supported in avs+
2738 if (processSingle) {
2739 ✗ auto t1 = stack1.back();
2740 ✗ stack1.pop_back();
2741 ✗ Reg a;
2742 ✗ mov(a, ptr[regptrs]);
2743 ✗ vcvtps2ph(qword_ptr[a], t1, 0);
2744 }
2745 else {
2746 ✗ auto t1 = stack.back();
2747 ✗ stack.pop_back();
2748 ✗ Reg a;
2749 ✗ mov(a, ptr[regptrs]);
2750 ✗ vcvtps2ph(qword_ptr[a], t1.first, 0);
2751 ✗ vcvtps2ph(qword_ptr[a + 8], t1.second, 0);
2752 }
2753 }
2754 ✗ else if (iter.op == opStoreVar || iter.op == opStoreVarAndDrop1) {
2755 if (processSingle) {
2756 ✗ auto t1 = stack1.back();
2757 // 16 bytes/variable
2758 ✗ int offset = sizeof(void *) * RWPTR_START_OF_USERVARIABLES + 16 * iter.e.ival;
2759 ✗ movaps(xmmword_ptr[regptrs + offset], t1);
2760 ✗ if (iter.op == opStoreVarAndDrop1)
2761 ✗ stack1.pop_back();
2762 }
2763 else {
2764 ✗ auto t1 = stack.back();
2765 // 32 byte/variable
2766 ✗ int offset = sizeof(void *) * RWPTR_START_OF_USERVARIABLES + 32 * iter.e.ival;
2767 ✗ movaps(xmmword_ptr[regptrs + offset], t1.first);
2768 ✗ movaps(xmmword_ptr[regptrs + offset + 16], t1.second);
2769 ✗ if (iter.op == opStoreVarAndDrop1)
2770 ✗ stack.pop_back();
2771 }
2772 ✗ }
2773 ✗ else if (iter.op == opAbs) {
2774 if (processSingle) {
2775 ✗ auto &t1 = stack1.back();
2776 ✗ andps(t1, CPTR(elabsmask));
2777 }
2778 else {
2779 ✗ auto &t1 = stack.back();
2780 ✗ andps(t1.first, CPTR(elabsmask));
2781 ✗ andps(t1.second, CPTR(elabsmask));
2782 }
2783 }
2784 ✗ else if (iter.op == opSgn) {
2785 // 1, 0, -1
2786 /*
2787 __m128 sgn(__m128 value) {
2788 const __m128 zero = _mm_set_ps1 (0.0f);
2789 __m128 p = _mm_and_ps(_mm_cmpgt_ps(value, zero), _mm_set_ps1(1.0f));
2790 __m128 n = _mm_and_ps(_mm_cmplt_ps(value, zero), _mm_set_ps1(-1.0f));
2791 return _mm_or_ps(p, n);
2792 }
2793 */
2794 if (processSingle) {
2795 ✗ auto &t1 = stack1.back();
2796 ✗ XmmReg r1, r2;
2797 ✗ xorps(r2, r2);
2798 ✗ movaps(r1, t1);
2799 ✗ movaps(t1, r2);
2800 ✗ cmpltps(t1, r1);
2801 ✗ cmpltps(r1, r2);
2802 ✗ andps(t1, CPTR(elfloat_one));
2803 ✗ andps(r1, CPTR(elfloat_minusone));
2804 ✗ orps(t1, r1);
2805 }
2806 else {
2807 ✗ auto &t1 = stack.back();
2808 ✗ XmmReg r2, r3, r4, r5;
2809 ✗ xorps(r2, r2);
2810 ✗ xorps(r3, r3);
2811 ✗ cmpltps(r3, t1.first);
2812 ✗ cmpltps(t1.first, r2);
2813 ✗ movaps(r4, t1.first);
2814 ✗ andnps(r4, r3);
2815 ✗ movaps(r3, CPTR(elfloat_one));
2816 ✗ xorps(r5, r5);
2817 ✗ cmpltps(r5, t1.second);
2818 ✗ cmpltps(t1.second, r2);
2819 ✗ movaps(r2, CPTR(elfloat_minusone));
2820 ✗ andps(t1.first, r2);
2821 ✗ andps(r2, t1.second);
2822 ✗ andnps(t1.second, r5);
2823 ✗ andps(r4, r3);
2824 ✗ orps(t1.first, r4);
2825 ✗ andps(t1.second, r3);
2826 ✗ orps(t1.second, r2);
2827 }
2828 }
2829 ✗ else if (iter.op == opNeg) {
2830 if (processSingle) {
2831 ✗ auto &t1 = stack1.back();
2832 ✗ cmpleps(t1, zero);
2833 ✗ andps(t1, CPTR(elfloat_one));
2834 }
2835 else {
2836 ✗ auto &t1 = stack.back();
2837 ✗ cmpleps(t1.first, zero);
2838 ✗ cmpleps(t1.second, zero);
2839 ✗ andps(t1.first, CPTR(elfloat_one));
2840 ✗ andps(t1.second, CPTR(elfloat_one));
2841 }
2842 }
2843 ✗ else if (iter.op == opNegSign) {
2844 if (processSingle) {
2845 ✗ auto& t1 = stack1.back();
2846 ✗ xorps(t1, CPTR(elsignmask));
2847 }
2848 else {
2849 ✗ auto& t1 = stack.back();
2850 ✗ xorps(t1.first, CPTR(elsignmask));
2851 ✗ xorps(t1.second, CPTR(elsignmask));
2852 }
2853 }
2854 ✗ else if (iter.op == opAnd) {
2855 if (processSingle) {
2856 ✗ LogicOp_Single(andps)
2857 }
2858 else {
2859 ✗ LogicOp(andps)
2860 }
2861 }
2862 ✗ else if (iter.op == opOr) {
2863 if (processSingle) {
2864 ✗ LogicOp_Single(orps)
2865 }
2866 else {
2867 ✗ LogicOp(orps)
2868 }
2869 }
2870 ✗ else if (iter.op == opXor) {
2871 if (processSingle) {
2872 ✗ LogicOp_Single(xorps)
2873 }
2874 else {
2875 ✗ LogicOp(xorps)
2876 }
2877 }
2878 ✗ else if (iter.op == opGt) { // a > b (gt) -> b < (lt) a
2879 if (processSingle) {
2880 ✗ CmpOp_Single(cmpltps)
2881 }
2882 else {
2883 ✗ CmpOp(cmpltps)
2884 }
2885 }
2886 ✗ else if (iter.op == opLt) { // a < b (lt) -> b > (gt,nle) a
2887 if (processSingle) {
2888 ✗ CmpOp_Single(cmpnleps)
2889 }
2890 else {
2891 ✗ CmpOp(cmpnleps)
2892 }
2893 }
2894 ✗ else if (iter.op == opEq) { // a == b -> b == a
2895 if (processSingle) {
2896 ✗ CmpOp_Single(cmpeqps)
2897 }
2898 else {
2899 ✗ CmpOp(cmpeqps)
2900 }
2901 }
2902 ✗ else if (iter.op == opNotEq) { // a != b
2903 if (processSingle) {
2904 ✗ CmpOp_Single(cmpneqps)
2905 }
2906 else {
2907 ✗ CmpOp(cmpneqps)
2908 }
2909 }
2910 ✗ else if (iter.op == opLE) { // a <= b -> b >= (ge,nlt) a
2911 if (processSingle) {
2912 ✗ CmpOp_Single(cmpnltps)
2913 }
2914 else {
2915 ✗ CmpOp(cmpnltps)
2916 }
2917 }
2918 ✗ else if (iter.op == opGE) { // a >= b -> b <= (le) a
2919 if (processSingle) {
2920 ✗ CmpOp_Single(cmpleps)
2921 }
2922 else {
2923 ✗ CmpOp(cmpleps)
2924 }
2925 }
2926 ✗ else if (iter.op == opTernary) {
2927 if (processSingle) {
2928 ✗ auto t1 = stack1.back();
2929 ✗ stack1.pop_back();
2930 ✗ auto t2 = stack1.back();
2931 ✗ stack1.pop_back();
2932 ✗ auto t3 = stack1.back();
2933 ✗ stack1.pop_back();
2934 ✗ XmmReg r1;
2935 ✗ xorps(r1, r1);
2936 ✗ cmpltps(r1, t3);
2937 ✗ andps(t2, r1);
2938 ✗ andnps(r1, t1);
2939 ✗ orps(r1, t2);
2940 ✗ stack1.push_back(r1);
2941 }
2942 else {
2943 ✗ auto t1 = stack.back();
2944 ✗ stack.pop_back();
2945 ✗ auto t2 = stack.back();
2946 ✗ stack.pop_back();
2947 ✗ auto t3 = stack.back();
2948 ✗ stack.pop_back();
2949 ✗ XmmReg r1, r2;
2950 ✗ xorps(r1, r1);
2951 ✗ xorps(r2, r2);
2952 ✗ cmpltps(r1, t3.first);
2953 ✗ cmpltps(r2, t3.second);
2954 ✗ andps(t2.first, r1);
2955 ✗ andps(t2.second, r2);
2956 ✗ andnps(r1, t1.first);
2957 ✗ andnps(r2, t1.second);
2958 ✗ orps(r1, t2.first);
2959 ✗ orps(r2, t2.second);
2960 ✗ stack.push_back(std::make_pair(r1, r2));
2961 }
2962 }
2963 ✗ else if (iter.op == opExp) {
2964 if (processSingle) {
2965 ✗ auto &t1 = stack1.back();
2966 ✗ EXP_PS(t1)
2967 }
2968 else {
2969 ✗ auto &t1 = stack.back();
2970 ✗ EXP_PS(t1.first)
2971 ✗ EXP_PS(t1.second)
2972 }
2973 }
2974 ✗ else if (iter.op == opLog) {
2975 if (processSingle) {
2976 ✗ auto &t1 = stack1.back();
2977 ✗ LOG_PS(t1)
2978 }
2979 else {
2980 ✗ auto &t1 = stack.back();
2981 ✗ LOG_PS(t1.first)
2982 ✗ LOG_PS(t1.second)
2983 }
2984 }
2985 ✗ else if (iter.op == opPow) {
2986 if (processSingle) {
2987 ✗ auto t1 = stack1.back();
2988 ✗ stack1.pop_back();
2989 ✗ auto &t2 = stack1.back();
2990 ✗ LOG_PS(t2)
2991 ✗ mulps(t2, t1);
2992 ✗ EXP_PS(t2)
2993 }
2994 else {
2995 ✗ auto t1 = stack.back();
2996 ✗ stack.pop_back();
2997 ✗ auto &t2 = stack.back();
2998 ✗ LOG_PS(t2.first)
2999 ✗ mulps(t2.first, t1.first);
3000 ✗ EXP_PS(t2.first)
3001 ✗ LOG_PS(t2.second)
3002 ✗ mulps(t2.second, t1.second);
3003 ✗ EXP_PS(t2.second)
3004 }
3005 }
3006 ✗ else if (iter.op == opSin) {
3007 if (processSingle) {
3008 ✗ auto& _t1 = stack1.back();
3009 ✗ SINCOS_PS(true, _t1, _t1)
3010 }
3011 else {
3012 ✗ auto& _t1 = stack.back();
3013 ✗ SINCOS_PS(true, _t1.first, _t1.first);
3014 ✗ SINCOS_PS(true, _t1.second, _t1.second);
3015 }
3016 }
3017 ✗ else if (iter.op == opCos) {
3018 if (processSingle) {
3019 ✗ auto& _t1 = stack1.back();
3020 ✗ SINCOS_PS(false, _t1, _t1)
3021 }
3022 else {
3023 ✗ auto& _t1 = stack.back();
3024 ✗ SINCOS_PS(false, _t1.first, _t1.first);
3025 ✗ SINCOS_PS(false, _t1.second, _t1.second);
3026 }
3027 }
3028 ✗ else if (iter.op == opTan) {
3029 if (processSingle) {
3030 ✗ auto& t1 = stack1.back();
3031 ✗ TAN_PS(t1)
3032 }
3033 else {
3034 ✗ auto& t1 = stack.back();
3035 ✗ TAN_PS(t1.first);
3036 ✗ TAN_PS(t1.second);
3037 }
3038 }
3039 ✗ else if (iter.op == opAtan2) {
3040 if (processSingle) {
3041 ✗ auto t1 = stack1.back();
3042 ✗ stack1.pop_back();
3043 ✗ auto& t2 = stack1.back();
3044 ✗ ATAN2_PS(t2, t1);
3045 }
3046 else {
3047 ✗ auto t1 = stack.back();
3048 ✗ stack.pop_back();
3049 ✗ auto& t2 = stack.back();
3050 ✗ ATAN2_PS(t2.first, t1.first);
3051 ✗ ATAN2_PS(t2.second, t1.second);
3052 }
3053 }
3054 ✗ else if (iter.op == opClip) {
3055 // clip(a, low, high) = min(max(a, low),high)
3056 if (processSingle) {
3057 ✗ auto t1 = stack1.back();
3058 ✗ stack1.pop_back();
3059 ✗ auto t2 = stack1.back();
3060 ✗ stack1.pop_back();
3061 ✗ auto &t3 = stack1.back();
3062 ✗ maxps(t3, t2);
3063 ✗ minps(t3, t1);
3064 }
3065 else {
3066 ✗ auto t1 = stack.back();
3067 ✗ stack.pop_back();
3068 ✗ auto t2 = stack.back();
3069 ✗ stack.pop_back();
3070 ✗ auto &t3 = stack.back();
3071 ✗ maxps(t3.first, t2.first);
3072 ✗ minps(t3.first, t1.first);
3073 ✗ maxps(t3.second, t2.second);
3074 ✗ minps(t3.second, t1.second);
3075 }
3076 }
3077 ✗ else if (iter.op == opRound || iter.op == opFloor || iter.op == opCeil || iter.op == opTrunc) {
3078 ✗ const int rounder_flag =
3079 ✗ (iter.op == opRound) ? (FROUND_TO_NEAREST_INT | FROUND_NO_EXC) :
3080 ✗ (iter.op == opFloor) ? (FROUND_TO_NEG_INF | FROUND_NO_EXC) :
3081 ✗ (iter.op == opCeil) ? (FROUND_TO_POS_INF | FROUND_NO_EXC) :
3082 (FROUND_TO_ZERO | FROUND_NO_EXC); // opTrunc
3083 if (processSingle) {
3084 ✗ auto& t1 = stack1.back();
3085 ✗ roundps(t1, t1, rounder_flag);
3086 }
3087 else {
3088 ✗ auto& t1 = stack.back();
3089 ✗ roundps(t1.first, t1.first, rounder_flag);
3090 ✗ roundps(t1.second, t1.second, rounder_flag);
3091 }
3092 }
3093
3094 }
3095 ✗ }
3096
3097 ✗ void main(Reg regptrs, Reg regoffs, Reg niter, Reg SpatialY)
3098 {
3099 ✗ XmmReg zero;
3100 ✗ pxor(zero, zero);
3101 ✗ Reg constptr;
3102 ✗ mov(constptr, (uintptr_t)logexpconst);
3103
3104 ✗ L("wloop");
3105 ✗ cmp(niter, 0);
3106 ✗ je("wend");
3107 //sub(niter, 1);
3108 ✗ dec(niter);
3109
3110 // process two sets, no partial input masking
3111 ✗ if (singleMode)
3112 processingLoop<true, false>(regptrs, zero, constptr, SpatialY);
3113 else
3114 processingLoop<false, false>(regptrs, zero, constptr, SpatialY);
3115
3116 ✗ const int EXTRA = 2; // output pointer, xcounter
3117 if constexpr(sizeof(void *) == 8) {
3118 ✗ int numIter = (numInputs + EXTRA + 1) / 2;
3119 ✗ for (int i = 0; i < numIter; i++) {
3120 ✗ XmmReg r1, r2;
3121 ✗ movdqu(r1, xmmword_ptr[regptrs + 16 * i]);
3122 ✗ movdqu(r2, xmmword_ptr[regoffs + 16 * i]);
3123 ✗ paddq(r1, r2);
3124 ✗ movdqu(xmmword_ptr[regptrs + 16 * i], r1);
3125 }
3126 } else {
3127 int numIter = (numInputs + EXTRA + 3) / 4;
3128 for (int i = 0; i < numIter; i++) {
3129 XmmReg r1, r2;
3130 movdqu(r1, xmmword_ptr[regptrs + 16 * i]);
3131 movdqu(r2, xmmword_ptr[regoffs + 16 * i]);
3132 paddd(r1, r2);
3133 movdqu(xmmword_ptr[regptrs + 16 * i], r1);
3134 }
3135 }
3136
3137 ✗ jmp("wloop");
3138 ✗ L("wend");
3139
3140 ✗ int nrestpixels = planewidth & (singleMode ? 3 : 7);
3141 ✗ if (nrestpixels > 4) // dual process with masking
3142 processingLoop<false, true>(regptrs, zero, constptr, SpatialY);
3143 ✗ else if (nrestpixels == 4) // single process, no masking
3144 processingLoop<true, false>(regptrs, zero, constptr, SpatialY);
3145 ✗ else if (nrestpixels > 0) // single process, masking
3146 processingLoop<true, true>(regptrs, zero, constptr, SpatialY);
3147 ✗ }
3148 };
3149
3150 // avx2 evaluator with two ymm registers
3151 struct ExprEvalAvx2 : public jitasm::function<void, ExprEvalAvx2, uint8_t *, const intptr_t *, intptr_t, intptr_t> {
3152
3153 std::vector<ExprOp> ops;
3154 int numInputs;
3155 int cpuFlags;
3156 int planewidth; // original, lut can overwrite
3157 int planeheight;
3158 bool singleMode;
3159
3160 ✗ ExprEvalAvx2(std::vector<ExprOp> &ops, int numInputs, int cpuFlags, int planewidth, int planeheight, bool singleMode) : ops(ops), numInputs(numInputs), cpuFlags(cpuFlags),
3161 ✗ planewidth(planewidth), planeheight(planeheight), singleMode(singleMode) {}
3162
3163 template<bool processSingle, bool maskUnused>
3164 AVS_FORCEINLINE void processingLoop(Reg &regptrs, YmmReg &zero, Reg &constptr, Reg &SpatialY)
3165 {
3166 ✗ std::list<std::pair<YmmReg, YmmReg>> stack;
3167 ✗ std::list<YmmReg> stack1;
3168
3169 // reason of masking: prevent loading 'junk', out of frame pixels, which can be NaN floats for example.
3170 // If processingLoop works in dual lane mode (!processSingle), masking occurs only for the high lane,
3171 // when there is no need for dual lanes (width mod 16 is <= 8 pixels), processSingle=true is used
3172 ✗ const bool maskIt = maskUnused && ((planewidth & 7) != 0);
3173 ✗ const int mask = ((1 << (planewidth & 7)) - 1);
3174
3175 // mask by zero when we have only 1-7 valid pixels in the lower (single-lane) or upper (dual-lane)
3176 // 1: 2-1 = 1 // 00000001
3177 // 2: 4-1 = 3 // 00000011
3178 // 7: 128-1 = 127 // 01111111
3179
3180 ✗ for (const auto &iter : ops) {
3181 ✗ if (iter.op == opLoadSpatialX) {
3182 if (processSingle) {
3183 ✗ YmmReg r1;
3184 ✗ XmmReg r1x;
3185 ✗ vmovd(r1x, dword_ptr[regptrs + sizeof(void *) * (RWPTR_START_OF_XCOUNTER)]);
3186 ✗ vcvtdq2ps(r1x, r1x);
3187 ✗ vbroadcastss(r1, r1x);
3188 ✗ vaddps(r1, r1, CPTR_AVX(spatialX));
3189 ✗ stack1.push_back(r1);
3190 }
3191 else {
3192 ✗ YmmReg r1, r2;
3193 ✗ XmmReg r1x;
3194 ✗ vmovd(r1x, dword_ptr[regptrs + sizeof(void *) * (RWPTR_START_OF_XCOUNTER)]);
3195 ✗ vcvtdq2ps(r1x, r1x);
3196 ✗ vbroadcastss(r1, r1x);
3197 ✗ vmovaps(r2, r1);
3198 ✗ vaddps(r1, r1, CPTR_AVX(spatialX));
3199 ✗ vaddps(r2, r2, CPTR_AVX(spatialX2));
3200 ✗ stack.push_back(std::make_pair(r1, r2));
3201 }
3202 }
3203 ✗ else if (iter.op == opLoadSpatialY) {
3204 if (processSingle) {
3205 ✗ YmmReg r1;
3206 ✗ XmmReg r1x;
3207 #ifdef JITASM64
3208 ✗ vmovq(r1x, SpatialY);
3209 #else
3210 vmovd(r1x, SpatialY);
3211 #endif
3212 ✗ vcvtdq2ps(r1x, r1x);
3213 ✗ vbroadcastss(r1, r1x);
3214 ✗ stack1.push_back(r1);
3215 }
3216 else {
3217 ✗ YmmReg r1, r2;
3218 ✗ XmmReg r1x;
3219 #ifdef JITASM64
3220 ✗ vmovq(r1x, SpatialY);
3221 #else
3222 vmovd(r1x, SpatialY);
3223 #endif
3224 ✗ vcvtdq2ps(r1x, r1x);
3225 ✗ vbroadcastss(r1, r1x);
3226 ✗ vmovaps(r2, r1);
3227 ✗ stack.push_back(std::make_pair(r1, r2));
3228 }
3229 }
3230 ✗ else if (iter.op == opLoadInternalVar) {
3231 if (processSingle) {
3232 ✗ YmmReg r1;
3233 ✗ XmmReg r1x;
3234 ✗ vmovd(r1x, dword_ptr[regptrs + sizeof(void *) * (iter.e.ival + RWPTR_START_OF_INTERNAL_VARIABLES)]);
3235 ✗ vbroadcastss(r1, r1x);
3236 ✗ stack1.push_back(r1);
3237 }
3238 else {
3239 ✗ YmmReg r1, r2;
3240 ✗ XmmReg r1x;
3241 ✗ vmovd(r1x, dword_ptr[regptrs + sizeof(void *) * (iter.e.ival + RWPTR_START_OF_INTERNAL_VARIABLES)]);
3242 ✗ vbroadcastss(r1, r1x);
3243 ✗ vmovaps(r2, r1);
3244 ✗ stack.push_back(std::make_pair(r1, r2));
3245 }
3246 }
3247 ✗ else if (iter.op == opLoadFramePropVar) {
3248 if (processSingle) {
3249 ✗ YmmReg r1;
3250 ✗ XmmReg r1x;
3251 ✗ vmovd(r1x, dword_ptr[regptrs + sizeof(void*) * (iter.e.ival + RWPTR_START_OF_INTERNAL_FRAMEPROP_VARIABLES)]);
3252 ✗ vbroadcastss(r1, r1x);
3253 ✗ stack1.push_back(r1);
3254 }
3255 else {
3256 ✗ YmmReg r1, r2;
3257 ✗ XmmReg r1x;
3258 ✗ vmovd(r1x, dword_ptr[regptrs + sizeof(void*) * (iter.e.ival + RWPTR_START_OF_INTERNAL_FRAMEPROP_VARIABLES)]);
3259 ✗ vbroadcastss(r1, r1x);
3260 ✗ vmovaps(r2, r1);
3261 ✗ stack.push_back(std::make_pair(r1, r2));
3262 }
3263 }
3264 ✗ else if (iter.op == opLoadSrc8) {
3265 if (processSingle) {
3266 ✗ XmmReg r1x;
3267 ✗ YmmReg r1;
3268 ✗ Reg a;
3269 ✗ mov(a, ptr[regptrs + sizeof(void *) * (iter.e.ival + RWPTR_START_OF_INPUTS)]);
3270 // 8 bytes, 8 pixels * uint8_t
3271 ✗ vmovq(r1x, mmword_ptr[a]);
3272 // 8->32 bits like _mm256_cvtepu8_epi32
3273 ✗ vpmovzxbd(r1, r1x);
3274 // int -> float
3275 ✗ vcvtdq2ps(r1, r1);
3276 ✗ if (maskIt)
3277 ✗ vblendps(r1, zero, r1, mask);
3278 ✗ stack1.push_back(r1);
3279 }
3280 else {
3281 ✗ XmmReg r1x;
3282 ✗ YmmReg r1, r2;
3283 ✗ Reg a;
3284 ✗ mov(a, ptr[regptrs + sizeof(void *) * (iter.e.ival + RWPTR_START_OF_INPUTS)]);
3285 // 16 bytes, 16 pixels * uint8_t
3286 ✗ vmovdqa(r1x, xmmword_ptr[a]);
3287 // 8->16 bits like _mm256_cvtepu8_epi16
3288 ✗ vpmovzxbw(r1, r1x);
3289 // 16->32 bit like _mm256_cvtepu16_epi32
3290 ✗ vextracti128(r1x, r1, 1); // upper 128
3291 ✗ vpmovzxwd(r2, r1x);
3292 ✗ vextracti128(r1x, r1, 0); // lower 128
3293 ✗ vpmovzxwd(r1, r1x);
3294 // int -> float
3295 ✗ vcvtdq2ps(r1, r1);
3296 ✗ vcvtdq2ps(r2, r2);
3297 ✗ if (maskIt)
3298 ✗ vblendps(r2, zero, r2, mask);
3299 ✗ stack.push_back(std::make_pair(r1, r2));
3300 }
3301 }
3302 ✗ else if (iter.op == opLoadSrc16) {
3303 if (processSingle) {
3304 ✗ XmmReg r1x;
3305 ✗ YmmReg r1;
3306 ✗ Reg a;
3307 ✗ mov(a, ptr[regptrs + sizeof(void *) * (iter.e.ival + RWPTR_START_OF_INPUTS)]);
3308 // 16 bytes, 8 pixels * uint16_t
3309 ✗ vmovdqa(r1x, xmmword_ptr[a]);
3310 // 16->32 bit like _mm256_cvtepu16_epi32
3311 ✗ vpmovzxwd(r1, r1x);
3312 // int -> float
3313 ✗ vcvtdq2ps(r1, r1);
3314 ✗ if (maskIt)
3315 ✗ vblendps(r1, zero, r1, mask);
3316 ✗ stack1.push_back(r1);
3317 }
3318 else {
3319 ✗ XmmReg r1x;
3320 ✗ YmmReg r1, r2;
3321 ✗ Reg a;
3322 ✗ mov(a, ptr[regptrs + sizeof(void *) * (iter.e.ival + RWPTR_START_OF_INPUTS)]);
3323 // 32 bytes, 16 pixels * uint16_t
3324 ✗ vmovdqa(r1, ymmword_ptr[a]);
3325 // 16->32 bit like _mm256_cvtepu16_epi32
3326 ✗ vextracti128(r1x, r1, 1); // upper 128
3327 ✗ vpmovzxwd(r2, r1x);
3328 ✗ vextracti128(r1x, r1, 0); // lower 128
3329 ✗ vpmovzxwd(r1, r1x);
3330 // int -> float
3331 ✗ vcvtdq2ps(r1, r1);
3332 ✗ vcvtdq2ps(r2, r2);
3333 ✗ if (maskIt)
3334 ✗ vblendps(r2, zero, r2, mask);
3335 ✗ stack.push_back(std::make_pair(r1, r2));
3336 }
3337 }
3338 ✗ else if (iter.op == opLoadSrcF32) {
3339 if (processSingle) {
3340 ✗ YmmReg r1;
3341 ✗ Reg a;
3342 ✗ mov(a, ptr[regptrs + sizeof(void *) * (iter.e.ival + RWPTR_START_OF_INPUTS)]);
3343 // 32 bytes, 8 * float
3344 ✗ vmovdqa(r1, ymmword_ptr[a]);
3345 ✗ if (maskIt)
3346 ✗ vblendps(r1, zero, r1, mask);
3347 ✗ stack1.push_back(r1);
3348 }
3349 else {
3350 ✗ YmmReg r1, r2;
3351 ✗ Reg a;
3352 ✗ mov(a, ptr[regptrs + sizeof(void *) * (iter.e.ival + RWPTR_START_OF_INPUTS)]);
3353 // 32 bytes, 8 * float
3354 ✗ vmovdqa(r1, ymmword_ptr[a]);
3355 ✗ vmovdqa(r2, ymmword_ptr[a + 32]); // needs 64 byte aligned data to prevent read past valid data!
3356 ✗ if (maskIt)
3357 ✗ vblendps(r2, zero, r2, mask);
3358 ✗ stack.push_back(std::make_pair(r1, r2));
3359 }
3360 }
3361 ✗ else if (iter.op == opLoadSrcF16) { // not supported in avs+
3362 if (processSingle) {
3363 ✗ YmmReg r1;
3364 ✗ Reg a;
3365 ✗ mov(a, ptr[regptrs + sizeof(void *) * (iter.e.ival + RWPTR_START_OF_INPUTS)]);
3366 ✗ vcvtph2ps(r1, xmmword_ptr[a]);
3367 ✗ if (maskIt)
3368 ✗ vblendps(r1, zero, r1, mask);
3369 ✗ stack1.push_back(r1);
3370 }
3371 else {
3372 ✗ YmmReg r1, r2;
3373 ✗ Reg a;
3374 ✗ mov(a, ptr[regptrs + sizeof(void *) * (iter.e.ival + RWPTR_START_OF_INPUTS)]);
3375 ✗ vcvtph2ps(r1, xmmword_ptr[a]);
3376 ✗ vcvtph2ps(r2, xmmword_ptr[a + 16]);
3377 ✗ if (maskIt)
3378 ✗ vblendps(r2, zero, r2, mask);
3379 ✗ stack.push_back(std::make_pair(r1, r2));
3380 }
3381 }
3382 ✗ else if (iter.op == opLoadVar) {
3383 if (processSingle) {
3384 ✗ YmmReg r1;
3385 // 32 bytes/variable
3386 ✗ int offset = sizeof(void *) * RWPTR_START_OF_USERVARIABLES + 32 * iter.e.ival;
3387 // 32 bytes, 8 * float
3388 ✗ vmovdqa(r1, ymmword_ptr[regptrs + offset]);
3389 ✗ if (maskIt)
3390 ✗ vblendps(r1, zero, r1, mask);
3391 ✗ stack1.push_back(r1);
3392 }
3393 else {
3394 ✗ YmmReg r1, r2;
3395 // 64 bytes/variable
3396 ✗ int offset = sizeof(void *) * RWPTR_START_OF_USERVARIABLES + 64 * iter.e.ival;
3397 // 32 bytes, 8 * float
3398 ✗ vmovdqa(r1, ymmword_ptr[regptrs + offset]);
3399 ✗ vmovdqa(r2, ymmword_ptr[regptrs + offset + 32]); // needs 64 byte aligned data to prevent read past valid data!
3400 ✗ if (maskIt)
3401 ✗ vblendps(r2, zero, r2, mask);
3402 ✗ stack.push_back(std::make_pair(r1, r2));
3403 }
3404 }
3405 ✗ else if (iter.op == opLoadConst) {
3406 if (processSingle) {
3407 ✗ YmmReg r1;
3408 ✗ Reg32 a;
3409 ✗ XmmReg r1x;
3410 ✗ mov(a, iter.e.ival);
3411 ✗ vmovd(r1x, a);
3412 ✗ vbroadcastss(r1, r1x);
3413 ✗ stack1.push_back(r1);
3414 }
3415 else {
3416 ✗ YmmReg r1, r2;
3417 ✗ Reg32 a;
3418 ✗ XmmReg r1x;
3419 ✗ mov(a, iter.e.ival);
3420 ✗ vmovd(r1x, a);
3421 ✗ vbroadcastss(r1, r1x);
3422 ✗ vmovaps(r2, r1);
3423 ✗ stack.push_back(std::make_pair(r1, r2));
3424 }
3425 }
3426 ✗ else if (iter.op == opDup) {
3427 if (processSingle) {
3428 ✗ auto p = std::next(stack1.rbegin(), iter.e.ival);
3429 ✗ YmmReg r1;
3430 ✗ vmovaps(r1, *p);
3431 ✗ stack1.push_back(r1);
3432 }
3433 else {
3434 ✗ auto p = std::next(stack.rbegin(), iter.e.ival);
3435 ✗ YmmReg r1, r2;
3436 ✗ vmovaps(r1, p->first);
3437 ✗ vmovaps(r2, p->second);
3438 ✗ stack.push_back(std::make_pair(r1, r2));
3439 }
3440 }
3441 ✗ else if (iter.op == opSwap) {
3442 if(processSingle)
3443 ✗ std::swap(stack1.back(), *std::next(stack1.rbegin(), iter.e.ival));
3444 else
3445 ✗ std::swap(stack.back(), *std::next(stack.rbegin(), iter.e.ival));
3446 }
3447 ✗ else if (iter.op == opAdd) {
3448 if (processSingle) {
3449 ✗ TwoArgOp_Single_Avx(vaddps);
3450 }
3451 else {
3452 ✗ TwoArgOp_Avx(vaddps);
3453 }
3454 }
3455 ✗ else if (iter.op == opSub) {
3456 if (processSingle) {
3457 ✗ TwoArgOp_Single_Avx(vsubps);
3458 }
3459 else {
3460 ✗ TwoArgOp_Avx(vsubps);
3461 }
3462 }
3463 ✗ else if (iter.op == opMul) {
3464 if (processSingle) {
3465 ✗ TwoArgOp_Single_Avx(vmulps);
3466 }
3467 else {
3468 ✗ TwoArgOp_Avx(vmulps);
3469 }
3470 }
3471 ✗ else if (iter.op == opDiv) {
3472 if (processSingle) {
3473 ✗ TwoArgOp_Single_Avx(vdivps);
3474 }
3475 else {
3476 ✗ TwoArgOp_Avx(vdivps);
3477 }
3478 }
3479 ✗ else if (iter.op == opFmod) {
3480 if (processSingle) {
3481 ✗ auto t1 = stack1.back();
3482 ✗ stack1.pop_back();
3483 ✗ auto &t2 = stack1.back();
3484 ✗ FMOD_PS_AVX(t2, t1)
3485 }
3486 else {
3487 ✗ auto t1 = stack.back();
3488 ✗ stack.pop_back();
3489 ✗ auto &t2 = stack.back();
3490 ✗ FMOD_PS_AVX(t2.first, t1.first)
3491 ✗ FMOD_PS_AVX(t2.second, t1.second)
3492 }
3493 }
3494 ✗ else if (iter.op == opMax) {
3495 if (processSingle) {
3496 ✗ TwoArgOp_Single_Avx(vmaxps);
3497 }
3498 else {
3499 ✗ TwoArgOp_Avx(vmaxps);
3500 }
3501 }
3502 ✗ else if (iter.op == opMin) {
3503 if (processSingle) {
3504 ✗ TwoArgOp_Single_Avx(vminps);
3505 }
3506 else {
3507 ✗ TwoArgOp_Avx(vminps);
3508 }
3509 }
3510 ✗ else if (iter.op == opSqrt) {
3511 if (processSingle) {
3512 ✗ auto &t1 = stack1.back();
3513 ✗ vmaxps(t1, t1, zero);
3514 ✗ vsqrtps(t1, t1);
3515 }
3516 else {
3517 ✗ auto &t1 = stack.back();
3518 ✗ vmaxps(t1.first, t1.first, zero);
3519 ✗ vmaxps(t1.second, t1.second, zero);
3520 ✗ vsqrtps(t1.first, t1.first);
3521 ✗ vsqrtps(t1.second, t1.second);
3522 }
3523 }
3524 ✗ else if (iter.op == opStore8) {
3525 if (processSingle) {
3526 ✗ auto t1 = stack1.back();
3527 ✗ stack1.pop_back();
3528 ✗ Reg a;
3529 ✗ vaddps(t1, t1, CPTR_AVX(elfloat_half)); // rounder for truncate! no banker's rounding
3530 ✗ vmaxps(t1, t1, zero);
3531 ✗ vminps(t1, t1, CPTR_AVX(elstore8));
3532 ✗ mov(a, ptr[regptrs]);
3533 ✗ vcvttps2dq(t1, t1); // float to int32 no bankers rounding
3534 ✗ XmmReg r1x, r2x;
3535 // 32 -> 16 bits from ymm 8 integers to xmm 8 words
3536 // first
3537 ✗ vextracti128(r1x, t1, 0);
3538 ✗ vextracti128(r2x, t1, 1);
3539 ✗ vpackusdw(r1x, r1x, r2x); // _mm_packus_epi32: w7 w6 w5 w4 w3 w2 w1 w0
3540 // 16 -> 8 bits
3541 ✗ vpackuswb(r1x, r1x, r1x); // _mm_packus_epi16: w3 w2 w1 w0 w3 w2 w1 w0
3542 ✗ vmovq(mmword_ptr[a], r1x); // store 8 bytes
3543 }
3544 else {
3545 ✗ auto t1 = stack.back();
3546 ✗ stack.pop_back();
3547 ✗ Reg a;
3548 ✗ vaddps(t1.first, t1.first, CPTR_AVX(elfloat_half)); // rounder for truncate! no banker's rounding
3549 ✗ vmaxps(t1.first, t1.first, zero);
3550 ✗ vaddps(t1.second, t1.second, CPTR_AVX(elfloat_half)); // rounder for truncate! no banker's rounding
3551 ✗ vmaxps(t1.second, t1.second, zero);
3552 ✗ vminps(t1.first, t1.first, CPTR_AVX(elstore8));
3553 ✗ vminps(t1.second, t1.second, CPTR_AVX(elstore8));
3554 ✗ mov(a, ptr[regptrs]);
3555 ✗ vcvttps2dq(t1.first, t1.first); // float to int32 no bankers rounding
3556 ✗ vcvttps2dq(t1.second, t1.second);
3557 // we have 8 integers in t.first and another 8 in t.second
3558 // second first
3559 // d15 d14 d13 d12 d11 d10 d9 d8 d7 d6 d5 d4 d3 d2 d1 d0 // 16x32 bit integers in two ymm registers. not really 256 bits, but 2x128 bits
3560 ✗ XmmReg r1x, r2x, r_lo_x;
3561 // 32 -> 16 bits from ymm 8 integers to xmm 8 words
3562 // first
3563 ✗ vextracti128(r1x, t1.first, 0);
3564 ✗ vextracti128(r2x, t1.first, 1);
3565 ✗ vpackusdw(r_lo_x, r1x, r2x); // _mm_packus_epi32: w7 w6 w5 w4 w3 w2 w1 w0
3566 // second
3567 ✗ vextracti128(r1x, t1.second, 0); // not perfect, lower 128 bits of t1 could be used as xmm in packus. Cannot tell jitasm that xxmN is lower ymmN
3568 ✗ vextracti128(r2x, t1.second, 1);
3569 ✗ vpackusdw(r1x, r1x, r2x); // _mm_packus_epi32: w7 w6 w5 w4 w3 w2 w1 w0
3570 // 16 -> 8 bits
3571 ✗ vpackuswb(r1x, r_lo_x, r1x); // _mm_packus_epi16: w3 w2 w1 w0 w3 w2 w1 w0
3572 ✗ vmovdqa(xmmword_ptr[a], r1x); // store 16 bytes
3573 }
3574 }
3575 ✗ else if (iter.op == opStore10 // avs+
3576 ✗ || iter.op == opStore12 // avs+
3577 ✗ || iter.op == opStore14 // avs+
3578 ✗ || iter.op == opStore16
3579 ) {
3580 if (processSingle) {
3581 ✗ auto t1 = stack1.back();
3582 ✗ stack1.pop_back();
3583 ✗ Reg a;
3584 ✗ vaddps(t1, t1, CPTR_AVX(elfloat_half)); // rounder for truncate! no banker's rounding
3585 ✗ vmaxps(t1, t1, zero);
3586 ✗ switch (iter.op) {
3587 ✗ case opStore10:
3588 ✗ vminps(t1, t1, CPTR_AVX(elstore10));
3589 ✗ break;
3590 ✗ case opStore12:
3591 ✗ vminps(t1, t1, CPTR_AVX(elstore12));
3592 ✗ break;
3593 ✗ case opStore14:
3594 ✗ vminps(t1, t1, CPTR_AVX(elstore14));
3595 ✗ break;
3596 ✗ case opStore16:
3597 ✗ vminps(t1, t1, CPTR_AVX(elstore16));
3598 ✗ break;
3599 }
3600 ✗ mov(a, ptr[regptrs]);
3601 ✗ vcvttps2dq(t1, t1); // min / max clamp ensures that high words are zero
3602 ✗ XmmReg r1x, r2x;
3603 // 32 -> 16 bits from ymm 8 integers to xmm 8 words
3604 ✗ vextracti128(r1x, t1, 0); // not perfect, lower 128 bits of t1 could be used as xmm in packus. Cannot tell jitasm that xxmN is lower ymmN
3605 ✗ vextracti128(r2x, t1, 1);
3606 ✗ vpackusdw(r1x, r1x, r2x); // _mm_packus_epi32: w7 w6 w5 w4 w3 w2 w1 w0
3607 ✗ vmovdqa(xmmword_ptr[a], r1x);
3608 }
3609 else {
3610 ✗ auto t1 = stack.back();
3611 ✗ stack.pop_back();
3612 ✗ Reg a;
3613 ✗ vaddps(t1.first, t1.first, CPTR_AVX(elfloat_half)); // rounder for truncate! no banker's rounding
3614 ✗ vmaxps(t1.first, t1.first, zero);
3615 ✗ vaddps(t1.second, t1.second, CPTR_AVX(elfloat_half)); // rounder for truncate! no banker's rounding
3616 ✗ vmaxps(t1.second, t1.second, zero);
3617 ✗ switch (iter.op) {
3618 ✗ case opStore10:
3619 ✗ vminps(t1.first, t1.first, CPTR_AVX(elstore10));
3620 ✗ vminps(t1.second, t1.second, CPTR_AVX(elstore10));
3621 ✗ break;
3622 ✗ case opStore12:
3623 ✗ vminps(t1.first, t1.first, CPTR_AVX(elstore12));
3624 ✗ vminps(t1.second, t1.second, CPTR_AVX(elstore12));
3625 ✗ break;
3626 ✗ case opStore14:
3627 ✗ vminps(t1.first, t1.first, CPTR_AVX(elstore14));
3628 ✗ vminps(t1.second, t1.second, CPTR_AVX(elstore14));
3629 ✗ break;
3630 ✗ case opStore16:
3631 ✗ vminps(t1.first, t1.first, CPTR_AVX(elstore16));
3632 ✗ vminps(t1.second, t1.second, CPTR_AVX(elstore16));
3633 ✗ break;
3634 }
3635 ✗ mov(a, ptr[regptrs]);
3636 ✗ vcvttps2dq(t1.first, t1.first); // min / max clamp ensures that high words are zero
3637 ✗ vcvttps2dq(t1.second, t1.second);
3638 // we have 8 integers in t.first and another 8 in t.second
3639 // second first
3640 // d15 d14 d13 d12 d11 d10 d9 d8 d7 d6 d5 d4 d3 d2 d1 d0 // 16x32 bit integers in two ymm registers. not really 256 bits, but 2x128 bits
3641 ✗ XmmReg r1x, r2x;
3642 // 32 -> 16 bits from ymm 8 integers to xmm 8 words
3643 // first
3644 ✗ vextracti128(r1x, t1.first, 0); // not perfect, lower 128 bits of t1 could be used as xmm in packus. Cannot tell jitasm that xxmN is lower ymmN
3645 ✗ vextracti128(r2x, t1.first, 1);
3646 ✗ vpackusdw(r1x, r1x, r2x); // _mm_packus_epi32: w7 w6 w5 w4 w3 w2 w1 w0
3647 ✗ vmovdqa(xmmword_ptr[a], r1x);
3648 // second
3649 ✗ vextracti128(r1x, t1.second, 0);
3650 ✗ vextracti128(r2x, t1.second, 1);
3651 ✗ vpackusdw(r1x, r1x, r2x); // _mm_packus_epi32: w7 w6 w5 w4 w3 w2 w1 w0
3652 ✗ vmovdqa(xmmword_ptr[a + 16], r1x);
3653 }
3654 ✗ }
3655 ✗ else if (iter.op == opStoreF32) {
3656 if (processSingle) {
3657 ✗ auto t1 = stack1.back();
3658 ✗ stack1.pop_back();
3659 ✗ Reg a;
3660 ✗ mov(a, ptr[regptrs]);
3661 ✗ vmovaps(ymmword_ptr[a], t1);
3662 } else {
3663 ✗ auto t1 = stack.back();
3664 ✗ stack.pop_back();
3665 ✗ Reg a;
3666 ✗ mov(a, ptr[regptrs]);
3667 ✗ vmovaps(ymmword_ptr[a], t1.first);
3668 ✗ vmovaps(ymmword_ptr[a + 32], t1.second); // this needs 64 byte aligned data to prevent overwrite!
3669 }
3670 }
3671 ✗ else if (iter.op == opStoreF16) { // not supported in avs+
3672 if (processSingle) {
3673 ✗ auto t1 = stack1.back();
3674 ✗ stack1.pop_back();
3675 ✗ Reg a;
3676 ✗ mov(a, ptr[regptrs]);
3677 ✗ vcvtps2ph(xmmword_ptr[a], t1, 0);
3678 } else {
3679 ✗ auto t1 = stack.back();
3680 ✗ stack.pop_back();
3681 ✗ Reg a;
3682 ✗ mov(a, ptr[regptrs]);
3683 ✗ vcvtps2ph(xmmword_ptr[a], t1.first, 0);
3684 ✗ vcvtps2ph(xmmword_ptr[a + 16], t1.second, 0);
3685 }
3686 }
3687 ✗ else if (iter.op == opStoreVar || iter.op == opStoreVarAndDrop1) {
3688 if (processSingle) {
3689 ✗ auto t1 = stack1.back();
3690 // 32 bytes/variable
3691 ✗ int offset = sizeof(void *) * RWPTR_START_OF_USERVARIABLES + 32 * iter.e.ival;
3692 ✗ vmovaps(ymmword_ptr[regptrs + offset], t1);
3693 ✗ if (iter.op == opStoreVarAndDrop1)
3694 ✗ stack1.pop_back();
3695 }
3696 else {
3697 ✗ auto t1 = stack.back();
3698 // 64 bytes/variable
3699 ✗ int offset = sizeof(void *) * RWPTR_START_OF_USERVARIABLES + 64 * iter.e.ival;
3700 ✗ vmovaps(ymmword_ptr[regptrs + offset], t1.first);
3701 ✗ vmovaps(ymmword_ptr[regptrs + offset + 32], t1.second); // this needs 64 byte aligned data to prevent overwrite!
3702 ✗ if (iter.op == opStoreVarAndDrop1)
3703 ✗ stack.pop_back();
3704 }
3705 ✗ }
3706 ✗ else if (iter.op == opAbs) {
3707 if (processSingle) {
3708 ✗ auto &t1 = stack1.back();
3709 ✗ vandps(t1, t1, CPTR_AVX(elabsmask));
3710 }
3711 else {
3712 ✗ auto &t1 = stack.back();
3713 ✗ vandps(t1.first, t1.first, CPTR_AVX(elabsmask));
3714 ✗ vandps(t1.second, t1.second, CPTR_AVX(elabsmask));
3715 }
3716 }
3717 ✗ else if (iter.op == opSgn) {
3718 // 1, 0, -1
3719 if (processSingle) {
3720 ✗ auto &t1 = stack1.back();
3721 ✗ YmmReg r1, r2;
3722 ✗ vxorps(r2, r2, r2);
3723 ✗ vcmpps(r1, t1, r2, _CMP_GT_OQ);
3724 ✗ vcmpps(t1, t1, r2, _CMP_LT_OQ);
3725 ✗ vandps(r1, r1, CPTR_AVX(elfloat_one));
3726 ✗ vandps(t1, t1, CPTR_AVX(elfloat_minusone));
3727 ✗ vorps(t1, r1, t1);
3728 }
3729 else {
3730 ✗ auto &t1 = stack.back();
3731 ✗ YmmReg r2, r3, r4, r5;
3732 ✗ vxorps(r2, r2, r2);
3733 ✗ vcmpps(r3, t1.first, r2, _CMP_GT_OQ);
3734 ✗ vcmpps(t1.first, t1.first, r2, _CMP_LT_OQ);
3735 ✗ vcmpps(r4, t1.second, r2, _CMP_GT_OQ);
3736 ✗ vcmpps(t1.second, t1.second, r2, _CMP_LT_OQ);
3737 ✗ vmovaps(r2, CPTR_AVX(elfloat_one));
3738 ✗ vandps(r3, r3, r2);
3739 ✗ vmovaps(r5, CPTR_AVX(elfloat_minusone));
3740 ✗ vblendvps(t1.first, r3, r5, t1.first);
3741 ✗ vandps(r2, r4, r2);
3742 ✗ vblendvps(t1.second, r2, r5, t1.second);
3743 }
3744 }
3745 ✗ else if (iter.op == opNeg) {
3746 if (processSingle) {
3747 ✗ auto &t1 = stack1.back();
3748 ✗ vcmpps(t1, t1, zero, _CMP_LE_OQ); // cmpleps
3749 ✗ vandps(t1, t1, CPTR_AVX(elfloat_one));
3750 }
3751 else {
3752 ✗ auto &t1 = stack.back();
3753 ✗ vcmpps(t1.first, t1.first, zero, _CMP_LE_OQ); // cmpleps
3754 ✗ vcmpps(t1.second, t1.second, zero, _CMP_LE_OQ);
3755 ✗ vandps(t1.first, t1.first, CPTR_AVX(elfloat_one));
3756 ✗ vandps(t1.second, t1.second, CPTR_AVX(elfloat_one));
3757 }
3758 }
3759 ✗ else if (iter.op == opNegSign) {
3760 if (processSingle) {
3761 ✗ auto& t1 = stack1.back();
3762 ✗ vxorps(t1, t1, CPTR_AVX(elsignmask));
3763 }
3764 else {
3765 ✗ auto& t1 = stack.back();
3766 ✗ vxorps(t1.first, t1.first, CPTR_AVX(elsignmask));
3767 ✗ vxorps(t1.second, t1.second, CPTR_AVX(elsignmask));
3768 }
3769 }
3770 ✗ else if (iter.op == opAnd) {
3771 if (processSingle) {
3772 ✗ LogicOp_Single_Avx(vandps);
3773 }
3774 else {
3775 ✗ LogicOp_Avx(vandps);
3776 }
3777 }
3778 ✗ else if (iter.op == opOr) {
3779 if (processSingle) {
3780 ✗ LogicOp_Single_Avx(vorps);
3781 }
3782 else {
3783 ✗ LogicOp_Avx(vorps);
3784 }
3785 }
3786 ✗ else if (iter.op == opXor) {
3787 if (processSingle) {
3788 ✗ LogicOp_Single_Avx(vxorps);
3789 }
3790 else {
3791 ✗ LogicOp_Avx(vxorps);
3792 }
3793 }
3794 ✗ else if (iter.op == opGt) { // a > b (gt) -> b < (lt) a
3795 if (processSingle) {
3796 ✗ CmpOp_Single_Avx(vcmpps, _CMP_LT_OQ); // cmpltps
3797 }
3798 else {
3799 ✗ CmpOp_Avx(vcmpps, _CMP_LT_OQ) // cmpltps
3800 }
3801 }
3802 ✗ else if (iter.op == opLt) { // a < b (lt) -> b > (gt,nle) a
3803 if (processSingle) {
3804 ✗ CmpOp_Single_Avx(vcmpps, _CMP_GT_OQ); // cmpnleps
3805 }
3806 else {
3807 ✗ CmpOp_Avx(vcmpps, _CMP_GT_OQ); // cmpnleps
3808 }
3809 }
3810 ✗ else if (iter.op == opEq) {
3811 if (processSingle) {
3812 ✗ CmpOp_Single_Avx(vcmpps, _CMP_EQ_OQ);
3813 }
3814 else {
3815 ✗ CmpOp_Avx(vcmpps, _CMP_EQ_OQ);
3816 }
3817 }
3818 ✗ else if (iter.op == opNotEq) { // avs+
3819 if (processSingle) {
3820 ✗ CmpOp_Single_Avx(vcmpps, _CMP_NEQ_OQ);
3821 }
3822 else {
3823 ✗ CmpOp_Avx(vcmpps, _CMP_NEQ_OQ);
3824 }
3825 }
3826 ✗ else if (iter.op == opLE) { // a <= b -> b >= (ge,nlt) a
3827 if (processSingle) {
3828 ✗ CmpOp_Single_Avx(vcmpps, _CMP_GE_OS); // cmpnltps
3829 }
3830 else {
3831 ✗ CmpOp_Avx(vcmpps, _CMP_GE_OS) // cmpnltps
3832 }
3833 }
3834 ✗ else if (iter.op == opGE) { // a >= b -> b <= (le) a
3835 if (processSingle) {
3836 ✗ CmpOp_Single_Avx(vcmpps, _CMP_LE_OS) // cmpleps
3837 }
3838 else {
3839 ✗ CmpOp_Avx(vcmpps, _CMP_LE_OS) // cmpleps
3840 }
3841 }
3842 ✗ else if (iter.op == opTernary) {
3843 if (processSingle) {
3844 ✗ auto t1 = stack1.back();
3845 ✗ stack1.pop_back();
3846 ✗ auto t2 = stack1.back();
3847 ✗ stack1.pop_back();
3848 ✗ auto t3 = stack1.back();
3849 ✗ stack1.pop_back();
3850 ✗ YmmReg r1;
3851 ✗ vxorps(r1, r1, r1);
3852 ✗ vcmpps(r1, r1, t3, _CMP_LT_OQ); // cmpltps -> vcmpps ... _CMP_LT_OQ
3853 ✗ vandps(t2, t2, r1);
3854 ✗ vandnps(r1, r1, t1);
3855 ✗ vorps(r1, r1, t2);
3856 ✗ stack1.push_back(r1);
3857 }
3858 else {
3859 ✗ auto t1 = stack.back();
3860 ✗ stack.pop_back();
3861 ✗ auto t2 = stack.back();
3862 ✗ stack.pop_back();
3863 ✗ auto t3 = stack.back();
3864 ✗ stack.pop_back();
3865 ✗ YmmReg r1, r2;
3866 ✗ vxorps(r1, r1, r1);
3867 ✗ vxorps(r2, r2, r2);
3868 ✗ vcmpps(r1, r1, t3.first, _CMP_LT_OQ); // cmpltps -> vcmpps ... _CMP_LT_OQ
3869 ✗ vcmpps(r2, r2, t3.second, _CMP_LT_OQ);
3870 ✗ vandps(t2.first, t2.first, r1);
3871 ✗ vandps(t2.second, t2.second, r2);
3872 ✗ vandnps(r1, r1, t1.first);
3873 ✗ vandnps(r2, r2, t1.second);
3874 ✗ vorps(r1, r1, t2.first);
3875 ✗ vorps(r2, r2, t2.second);
3876 ✗ stack.push_back(std::make_pair(r1, r2));
3877 }
3878 }
3879 ✗ else if (iter.op == opExp) {
3880 if (processSingle) {
3881 ✗ auto &t1 = stack1.back();
3882 ✗ EXP_PS_AVX(t1);
3883 }
3884 else {
3885 ✗ auto &t1 = stack.back();
3886 ✗ EXP_PS_AVX(t1.first);
3887 ✗ EXP_PS_AVX(t1.second);
3888 }
3889 }
3890 ✗ else if (iter.op == opLog) {
3891 if (processSingle) {
3892 ✗ auto &t1 = stack1.back();
3893 ✗ LOG_PS_AVX(t1);
3894 } else {
3895 ✗ auto &t1 = stack.back();
3896 ✗ LOG_PS_AVX(t1.first);
3897 ✗ LOG_PS_AVX(t1.second);
3898 }
3899 }
3900 ✗ else if (iter.op == opPow) {
3901 if (processSingle) {
3902 ✗ auto t1 = stack1.back();
3903 ✗ stack1.pop_back();
3904 ✗ auto &t2 = stack1.back();
3905 ✗ LOG_PS_AVX(t2);
3906 ✗ vmulps(t2, t2, t1);
3907 ✗ EXP_PS_AVX(t2);
3908 } else {
3909 ✗ auto t1 = stack.back();
3910 ✗ stack.pop_back();
3911 ✗ auto &t2 = stack.back();
3912 ✗ LOG_PS_AVX(t2.first);
3913 ✗ vmulps(t2.first, t2.first, t1.first);
3914 ✗ EXP_PS_AVX(t2.first);
3915 ✗ LOG_PS_AVX(t2.second);
3916 ✗ vmulps(t2.second, t2.second, t1.second);
3917 ✗ EXP_PS_AVX(t2.second);
3918 }
3919 }
3920 ✗ else if (iter.op == opSin) {
3921 if (processSingle) {
3922 ✗ auto& _t1 = stack1.back();
3923 ✗ SINCOS_PS_AVX(true, _t1, _t1);
3924 }
3925 else {
3926 ✗ auto& _t1 = stack.back();
3927 ✗ SINCOS_PS_AVX(true, _t1.first, _t1.first);
3928 ✗ SINCOS_PS_AVX(true, _t1.second, _t1.second);
3929 }
3930 }
3931 ✗ else if (iter.op == opCos) {
3932 if (processSingle) {
3933 ✗ auto& _t1 = stack1.back();
3934 ✗ SINCOS_PS_AVX(false, _t1, _t1);
3935 }
3936 else {
3937 ✗ auto& _t1 = stack.back();
3938 ✗ SINCOS_PS_AVX(false, _t1.first, _t1.first);
3939 ✗ SINCOS_PS_AVX(false, _t1.second, _t1.second);
3940 }
3941 }
3942 ✗ else if (iter.op == opTan) {
3943 if (processSingle) {
3944 ✗ auto& t1 = stack1.back();
3945 ✗ TAN_PS_AVX(t1);
3946 }
3947 else {
3948 ✗ auto& t1 = stack.back();
3949 ✗ TAN_PS_AVX(t1.first);
3950 ✗ TAN_PS_AVX(t1.second);
3951 }
3952 }
3953 ✗ else if (iter.op == opAtan2) {
3954 if (processSingle) {
3955 ✗ auto t1 = stack1.back();
3956 ✗ stack1.pop_back();
3957 ✗ auto &t2 = stack1.back();
3958 ✗ ATAN2_PS_AVX(t2, t1);
3959 } else {
3960 ✗ auto t1 = stack.back();
3961 ✗ stack.pop_back();
3962 ✗ auto &t2 = stack.back();
3963 ✗ ATAN2_PS_AVX(t2.first, t1.first);
3964 ✗ ATAN2_PS_AVX(t2.second, t1.second);
3965 }
3966 }
3967 ✗ else if (iter.op == opClip) {
3968 // clip(a, low, high) = min(max(a, low),high)
3969 if (processSingle) {
3970 ✗ auto t1 = stack1.back();
3971 ✗ stack1.pop_back();
3972 ✗ auto t2 = stack1.back();
3973 ✗ stack1.pop_back();
3974 ✗ auto &t3 = stack1.back();
3975 ✗ vmaxps(t3, t3, t2);
3976 ✗ vminps(t3, t3, t1);
3977 }
3978 else {
3979 ✗ auto t1 = stack.back();
3980 ✗ stack.pop_back();
3981 ✗ auto t2 = stack.back();
3982 ✗ stack.pop_back();
3983 ✗ auto &t3 = stack.back();
3984 ✗ vmaxps(t3.first, t3.first, t2.first);
3985 ✗ vminps(t3.first, t3.first, t1.first);
3986 ✗ vmaxps(t3.second, t3.second, t2.second);
3987 ✗ vminps(t3.second, t3.second, t1.second);
3988 }
3989 }
3990 ✗ else if (iter.op == opRound || iter.op == opFloor || iter.op == opCeil || iter.op == opTrunc) {
3991 ✗ const int rounder_flag =
3992 ✗ (iter.op == opRound) ? (FROUND_TO_NEAREST_INT | FROUND_NO_EXC) :
3993 ✗ (iter.op == opFloor) ? (FROUND_TO_NEG_INF | FROUND_NO_EXC) :
3994 ✗ (iter.op == opCeil) ? (FROUND_TO_POS_INF | FROUND_NO_EXC) :
3995 (FROUND_TO_ZERO | FROUND_NO_EXC); // opTrunc
3996 if (processSingle) {
3997 ✗ auto& t1 = stack1.back();
3998 ✗ vroundps(t1, t1, rounder_flag);
3999 }
4000 else {
4001 ✗ auto& t1 = stack.back();
4002 ✗ vroundps(t1.first, t1.first, rounder_flag);
4003 ✗ vroundps(t1.second, t1.second, rounder_flag);
4004 }
4005 }
4006 }
4007 ✗ }
4008 /*
4009 In brief:
4010 jitasm was modded to accept avx_epilog_=true for code generation
4011
4012 Why: couldn't use vzeroupper because prolog/epilog was saving all xmm6:xmm15 registers even if they were not used at all
4013 Why2: movaps was generated instead of vmovaps for prolog/epilog
4014 Why3: internal register reordering/saving was non-vex encoded
4015 All these issues resulted in AVX->SSE2 penalty
4016
4017 From MSDN:
4018 XMM6:XMM15, YMM6:YMM15 rules for x64:
4019 Nonvolatile (XMM), Volatile (upper half of YMM)
4020 XMM6:XMM15 Must be preserved as needed by callee.
4021 YMM registers must be preserved as needed by caller. (they do not need to be preserved)
4022
4023 Problem:
4024 - Jitasm saves xmm6..xmm15 when vzeroupper is used,even if only an xmm0 is used (Why?)
4025 No problem (looking at the disassembly list):
4026 - when there is no vzeroupper, then the xmm6:xmm11 is properly saved/restored in prolog/epilog but only
4027 if ymm6:ymm11 (in this example) is used. If no register is used over xmm6/ymm6 then xmm registers are not saved at all.
4028 - question: does it have any penalty when movaps is used w/o vzeroupper?
4029 The epilog generates movaps
4030 movaps xmm11,xmmword ptr [rbx-10h]
4031
4032 0000000002430000 push rbp
4033 0000000002430001 mov rbp,rsp
4034 0000000002430004 push rbx
4035 0000000002430005 lea rbx,[rsp-8]
4036 000000000243000A sub rsp,0A8h
4037 0000000002430011 movaps xmmword ptr [rbx-0A0h],xmm6
4038 0000000002430018 movaps xmmword ptr [rbx-90h],xmm7
4039 000000000243001F movaps xmmword ptr [rbx-80h],xmm8
4040 0000000002430024 movaps xmmword ptr [rbx-70h],xmm9
4041 0000000002430029 movaps xmmword ptr [rbx-60h],xmm10
4042 000000000243002E movaps xmmword ptr [rbx-50h],xmm11
4043 0000000002430033 movaps xmmword ptr [rbx-40h],xmm12
4044 0000000002430038 movaps xmmword ptr [rbx-30h],xmm13
4045 000000000243003D movaps xmmword ptr [rbx-20h],xmm14
4046 0000000002430042 movaps xmmword ptr [rbx-10h],xmm15
4047 -- end of jitasm generated prolog
4048
4049 -- PF AVX+: passing avx_epilog_ = true for codegen, vmovaps is generated instead of movaps
4050 0000000001E60010 vmovaps xmmword ptr [rbx-60h],xmm6
4051 0000000001E60015 vmovaps xmmword ptr [rbx-50h],xmm7
4052 0000000001E6001A vmovaps xmmword ptr [rbx-40h],xmm8
4053 0000000001E6001F vmovaps xmmword ptr [rbx-30h],xmm9
4054 0000000001E60024 vmovaps xmmword ptr [rbx-20h],xmm10
4055 0000000001E60029 vmovaps xmmword ptr [rbx-10h],xmm11
4056
4057 // PF comment: user's code like this:
4058 YmmReg zero;
4059 vpxor(zero, zero, zero);
4060 Reg constptr;
4061 mov(constptr, (uintptr_t)logexpconst_avx);
4062 vzeroupper();
4063 And the generated instructions:
4064 0000000002430047 vpxor ymm0,ymm0,ymm0
4065 000000000243004B mov rax,7FECCBD15C0h
4066 0000000002430055 vzeroupper
4067 Note: Don't use vzeroupper manually. When vzeroupper is issued manually, jitasm is not too generous: marks all xmm6:xmm15 registers as used
4068 and epilog and prolog will save all of them, even if none of those xmm/ymm registers are used in the code.
4069 Modded jitasm: pass avx_epilog_ = true for codegen, it will issue vzeroupper automatically (and has other benefits)
4070
4071 -- start of jitasm generated epilog (old)
4072 0000000002430058 movaps xmm15,xmmword ptr [rbx-10h]
4073 000000000243005D movaps xmm14,xmmword ptr [rbx-20h]
4074 0000000002430062 movaps xmm13,xmmword ptr [rbx-30h]
4075 0000000002430067 movaps xmm12,xmmword ptr [rbx-40h]
4076 000000000243006C movaps xmm11,xmmword ptr [rbx-50h]
4077 0000000002430071 movaps xmm10,xmmword ptr [rbx-60h]
4078 0000000002430076 movaps xmm9,xmmword ptr [rbx-70h]
4079 000000000243007B movaps xmm8,xmmword ptr [rbx-80h]
4080 0000000002430080 movaps xmm7,xmmword ptr [rbx-90h]
4081 0000000002430087 movaps xmm6,xmmword ptr [rbx-0A0h]
4082 000000000243008E add rsp,0A8h
4083 0000000002430095 pop rbx
4084 0000000002430096 pop rbp
4085 0000000002430097 ret
4086 -- end of jitasm generated epilog (old)
4087
4088 PF: modded jitasm (calling codegen with avx_epilog_ = true) generates vmovaps instead of movaps and and automatic vzeroupper before the ret instruction
4089 generated epilog example (new):
4090 0000000001E70613 vmovaps xmm11,xmmword ptr [rbx-10h]
4091 0000000001E70618 vmovaps xmm10,xmmword ptr [rbx-20h]
4092 0000000001E7061D vmovaps xmm9,xmmword ptr [rbx-30h]
4093 0000000001E70622 vmovaps xmm8,xmmword ptr [rbx-40h]
4094 0000000001E70627 vmovaps xmm7,xmmword ptr [rbx-50h]
4095 0000000001E7062C vmovaps xmm6,xmmword ptr [rbx-60h]
4096 0000000001E70631 add rsp,68h
4097 0000000001E70635 pop rdi
4098 0000000001E70636 pop rsi
4099 0000000001E70637 pop rbx
4100 0000000001E70638 pop rbp
4101 0000000001E70639 vzeroupper
4102 0000000001E7063C ret
4103
4104 */
4105 ✗ void main(Reg regptrs, Reg regoffs, Reg niter, Reg SpatialY)
4106 {
4107 ✗ YmmReg zero;
4108 ✗ vpxor(zero, zero, zero);
4109 ✗ Reg constptr;
4110 ✗ mov(constptr, (uintptr_t)logexpconst_avx);
4111
4112 ✗ L("wloop");
4113 ✗ cmp(niter, 0); // while(niter>0)
4114 ✗ je("wend");
4115 ✗ sub(niter, 1);
4116
4117 // process two sets, no partial input masking
4118 ✗ if(singleMode)
4119 processingLoop<true, false>(regptrs, zero, constptr, SpatialY);
4120 else
4121 processingLoop<false, false>(regptrs, zero, constptr, SpatialY);
4122
4123 // increase read and write pointers by 16 pixels
4124 ✗ const int EXTRA = 2; // output pointer, xcounter
4125 if constexpr(sizeof(void *) == 8) {
4126 // x64: two 8 byte pointers in an xmm
4127 ✗ int numIter = (numInputs + EXTRA + 1) / 2;
4128
4129 ✗ for (int i = 0; i < numIter; i++) {
4130 ✗ XmmReg r1, r2;
4131 ✗ vmovdqu(r1, xmmword_ptr[regptrs + 16 * i]);
4132 ✗ vmovdqu(r2, xmmword_ptr[regoffs + 16 * i]);
4133 ✗ vpaddq(r1, r1, r2); // pointers are 64 bits
4134 ✗ vmovdqu(xmmword_ptr[regptrs + 16 * i], r1);
4135 }
4136 }
4137 else {
4138 // x86: four 4 byte pointers in an xmm
4139 int numIter = (numInputs + EXTRA + 3) / 4;
4140 for (int i = 0; i < numIter; i++) {
4141 XmmReg r1, r2;
4142 vmovdqu(r1, xmmword_ptr[regptrs + 16 * i]);
4143 vmovdqu(r2, xmmword_ptr[regoffs + 16 * i]);
4144 vpaddd(r1, r1, r2); // pointers are 32 bits
4145 vmovdqu(xmmword_ptr[regptrs + 16 * i], r1);
4146 }
4147 }
4148
4149 ✗ jmp("wloop");
4150 ✗ L("wend");
4151
4152 ✗ int nrestpixels = planewidth & (singleMode ? 7 : 15);
4153 ✗ if(nrestpixels > 8) // dual process with masking
4154 processingLoop<false, true>(regptrs, zero, constptr, SpatialY);
4155 ✗ else if (nrestpixels == 8) // single process, no masking
4156 processingLoop<true, false>(regptrs, zero, constptr, SpatialY);
4157 ✗ else if (nrestpixels > 0) // single process, masking
4158 processingLoop<true, true>(regptrs, zero, constptr, SpatialY);
4159 // bug in jitasm?
4160 // on x64, when this is here, debug throws an assert, that a register save/load has an
4161 // operand size 8 bit, instead of 128 (XMM) or 256 (YMM)
4162 // vzeroupper(); // don't use it directly. Generate code with avx_epilog_=true
4163 ✗ }
4164 };
4165
4166 #endif
4167
4168
4169 /********************************************************************
4170 ***** Declare index of new filters for Avisynth's filter engine *****
4171 ********************************************************************/
4172
4173 extern const AVSFunction Exprfilter_filters[] = {
4174 { "Expr", BUILTIN_FUNC_PREFIX, "c+s+[format]s[optAvx2]b[optSingleMode]b[optSSE2]b[scale_inputs]s[clamp_float]b[clamp_float_UV]b[lut]i[optVectorC]b", Exprfilter::Create },
4175 { 0 }
4176 };
4177
4178
4179 ✗ AVSValue __cdecl Exprfilter::Create(AVSValue args, void* , IScriptEnvironment* env) {
4180
4181 ✗ std::vector<PClip> children;
4182 ✗ std::vector<std::string> expressions;
4183 int next_paramindex;
4184
4185 // one or more clips
4186 ✗ if (args[0].IsArray() && args[0][0].IsClip()) { // c+s+ case
4187 ✗ children.resize(args[0].ArraySize());
4188
4189 ✗ for (int i = 0; i < (int)children.size(); ++i) // Copy all
4190 ✗ children[i] = args[0][i].AsClip();
4191
4192 ✗ next_paramindex = 1;
4193 }
4194 ✗ else if (args[1].IsArray() && args[1][0].IsClip()) { // cc+s+ case
4195 ✗ children.resize(1 + args[1].ArraySize());
4196
4197 ✗ children[0] = args[0].AsClip(); // Copy 1st
4198 ✗ for (int i = 1; i < (int)children.size(); ++i) // Copy rest
4199 ✗ children[i] = args[1][i - 1].AsClip();
4200
4201 ✗ next_paramindex = 2;
4202 }
4203 ✗ else if (args[1].IsClip()) { //cc case
4204 ✗ children.resize(2);
4205
4206 ✗ children[0] = args[0].AsClip();
4207 ✗ children[1] = args[1].AsClip();
4208
4209 ✗ next_paramindex = 2;
4210 }
4211 ✗ else if (args[0].IsClip()) { // single clip, cs+ case
4212 ✗ children.resize(1);
4213 ✗ children[0] = args[0].AsClip();
4214
4215 ✗ next_paramindex = 1;
4216 }
4217 else {
4218 ✗ env->ThrowError("Expr: Invalid parameter type");
4219 }
4220
4221 // one or more expressions: s+
4222 ✗ if (args[next_paramindex].Defined()) {
4223 ✗ AVSValue exprarg = args[next_paramindex++];
4224 ✗ if (exprarg.IsArray()) {
4225 ✗ int nexpr = exprarg.ArraySize();
4226 ✗ expressions.resize(nexpr);
4227 ✗ for (int i = 0; i < nexpr; i++)
4228 ✗ expressions[i] = exprarg[i].AsString();
4229 }
4230 ✗ else if (exprarg.IsString()) {
4231 ✗ expressions.resize(1);
4232 ✗ expressions[0] = exprarg.AsString();
4233 }
4234 else {
4235 ✗ env->ThrowError("Expr: Invalid parameter type for expression string");
4236 }
4237 ✗ }
4238
4239 // optional named argument: format
4240 ✗ const char *newformat = nullptr;
4241 ✗ if (args[next_paramindex].Defined()) {
4242 // always string
4243 ✗ newformat = args[next_paramindex].AsString();
4244 }
4245 ✗ next_paramindex++;
4246
4247 #ifdef VS_TARGET_CPU_X86
4248 // test parameter for avx2-less mode even with avx2 available
4249 #ifdef TEST_AVX2_CODEGEN_IN_AVX
4250 bool optAvx2 = !!(env->GetCPUFlags() & CPUF_AVX);
4251 #else
4252 ✗ bool optAvx2 = !!(env->GetCPUFlags() & CPUF_AVX2);
4253 #endif
4254 ✗ bool optSSE2 = !!(env->GetCPUFlags() & CPUF_SSE2);
4255 #else
4256 bool optAvx2 = false;
4257 bool optSSE2 = false;
4258 #endif
4259
4260 ✗ if (args[next_paramindex].Defined()) {
4261 ✗ if (optAvx2) // disable only
4262 ✗ optAvx2 = args[next_paramindex].AsBool();
4263 }
4264 ✗ next_paramindex++;
4265
4266 ✗ bool optSingleMode = false;
4267 ✗ if (args[next_paramindex].Defined()) {
4268 ✗ optSingleMode = args[next_paramindex].AsBool();
4269 }
4270 ✗ next_paramindex++;
4271
4272 ✗ if (args[next_paramindex].Defined()) {
4273 ✗ if (optSSE2) // disable only
4274 ✗ optSSE2 = args[next_paramindex].AsBool();
4275 }
4276 ✗ next_paramindex++;
4277
4278 ✗ std::string scale_inputs = args[next_paramindex].Defined() ? args[next_paramindex].AsString("none") : "none";
4279 ✗ transform(scale_inputs.begin(), scale_inputs.end(), scale_inputs.begin(), ::tolower);
4280 ✗ next_paramindex++;
4281
4282 ✗ const bool clamp_float = args[next_paramindex].AsBool(false);
4283 ✗ next_paramindex++;
4284
4285 ✗ const bool clamp_float_UV = args[next_paramindex].AsBool(false);
4286 ✗ next_paramindex++;
4287
4288 // clamp_float clamp_float_uv -> clamp_float_i clamp range for Y clamp range for UV
4289 // false x 0 0..1 -0.5..+0.5
4290 // true false 1 0..1 -0.5..+0.5
4291 // true true 2 0..1 0..1
4292
4293 int clamp_float_i;
4294 ✗ if (clamp_float)
4295 ✗ clamp_float_i = clamp_float_UV ? 2 : 1;
4296 else
4297 ✗ clamp_float_i = 0;
4298
4299 ✗ const int lutmode = args[next_paramindex].AsInt(0); // 0, 1, 2
4300 ✗ next_paramindex++;
4301
4302 ✗ const bool optVectorC = args[next_paramindex].AsBool(true);
4303
4304 ✗ return new Exprfilter(children, expressions, newformat, optAvx2, optSingleMode, optSSE2, optVectorC, scale_inputs, clamp_float_i, lutmode, env);
4305
4306 ✗ }
4307
4308 // Base SIMD Processor interface
4309 class ISIMDProcessor {
4310 public:
4311 virtual void processVector(
4312 std::vector<const uint8_t*>&srcp,
4313 uint8_t*& dstp,
4314 int x, int y) = 0;
4315 ✗ virtual ~ISIMDProcessor() = default;
4316 };
4317
4318 /**
4319 * Custom aligned memory allocator for STL containers like std::vector
4320 *
4321 * This allocator ensures that the underlying data pointer returned by container.data()
4322 * is aligned to the specified alignment boundary, which is critical for:
4323 * - SIMD vector operations that require aligned memory access
4324 * - Cache-friendly data structures
4325 * - Hardware-specific memory alignment requirements
4326 *
4327 * Platform-specific implementation:
4328 * - Windows (MSVC, ClangCL): Uses _aligned_malloc/_aligned_free
4329 * - MinGW: Uses _aligned_malloc/_aligned_free
4330 * - POSIX systems (Linux, Unix, macOS): Uses aligned_alloc/free
4331 * - C++17 fallback: Uses std::aligned_alloc/free
4332 *
4333 * Usage:
4334 * std::vector<float, aligned_allocator<float, 32>> aligned_vector;
4335 */
4336 template <typename T, size_t Alignment>
4337 struct aligned_allocator {
4338 // Standard allocator typedefs
4339 typedef T value_type;
4340 typedef T* pointer;
4341 typedef const T* const_pointer;
4342 typedef T& reference;
4343 typedef const T& const_reference;
4344 typedef std::size_t size_type;
4345 typedef std::ptrdiff_t difference_type;
4346 // Rebind allocator to type U
4347 template <typename U>
4348 struct rebind {
4349 typedef aligned_allocator<U, Alignment> other;
4350 };
4351 ✗ aligned_allocator() noexcept {}
4352 template <typename U>
4353 aligned_allocator(const aligned_allocator<U, Alignment>&) noexcept {}
4354 ✗ T* allocate(std::size_t n) {
4355 #if defined(_MSC_VER) || (defined(__clang__) && defined(_MSC_VER))
4356 // Both MSVC and ClangCL should use _aligned_malloc
4357 void* ptr = _aligned_malloc(n * sizeof(T), Alignment);
4358 if (!ptr) throw std::bad_alloc();
4359 #elif defined(__MINGW32__) || defined(__MINGW64__)
4360 // MinGW/MinGW-w64 specific
4361 void* ptr = _aligned_malloc(n * sizeof(T), Alignment);
4362 if (!ptr) throw std::bad_alloc();
4363 #elif defined(__INTEL_COMPILER) || defined(__INTEL_LLVM_COMPILER) || defined(__APPLE__) || (defined(__GNUC__) && !defined(_WIN32) && !defined(__CYGWIN__))
4364 // POSIX-compliant systems: Intel compilers, macOS, GCC on non-Windows
4365 // aligned_alloc requires size to be a multiple of alignment
4366 ✗ size_t size = n * sizeof(T);
4367 ✗ if (size % Alignment != 0) {
4368 ✗ size = (size / Alignment + 1) * Alignment;
4369 }
4370 ✗ void* ptr = aligned_alloc(Alignment, size);
4371 ✗ if (!ptr) throw std::bad_alloc();
4372 #else
4373 // Generic fallback for C++17 and later
4374 // std::aligned_alloc requires size to be a multiple of alignment
4375 size_t size = n * sizeof(T);
4376 if (size % Alignment != 0) {
4377 size = (size / Alignment + 1) * Alignment;
4378 }
4379 void* ptr = std::aligned_alloc(Alignment, size);
4380 if (!ptr) throw std::bad_alloc();
4381 #endif
4382 ✗ return static_cast<T*>(ptr);
4383 }
4384 ✗ void deallocate(T* p, std::size_t) noexcept {
4385 #if defined(_MSC_VER) || defined(__MINGW32__) || defined(__MINGW64__) || (defined(__clang__) && defined(_MSC_VER))
4386 _aligned_free(p);
4387 #else
4388 ✗ free(p);
4389 #endif
4390 ✗ }
4391 };
4392
4393 // convenient type def
4394 using aligned_float_vector = std::vector<float, aligned_allocator<float, 32>>;
4395
4396 template<int VectorSize>
4397 class SIMDProcessor : public ISIMDProcessor {
4398 int w, h;
4399 size_t maxStackSize;
4400
4401 // Define the aligned stack with custom allocator
4402 std::vector<aligned_float_vector> stack;
4403
4404 aligned_float_vector& variable_area;
4405 std::vector<float>& internal_vars;
4406 std::vector<int>& src_stride;
4407 std::vector<const uint8_t*>& srcp_orig;
4408 const ExprOp* vops;
4409 public:
4410 ✗ SIMDProcessor(int _w, int _h, size_t _maxStackSize,
4411 aligned_float_vector& var_area,
4412 std::vector<float>& int_vars, std::vector<int>& src_str, std::vector<const uint8_t*>& src_orig, const ExprOp* vops)
4413 ✗ : w(_w), h(_h), maxStackSize(_maxStackSize),
4414 ✗ stack(_maxStackSize, aligned_float_vector(VectorSize)),
4415 ✗ variable_area(var_area), internal_vars(int_vars), src_stride(src_str), srcp_orig(src_orig), vops(vops) {
4416 ✗ }
4417
4418 int stackIndex = 0;
4419 alignas(32) float stacktop[VectorSize] = {};
4420
4421 // Broadcast a single value across the vector
4422 ✗ inline void push_and_broadcast(float val) {
4423 ✗ auto& current_stack = stack[stackIndex];
4424
4425 // First loop: copy stacktop to stack - help vectorization pattern
4426 ✗ for (int i = 0; i < VectorSize; ++i)
4427 ✗ current_stack[i] = stacktop[i];
4428 ✗ for (int i = 0; i < VectorSize; ++i)
4429 ✗ stacktop[i] = val;
4430 ✗ stackIndex++;
4431 ✗ }
4432
4433 ✗ inline void broadcast(float val) {
4434 ✗ for (int i = 0; i < VectorSize; ++i)
4435 ✗ stacktop[i] = val;
4436 ✗ }
4437
4438 // Load spatial X coordinate
4439 ✗ inline void loadSpatialX(int x) {
4440 // Push current stacktop and fills x, x+1, x+2, x+3, ...
4441 ✗ auto& current_stack = stack[stackIndex];
4442 ✗ for (int i = 0; i < VectorSize; ++i)
4443 ✗ current_stack[i] = stacktop[i];
4444 ✗ for (int i = 0; i < VectorSize; ++i)
4445 ✗ stacktop[i] = static_cast<float>(x + i);
4446 ✗ stackIndex++;
4447 ✗ }
4448
4449 // Load vector from source from x
4450 template<typename T>
4451 ✗ inline void loadSource(const T* src, int x) {
4452 ✗ auto& current_stack = stack[stackIndex];
4453 ✗ for (int i = 0; i < VectorSize; ++i)
4454 ✗ current_stack[i] = stacktop[i];
4455 ✗ for (int i = 0; i < VectorSize; ++i)
4456 ✗ stacktop[i] = static_cast<float>(src[x + i]);
4457 ✗ stackIndex++;
4458 ✗ }
4459
4460 // Load from source with relative offset
4461 template<typename T>
4462 ✗ void loadRelSource(const T* src, int x, int dx, int dy, int width, int height, int stride) {
4463 ✗ auto& current_stack = stack[stackIndex];
4464 ✗ for (int i = 0; i < VectorSize; ++i)
4465 ✗ current_stack[i] = stacktop[i];
4466 // At edges: repeat, no mirror
4467 ✗ int newY = std::max(0, std::min(dy, height - 1));
4468 ✗ for (int i = 0; i < VectorSize; ++i) {
4469 ✗ int newX = std::max(0, std::min(x + dx + i, width - 1));
4470 ✗ stacktop[i] = static_cast<float>(reinterpret_cast<const T*>((uint8_t*)&src[newY * stride])[newX]);
4471 }
4472 ✗ stackIndex++;
4473 ✗ }
4474
4475 // Load from user variable area
4476 ✗ inline void loadVar(int index) {
4477 ✗ auto& current_stack = stack[stackIndex];
4478 ✗ for (int i = 0; i < VectorSize; ++i)
4479 ✗ current_stack[i] = stacktop[i];
4480 // Push current stacktop and broadcast x
4481 ✗ for (int i = 0; i < VectorSize; ++i)
4482 ✗ stacktop[i] = variable_area[index * MAX_C_VECT + i];
4483 ✗ stackIndex++;
4484 ✗ }
4485
4486 // Vectorized duplicate
4487 ✗ inline void dup(int offset) {
4488 ✗ auto& current_stack = stack[stackIndex];
4489 ✗ for (int i = 0; i < VectorSize; ++i)
4490 ✗ current_stack[i] = stacktop[i];
4491 ✗ auto& prev = stack[stackIndex - offset];
4492 ✗ for (int i = 0; i < VectorSize; ++i)
4493 ✗ stacktop[i] = prev[i];
4494 ✗ stackIndex++;
4495 ✗ }
4496
4497 // Vectorized swap
4498 ✗ inline void swap(int offset) {
4499 ✗ auto& prev = stack[stackIndex - offset];
4500 ✗ for (int i = 0; i < VectorSize; ++i)
4501 ✗ std::swap(stacktop[i], prev[i]);
4502 ✗ }
4503
4504 // Vectorized addition
4505 ✗ inline void add() {
4506 ✗ stackIndex--;
4507 ✗ auto& prev = stack[stackIndex];
4508 ✗ for (int i = 0; i < VectorSize; ++i)
4509 ✗ stacktop[i] += prev[i];
4510 ✗ }
4511
4512 // Vectorized subtract
4513 ✗ inline void sub() {
4514 ✗ stackIndex--;
4515 ✗ auto& prev = stack[stackIndex];
4516 ✗ for (int i = 0; i < VectorSize; ++i)
4517 ✗ stacktop[i] = prev[i] - stacktop[i];
4518 ✗ }
4519
4520 // Vectorized multiplication
4521 ✗ inline void multiply() {
4522 ✗ stackIndex--;
4523 ✗ auto& prev = stack[stackIndex];
4524 ✗ for (int i = 0; i < VectorSize; ++i)
4525 ✗ stacktop[i] *= prev[i];
4526 ✗ }
4527
4528 ✗ inline void divide() {
4529 ✗ stackIndex--;
4530 ✗ auto& prev = stack[stackIndex];
4531 ✗ for (int i = 0; i < VectorSize; ++i)
4532 ✗ stacktop[i] = prev[i] / stacktop[i];
4533 ✗ }
4534
4535 ✗ void fmod() {
4536 ✗ stackIndex--;
4537 ✗ auto& prev = stack[stackIndex];
4538 ✗ for (int i = 0; i < VectorSize; ++i)
4539 ✗ stacktop[i] = std::fmod(prev[i], stacktop[i]);
4540 ✗ }
4541
4542 ✗ inline void max() {
4543 ✗ stackIndex--;
4544 ✗ auto& prev = stack[stackIndex];
4545 ✗ for (int i = 0; i < VectorSize; ++i)
4546 ✗ stacktop[i] = std::max(prev[i], stacktop[i]);
4547 ✗ }
4548
4549 ✗ inline void min() {
4550 ✗ stackIndex--;
4551 ✗ auto& prev = stack[stackIndex];
4552 ✗ for (int i = 0; i < VectorSize; ++i)
4553 ✗ stacktop[i] = std::min(prev[i], stacktop[i]);
4554 ✗ }
4555
4556 ✗ void exp() {
4557 ✗ for (int i = 0; i < VectorSize; ++i)
4558 ✗ stacktop[i] = std::exp(stacktop[i]);
4559 ✗ }
4560
4561 ✗ void log() {
4562 ✗ for (int i = 0; i < VectorSize; ++i)
4563 ✗ stacktop[i] = std::log(stacktop[i]);
4564 ✗ }
4565
4566 ✗ void pow() {
4567 ✗ stackIndex--;
4568 ✗ auto& prev = stack[stackIndex];
4569 ✗ for (int i = 0; i < VectorSize; ++i)
4570 ✗ stacktop[i] = std::pow(prev[i], stacktop[i]);
4571 ✗ }
4572
4573 // Vectorized clip
4574 ✗ inline void clip() {
4575 ✗ stackIndex -= 2;
4576 ✗ auto& prev = stack[stackIndex];
4577 ✗ auto& prev1 = stack[stackIndex + 1];
4578 ✗ for (int i = 0; i < VectorSize; ++i)
4579 ✗ stacktop[i] = std::max(std::min(prev[i], stacktop[i]), prev1[i]);
4580 ✗ }
4581
4582 // Vectorized round
4583 ✗ inline void round() {
4584 ✗ for (int i = 0; i < VectorSize; ++i)
4585 ✗ stacktop[i] = std::round(stacktop[i]);
4586 ✗ }
4587
4588 // Vectorized floor
4589 ✗ inline void floor() {
4590 ✗ for (int i = 0; i < VectorSize; ++i)
4591 ✗ stacktop[i] = std::floor(stacktop[i]);
4592 ✗ }
4593
4594 // Vectorized ceil
4595 ✗ void ceil() {
4596 ✗ for (int i = 0; i < VectorSize; ++i)
4597 ✗ stacktop[i] = std::ceil(stacktop[i]);
4598 ✗ }
4599
4600 // Vectorized trunc
4601 ✗ inline void trunc() {
4602 ✗ for (int i = 0; i < VectorSize; ++i)
4603 ✗ stacktop[i] = std::trunc(stacktop[i]);
4604 ✗ }
4605
4606 // Vectorized sqrt
4607 ✗ void sqrt() {
4608 ✗ for (int i = 0; i < VectorSize; ++i)
4609 ✗ stacktop[i] = std::sqrt(stacktop[i]);
4610 ✗ }
4611
4612 // Vectorized abs
4613 ✗ inline void abs() {
4614 ✗ for (int i = 0; i < VectorSize; ++i)
4615 ✗ stacktop[i] = std::abs(stacktop[i]);
4616 ✗ }
4617
4618 // Vectorized sgn
4619 ✗ inline void sgn() {
4620 ✗ for (int i = 0; i < VectorSize; ++i)
4621 ✗ stacktop[i] = stacktop[i] < 0 ? -1.0f : stacktop[i] > 0 ? 1.0f : 0.0f;
4622 ✗ }
4623
4624 // Vectorized sin
4625 ✗ void sin() {
4626 ✗ for (int i = 0; i < VectorSize; ++i)
4627 ✗ stacktop[i] = std::sin(stacktop[i]);
4628 ✗ }
4629
4630 // Vectorized cos
4631 ✗ void cos() {
4632 ✗ for (int i = 0; i < VectorSize; ++i)
4633 ✗ stacktop[i] = std::cos(stacktop[i]);
4634 ✗ }
4635
4636 // Vectorized tan
4637 ✗ void tan() {
4638 ✗ for (int i = 0; i < VectorSize; ++i)
4639 ✗ stacktop[i] = std::tan(stacktop[i]);
4640 // reference of JITAsm code test: fast_tanf(stacktop[i]);
4641 ✗ }
4642
4643 // Vectorized asin
4644 ✗ void asin() {
4645 ✗ for (int i = 0; i < VectorSize; ++i)
4646 ✗ stacktop[i] = std::asin(stacktop[i]);
4647 ✗ }
4648
4649 // Vectorized acos
4650 ✗ void acos() {
4651 ✗ for (int i = 0; i < VectorSize; ++i)
4652 ✗ stacktop[i] = std::acos(stacktop[i]);
4653 ✗ }
4654
4655 // Vectorized atan
4656 ✗ void atan() {
4657 ✗ for (int i = 0; i < VectorSize; ++i)
4658 ✗ stacktop[i] = std::atan(stacktop[i]);
4659 ✗ }
4660
4661 // Vectorized atan2
4662 ✗ void atan2() {
4663 ✗ stackIndex--;
4664 ✗ auto& prev = stack[stackIndex];
4665 ✗ for (int i = 0; i < VectorSize; ++i)
4666 ✗ stacktop[i] = std::atan2(prev[i], stacktop[i]); // y, x -> -Pi..+Pi
4667 ✗ }
4668
4669 // Vectorized greater than
4670 ✗ inline void gt() {
4671 ✗ stackIndex--;
4672 ✗ auto& prev = stack[stackIndex];
4673 ✗ for (int i = 0; i < VectorSize; ++i)
4674 ✗ stacktop[i] = (prev[i] > stacktop[i]) ? 1.0f : 0.0f;
4675 ✗ }
4676
4677 // Vectorized less than
4678 ✗ inline void lt() {
4679 ✗ stackIndex--;
4680 ✗ auto& prev = stack[stackIndex];
4681 ✗ for (int i = 0; i < VectorSize; ++i)
4682 ✗ stacktop[i] = (prev[i] < stacktop[i]) ? 1.0f : 0.0f;
4683 ✗ }
4684
4685 // Vectorized equal
4686 ✗ inline void eq() {
4687 ✗ stackIndex--;
4688 ✗ auto& prev = stack[stackIndex];
4689 ✗ for (int i = 0; i < VectorSize; ++i)
4690 ✗ stacktop[i] = (prev[i] == stacktop[i]) ? 1.0f : 0.0f; // consider with not 100% match, use epsilon
4691 ✗ }
4692
4693 // Vectorized not equal
4694 ✗ inline void notEq() {
4695 ✗ stackIndex--;
4696 ✗ auto& prev = stack[stackIndex];
4697 ✗ for (int i = 0; i < VectorSize; ++i)
4698 ✗ stacktop[i] = (prev[i] != stacktop[i]) ? 1.0f : 0.0f; // consider with not 100% match, use epsilon
4699 ✗ }
4700
4701 // Vectorized less than or equal
4702 ✗ inline void le() {
4703 ✗ stackIndex--;
4704 ✗ auto& prev = stack[stackIndex];
4705 ✗ for (int i = 0; i < VectorSize; ++i)
4706 ✗ stacktop[i] = (prev[i] <= stacktop[i]) ? 1.0f : 0.0f;
4707 ✗ }
4708
4709 // Vectorized greater than or equal
4710 ✗ inline void ge() {
4711 ✗ stackIndex--;
4712 ✗ auto& prev = stack[stackIndex];
4713 ✗ for (int i = 0; i < VectorSize; ++i)
4714 ✗ stacktop[i] = (prev[i] >= stacktop[i]) ? 1.0f : 0.0f;
4715 ✗ }
4716
4717 // Vectorized ternary
4718 ✗ inline void ternary() {
4719 ✗ stackIndex -= 2;
4720 ✗ auto& prev = stack[stackIndex];
4721 ✗ auto& prev1 = stack[stackIndex + 1];
4722 ✗ for (int i = 0; i < VectorSize; ++i)
4723 ✗ stacktop[i] = (prev[i] > 0) ? prev1[i] : stacktop[i];
4724 ✗ }
4725
4726 // Vectorized logical AND
4727 ✗ inline void logicalAnd() {
4728 ✗ stackIndex--;
4729 ✗ auto& prev = stack[stackIndex];
4730 ✗ for (int i = 0; i < VectorSize; ++i)
4731 ✗ stacktop[i] = (stacktop[i] > 0 && prev[i] > 0) ? 1.0f : 0.0f;
4732 ✗ }
4733
4734 // Vectorized logical OR
4735 ✗ inline void logicalOr() {
4736 ✗ stackIndex--;
4737 ✗ auto& prev = stack[stackIndex];
4738 ✗ for (int i = 0; i < VectorSize; ++i)
4739 ✗ stacktop[i] = (stacktop[i] > 0 || prev[i] > 0) ? 1.0f : 0.0f;
4740 ✗ }
4741
4742 // Vectorized logical XOR
4743 ✗ inline void logicalXor() {
4744 ✗ stackIndex--;
4745 ✗ auto& prev = stack[stackIndex];
4746 ✗ for (int i = 0; i < VectorSize; ++i)
4747 ✗ stacktop[i] = ((stacktop[i] > 0) != (prev[i] > 0)) ? 1.0f : 0.0f;
4748 ✗ }
4749
4750 // Vectorized logical NOT
4751 ✗ inline void logicalNot() {
4752 ✗ for (int i = 0; i < VectorSize; ++i)
4753 ✗ stacktop[i] = (stacktop[i] > 0) ? 0.0f : 1.0f;
4754 ✗ }
4755
4756 // Vectorized negation
4757 ✗ inline void negSign() {
4758 ✗ for (int i = 0; i < VectorSize; ++i)
4759 ✗ stacktop[i] = -stacktop[i];
4760 ✗ }
4761
4762 // Templatized store function
4763 template<typename T, int MaxValue>
4764 ✗ inline void store(T* dst, int x) {
4765 ✗ for (int i = 0; i < VectorSize; ++i)
4766 ✗ dst[x + i] = static_cast<T>(std::max(0.0f, std::min(stacktop[i], static_cast<float>(MaxValue))) + 0.5f);
4767 ✗ }
4768
4769 // Specialized store function for float
4770 ✗ inline void store(float* dst, int x) {
4771 ✗ for (int i = 0; i < VectorSize; ++i)
4772 ✗ dst[x + i] = stacktop[i];
4773 ✗ }
4774
4775 // Store variable
4776 ✗ inline void storeVar(int index) {
4777 ✗ for (int i = 0; i < VectorSize; ++i)
4778 ✗ variable_area[index * MAX_C_VECT + i] = stacktop[i];
4779 ✗ }
4780
4781 // Store variable and drop one element from the stack
4782 ✗ inline void storeVarAndDrop1(int index) {
4783 ✗ for (int i = 0; i < VectorSize; ++i)
4784 ✗ variable_area[index * MAX_C_VECT + i] = stacktop[i];
4785 ✗ stackIndex--;
4786 ✗ if (stackIndex >= 0) {
4787 ✗ auto& prev = stack[stackIndex];
4788 ✗ for (int i = 0; i < VectorSize; ++i)
4789 ✗ stacktop[i] = prev[i];
4790 }
4791 ✗ }
4792
4793 // Process a sequence of operations
4794 ✗ void processVector(std::vector<const uint8_t*>& srcp, uint8_t*& dstp, int x, int y) override {
4795
4796 // Reset stack
4797 ✗ stackIndex = 0;
4798 ✗ broadcast(0); // stacktop = 0
4799
4800 ✗ const ExprOp* vops_current = vops; // reset instruction pointer
4801
4802 while (true) {
4803 // Process instruction sequence
4804 ✗ switch (vops_current->op) {
4805 ✗ case opLoadSrc8:
4806 ✗ loadSource<uint8_t>(reinterpret_cast<const uint8_t*>(srcp[vops_current->e.ival]), x);
4807 ✗ break;
4808 ✗ case opLoadSrc16:
4809 ✗ loadSource<uint16_t>(reinterpret_cast<const uint16_t*>(srcp[vops_current->e.ival]), x);
4810 ✗ break;
4811 ✗ case opLoadSrcF32:
4812 ✗ loadSource<float>(reinterpret_cast<const float*>(srcp[vops_current->e.ival]), x);
4813 ✗ break;
4814 ✗ case opLoadRelSrc8:
4815 ✗ loadRelSource<uint8_t>(reinterpret_cast<const uint8_t*>(srcp[vops_current->e.ival]), x, vops_current->dx, vops_current->dy, w, h, src_stride[vops_current->e.ival]);
4816 ✗ break;
4817 ✗ case opLoadRelSrc16:
4818 ✗ loadRelSource<uint16_t>(reinterpret_cast<const uint16_t*>(srcp[vops_current->e.ival]), x, vops_current->dx, vops_current->dy, w, h, src_stride[vops_current->e.ival]);
4819 ✗ break;
4820 ✗ case opLoadRelSrcF32:
4821 ✗ loadRelSource<float>(reinterpret_cast<const float*>(srcp[vops_current->e.ival]), x, vops_current->dx, vops_current->dy, w, h, src_stride[vops_current->e.ival]);
4822 ✗ break;
4823 ✗ case opLoadConst:
4824 ✗ push_and_broadcast(vops_current->e.fval);
4825 ✗ break;
4826 ✗ case opLoadSpatialX:
4827 ✗ loadSpatialX(x);
4828 ✗ break;
4829 ✗ case opLoadSpatialY:
4830 ✗ push_and_broadcast(static_cast<float>(y));
4831 ✗ break;
4832 ✗ case opLoadInternalVar:
4833 ✗ push_and_broadcast(internal_vars[vops_current->e.ival]);
4834 ✗ break;
4835 ✗ case opStore8:
4836 ✗ store<uint8_t, 255>(reinterpret_cast<uint8_t*>(dstp), x);
4837 ✗ goto loopend;
4838 ✗ case opStore10:
4839 ✗ store<uint16_t, 1023>(reinterpret_cast<uint16_t*>(dstp), x);
4840 ✗ goto loopend;
4841 ✗ case opStore12:
4842 ✗ store<uint16_t, 4095>(reinterpret_cast<uint16_t*>(dstp), x);
4843 ✗ goto loopend;
4844 ✗ case opStore14:
4845 ✗ store<uint16_t, 16383>(reinterpret_cast<uint16_t*>(dstp), x);
4846 ✗ goto loopend;
4847 ✗ case opStore16:
4848 ✗ store<uint16_t, 65535>(reinterpret_cast<uint16_t*>(dstp), x);
4849 ✗ goto loopend;
4850 ✗ case opStoreF32:
4851 ✗ store(reinterpret_cast<float*>(dstp), x);
4852 ✗ goto loopend;
4853
4854 ✗ case opDup: dup(vops_current->e.ival); break;
4855 ✗ case opSwap: swap(vops_current->e.ival); break;
4856 ✗ case opAdd: add(); break;
4857 ✗ case opSub: sub(); break;
4858 ✗ case opMul: multiply(); break;
4859 ✗ case opDiv: divide(); break;
4860 ✗ case opMax: max(); break;
4861 ✗ case opMin: min(); break;
4862 ✗ case opSqrt: sqrt(); break;
4863 ✗ case opAbs: abs(); break;
4864 ✗ case opSgn: sgn(); break;
4865 ✗ case opFmod: fmod(); break;
4866 ✗ case opGt: gt(); break;
4867 ✗ case opLt: lt(); break;
4868 ✗ case opEq: eq(); break;
4869 ✗ case opNotEq: notEq(); break;
4870 ✗ case opLE: le(); break;
4871 ✗ case opGE: ge(); break;
4872 ✗ case opTernary: ternary(); break;
4873 ✗ case opAnd: logicalAnd(); break;
4874 ✗ case opOr: logicalOr(); break;
4875 ✗ case opXor: logicalXor(); break;
4876 ✗ case opNeg: logicalNot(); break;
4877 ✗ case opNegSign: negSign(); break;
4878 ✗ case opExp: exp(); break;
4879 ✗ case opLog: log(); break;
4880 ✗ case opPow: pow(); break;
4881 ✗ case opSin: sin(); break;
4882 ✗ case opCos: cos(); break;
4883 ✗ case opTan: tan(); break;
4884 ✗ case opAsin: asin(); break;
4885 ✗ case opAcos: acos(); break;
4886 ✗ case opAtan: atan(); break;
4887 ✗ case opAtan2: atan2(); break;
4888 ✗ case opClip: clip(); break;
4889 ✗ case opRound: round(); break;
4890 ✗ case opFloor: floor(); break;
4891 ✗ case opCeil: ceil(); break;
4892 ✗ case opTrunc: trunc(); break;
4893
4894 ✗ case opStoreVar:
4895 ✗ storeVar(vops_current->e.ival);
4896 ✗ break;
4897 ✗ case opLoadVar:
4898 ✗ loadVar(vops_current->e.ival);
4899 ✗ break;
4900 ✗ case opLoadFramePropVar:
4901 ✗ push_and_broadcast(internal_vars[INTERNAL_VAR_FRAMEPROP_VARIABLES_START + vops_current->e.ival]);
4902 ✗ break;
4903 ✗ case opStoreVarAndDrop1:
4904 ✗ storeVarAndDrop1(vops_current->e.ival);
4905 ✗ break;
4906 }
4907 ✗ vops_current++; // next opcode
4908 }
4909 // store makes the sequence end, wherever it was
4910 ✗ loopend:;
4911 ✗ }
4912 };
4913
4914
4915 // Factory to create appropriate SIMD processors
4916 class SIMDProcessorFactory {
4917 public:
4918 template<int MaxVectorSize>
4919 ✗ static std::unique_ptr<ISIMDProcessor> createProcessor(
4920 int w, int h, size_t maxStackSize,
4921 aligned_float_vector& var_area,
4922 std::vector<float>& int_vars, std::vector<int>& src_str, std::vector<const uint8_t*>& src_orig, const ExprOp* vops) {
4923 if constexpr (MaxVectorSize >= 16) {
4924 ✗ return std::make_unique<SIMDProcessor<16>>(w, h, maxStackSize, var_area, int_vars, src_str, src_orig, vops);
4925 }
4926 if constexpr (MaxVectorSize >= 8) {
4927 ✗ return std::make_unique<SIMDProcessor<8>>(w, h, maxStackSize, var_area, int_vars, src_str, src_orig, vops);
4928 }
4929 else if constexpr (MaxVectorSize >= 4) {
4930 ✗ return std::make_unique<SIMDProcessor<4>>(w, h, maxStackSize, var_area, int_vars, src_str, src_orig, vops);
4931 }
4932 else {
4933 ✗ return std::make_unique<SIMDProcessor<1>>(w, h, maxStackSize, var_area, int_vars, src_str, src_orig, vops);
4934 }
4935 }
4936 };
4937
4938
4939 template<int MaxVectorSize>
4940 ✗ void processFrameWithDynamicVectors(int plane, int w, int h, int pixels_per_iter, float framecount, float relative_time, int numInputs,
4941 uint8_t* &dstp, int dst_stride,
4942 std::vector<const uint8_t*>& srcp, std::vector<int>& src_stride, std::vector<intptr_t>& ptroffsets, std::vector<const uint8_t*>& srcp_orig, ExprData& d) {
4943
4944 ✗ const ExprOp* vops = d.ops[plane].data();
4945 ✗ aligned_float_vector variable_area(MAX_USER_VARIABLES * MAX_C_VECT); // for C, place for expr variables (each is a vector)
4946
4947 ✗ std::vector<float> internal_vars(INTERNAL_VARIABLES + MAX_FRAMEPROP_VARIABLES);
4948 // frame dependent internal variables
4949 ✗ internal_vars[INTERNAL_VAR_CURRENT_FRAME] = (float)framecount;
4950 ✗ internal_vars[INTERNAL_VAR_RELTIME] = (float)relative_time;
4951 // followed by dynamic frame properties
4952 ✗ for (auto& framePropToRead : d.frameprops[plane]) {
4953 ✗ int whereToPut = framePropToRead.var_index;
4954 ✗ internal_vars[INTERNAL_VAR_FRAMEPROP_VARIABLES_START + whereToPut] = framePropToRead.value;
4955 };
4956
4957 // Create processors dynamically based on MaxVectorSize
4958 ✗ std::unique_ptr<ISIMDProcessor> processor16 = MaxVectorSize >= 16 ?
4959 SIMDProcessorFactory::createProcessor<16>(w, h, d.maxStackSize, variable_area, internal_vars, src_stride, srcp_orig, vops) : nullptr;
4960 ✗ std::unique_ptr<ISIMDProcessor> processor8 = MaxVectorSize >= 8 ?
4961 SIMDProcessorFactory::createProcessor<8>(w, h, d.maxStackSize, variable_area, internal_vars, src_stride, srcp_orig, vops) : nullptr;
4962 ✗ std::unique_ptr<ISIMDProcessor> processor4 = MaxVectorSize >= 4 ?
4963 SIMDProcessorFactory::createProcessor<4>(w, h, d.maxStackSize, variable_area, internal_vars, src_stride, srcp_orig, vops) : nullptr;
4964 ✗ std::unique_ptr<ISIMDProcessor> processor1 = SIMDProcessorFactory::createProcessor<1>(w, h, d.maxStackSize, variable_area, internal_vars, src_stride, srcp_orig, vops);
4965
4966 ✗ for (int y = 0; y < h; y++) {
4967 ✗ int x = 0;
4968
4969 // Conditionally process larger vector sizes
4970 ✗ if (MaxVectorSize >= 16 && processor16) {
4971 ✗ for (; x + 16 <= w; x += 16) {
4972 ✗ processor16->processVector(srcp, dstp, x, y);
4973 }
4974 }
4975
4976 ✗ if (MaxVectorSize >= 8 && processor8) {
4977 ✗ for (; x + 8 <= w; x += 8) {
4978 ✗ processor8->processVector(srcp, dstp, x, y);
4979 }
4980 }
4981
4982 ✗ if (MaxVectorSize >= 4 && processor4) {
4983 ✗ for (; x + 4 <= w; x += 4) {
4984 ✗ processor4->processVector(srcp, dstp, x, y);
4985 }
4986 }
4987
4988 // Always process remaining pixels
4989 ✗ for (; x < w; x++) {
4990 ✗ processor1->processVector(srcp, dstp, x, y);
4991 }
4992
4993 // Update destination and source pointers for next row
4994 ✗ dstp += dst_stride;
4995 ✗ if (d.lutmode == 0) {
4996 ✗ for (int i = 0; i < numInputs; i++)
4997 ✗ srcp[i] += src_stride[i];
4998 }
4999 }
5000 ✗ }
5001
5002 ✗ void Exprfilter::processFrame(int plane, int w, int h, int pixels_per_iter, float framecount, float relative_time, int numInputs,
5003 uint8_t*& dstp, int dst_stride,
5004 std::vector<const uint8_t*>& srcp, std::vector<int>& src_stride, std::vector<intptr_t>& ptroffsets, std::vector<const uint8_t*>& srcp_orig)
5005 {
5006 #ifdef VS_TARGET_CPU_X86
5007 ✗ if (optSSE2 && d.planeOptSSE2[plane]) {
5008
5009 ✗ int nfulliterations = w / pixels_per_iter;
5010
5011 ✗ ExprData::ProcessLineProc proc = d.proc[plane];
5012
5013 alignas(32) intptr_t rwptrs[RWPTR_SIZE]; // should work, gcc 8.3 gives false warning
5014
5015 ✗ *reinterpret_cast<float*>(&rwptrs[RWPTR_START_OF_INTERNAL_VARIABLES + INTERNAL_VAR_CURRENT_FRAME]) = (float)framecount;
5016 ✗ *reinterpret_cast<float*>(&rwptrs[RWPTR_START_OF_INTERNAL_VARIABLES + INTERNAL_VAR_RELTIME]) = (float)relative_time;
5017 // refresh frame properties
5018 ✗ for (auto& framePropToRead : d.frameprops[plane]) {
5019 ✗ int whereToPut = framePropToRead.var_index;
5020 ✗ *reinterpret_cast<float*>(&rwptrs[RWPTR_START_OF_INTERNAL_FRAMEPROP_VARIABLES + whereToPut]) = framePropToRead.value;
5021 };
5022 ✗ for (int y = 0; y < h; y++) {
5023 ✗ rwptrs[RWPTR_START_OF_OUTPUT] = reinterpret_cast<intptr_t>(dstp + dst_stride * y);
5024 ✗ rwptrs[RWPTR_START_OF_XCOUNTER] = 0; // xcounter internal variable
5025 ✗ for (int i = 0; i < numInputs; i++) {
5026 ✗ rwptrs[i + RWPTR_START_OF_INPUTS] = reinterpret_cast<intptr_t>(srcp[i] + src_stride[i] * y); // input pointers 1..Nth
5027 ✗ rwptrs[i + RWPTR_START_OF_STRIDES] = static_cast<intptr_t>(src_stride[i]);
5028 }
5029 // a single line at a time
5030 ✗ proc(rwptrs, ptroffsets.data(), nfulliterations, y); // parameters are put directly in registers
5031 }
5032 ✗ }
5033 else
5034 #endif // VS_TARGET_CPU_X86
5035 ✗ if (optVectorC) {
5036 // SIMD factory, vector friendly C version 16 then 8, 4, 1 floats at a time
5037 // Even if the compiler does not vectorize, we have less overhead during the opcode flow processing
5038 // As of 2025: original:1-2.5fps, new MAX_C_VECT=16: MSVC~6fps MSVC AVX2:~6fps, LLVM-14,7fps, LLVM AVX2-19fps
5039
5040 ✗ processFrameWithDynamicVectors<MAX_C_VECT>(
5041 plane, w, h, pixels_per_iter, framecount, relative_time, numInputs,
5042 dstp, dst_stride,
5043 ✗ srcp, src_stride, ptroffsets, srcp_orig, d);
5044 }
5045 else
5046 {
5047 // C version, single pixel/loop reference
5048 ✗ std::vector<float> stackVector(d.maxStackSize);
5049
5050 ✗ const ExprOp* vops = d.ops[plane].data();
5051 ✗ float* stack = stackVector.data();
5052 ✗ float stacktop = 0;
5053
5054 ✗ std::vector<float> variable_area(MAX_USER_VARIABLES); // for C, place for expr variables A..Z
5055 ✗ std::vector<float> internal_vars(INTERNAL_VARIABLES + MAX_FRAMEPROP_VARIABLES);
5056 ✗ internal_vars[INTERNAL_VAR_CURRENT_FRAME] = (float)framecount;
5057 ✗ internal_vars[INTERNAL_VAR_RELTIME] = (float)relative_time;
5058 // followed by dynamic frame properties
5059 ✗ for (auto& framePropToRead : d.frameprops[plane]) {
5060 ✗ int whereToPut = framePropToRead.var_index;
5061 ✗ internal_vars[INTERNAL_VAR_FRAMEPROP_VARIABLES_START + whereToPut] = framePropToRead.value;
5062 };
5063
5064 ✗ for (int y = 0; y < h; y++) {
5065 ✗ for (int x = 0; x < w; x++) {
5066 ✗ int si = 0;
5067 ✗ int i = -1;
5068 while (true) {
5069 ✗ i++;
5070 ✗ switch (vops[i].op) {
5071 ✗ case opLoadSpatialX:
5072 ✗ stack[si] = stacktop;
5073 ✗ stacktop = (float)x;
5074 ✗ ++si;
5075 ✗ break;
5076 ✗ case opLoadSpatialY:
5077 ✗ stack[si] = stacktop;
5078 ✗ stacktop = (float)y;
5079 ✗ ++si;
5080 ✗ break;
5081 ✗ case opLoadInternalVar:
5082 ✗ stack[si] = stacktop;
5083 ✗ stacktop = internal_vars[vops[i].e.ival];
5084 ✗ ++si;
5085 ✗ break;
5086 ✗ case opLoadFramePropVar:
5087 ✗ stack[si] = stacktop;
5088 ✗ stacktop = internal_vars[INTERNAL_VAR_FRAMEPROP_VARIABLES_START + vops[i].e.ival];
5089 ✗ ++si;
5090 ✗ break;
5091 ✗ case opLoadSrc8:
5092 ✗ stack[si] = stacktop;
5093 ✗ stacktop = srcp[vops[i].e.ival][x];
5094 ✗ ++si;
5095 ✗ break;
5096 ✗ case opLoadSrc16:
5097 ✗ stack[si] = stacktop;
5098 ✗ stacktop = reinterpret_cast<const uint16_t*>(srcp[vops[i].e.ival])[x];
5099 ✗ ++si;
5100 ✗ break;
5101 ✗ case opLoadSrcF32:
5102 ✗ stack[si] = stacktop;
5103 ✗ stacktop = reinterpret_cast<const float*>(srcp[vops[i].e.ival])[x];
5104 ✗ ++si;
5105 ✗ break;
5106 ✗ case opLoadRelSrc8:
5107 ✗ stack[si] = stacktop;
5108 {
5109 ✗ const int newx = x + vops[i].dx;
5110 ✗ const int newy = y + vops[i].dy;
5111 ✗ const int clipIndex = vops[i].e.ival;
5112 ✗ const uint8_t* srcp2 = srcp_orig[clipIndex] + max(0, min(newy, h - 1)) * src_stride[clipIndex];
5113 ✗ stacktop = srcp2[max(0, min(newx, w - 1))];
5114 }
5115 ✗ ++si;
5116 ✗ break;
5117 ✗ case opLoadRelSrc16:
5118 ✗ stack[si] = stacktop;
5119 {
5120 ✗ const int newx = x + vops[i].dx;
5121 ✗ const int newy = y + vops[i].dy;
5122 ✗ const int clipIndex = vops[i].e.ival;
5123 ✗ const uint16_t* srcp2 = reinterpret_cast<const uint16_t*>(srcp_orig[clipIndex] + max(0, min(newy, h - 1)) * src_stride[clipIndex]);
5124 ✗ stacktop = srcp2[max(0, min(newx, w - 1))];
5125 }
5126 ✗ ++si;
5127 ✗ break;
5128 ✗ case opLoadRelSrcF32:
5129 ✗ stack[si] = stacktop;
5130 {
5131 ✗ const int newx = x + vops[i].dx;
5132 ✗ const int newy = y + vops[i].dy;
5133 ✗ const int clipIndex = vops[i].e.ival;
5134 ✗ const float* srcp2 = reinterpret_cast<const float*>(srcp_orig[clipIndex] + max(0, min(newy, h - 1)) * src_stride[clipIndex]);
5135 ✗ stacktop = srcp2[max(0, min(newx, w - 1))];
5136 }
5137 ✗ ++si;
5138 ✗ break;
5139 ✗ case opLoadConst:
5140 ✗ stack[si] = stacktop;
5141 ✗ stacktop = vops[i].e.fval;
5142 ✗ ++si;
5143 ✗ break;
5144 ✗ case opLoadVar:
5145 ✗ stack[si] = stacktop;
5146 ✗ stacktop = variable_area[vops[i].e.ival];
5147 ✗ ++si;
5148 ✗ break;
5149 ✗ case opDup:
5150 ✗ stack[si] = stacktop;
5151 ✗ stacktop = stack[si - vops[i].e.ival];
5152 ✗ ++si;
5153 ✗ break;
5154 ✗ case opSwap:
5155 ✗ std::swap(stacktop, stack[si - vops[i].e.ival]);
5156 ✗ break;
5157 ✗ case opAdd:
5158 ✗ --si;
5159 ✗ stacktop += stack[si];
5160 ✗ break;
5161 ✗ case opSub:
5162 ✗ --si;
5163 ✗ stacktop = stack[si] - stacktop;
5164 ✗ break;
5165 ✗ case opMul:
5166 ✗ --si;
5167 ✗ stacktop *= stack[si];
5168 ✗ break;
5169 ✗ case opDiv:
5170 ✗ --si;
5171 ✗ stacktop = stack[si] / stacktop;
5172 ✗ break;
5173 ✗ case opFmod:
5174 ✗ --si;
5175 ✗ stacktop = std::fmod(stack[si], stacktop);
5176 ✗ break;
5177 ✗ case opMax:
5178 ✗ --si;
5179 ✗ stacktop = std::max(stacktop, stack[si]);
5180 ✗ break;
5181 ✗ case opMin:
5182 ✗ --si;
5183 ✗ stacktop = std::min(stacktop, stack[si]);
5184 ✗ break;
5185 ✗ case opExp:
5186 ✗ stacktop = std::exp(stacktop);
5187 ✗ break;
5188 ✗ case opLog:
5189 ✗ stacktop = std::log(stacktop);
5190 ✗ break;
5191 ✗ case opPow:
5192 ✗ --si;
5193 ✗ stacktop = std::pow(stack[si], stacktop);
5194 ✗ break;
5195 ✗ case opClip:
5196 // clip(a, low, high) = min(max(a, low),high)
5197 ✗ si -= 2;
5198 ✗ stacktop = std::max(std::min(stack[si], stacktop), stack[si + 1]);
5199 ✗ break;
5200 ✗ case opRound:
5201 ✗ stacktop = std::round(stacktop);
5202 ✗ break;
5203 ✗ case opFloor:
5204 ✗ stacktop = std::floor(stacktop);
5205 ✗ break;
5206 ✗ case opCeil:
5207 ✗ stacktop = std::ceil(stacktop);
5208 ✗ break;
5209 ✗ case opTrunc:
5210 ✗ stacktop = std::trunc(stacktop);
5211 ✗ break;
5212 ✗ case opSqrt:
5213 ✗ stacktop = std::sqrt(stacktop);
5214 ✗ break;
5215 ✗ case opAbs:
5216 ✗ stacktop = std::abs(stacktop);
5217 ✗ break;
5218 ✗ case opSgn:
5219 ✗ stacktop = stacktop < 0 ? -1.0f : stacktop > 0 ? 1.0f : 0.0f;
5220 ✗ break;
5221 ✗ case opSin:
5222 ✗ stacktop = std::sin(stacktop);
5223 ✗ break;
5224 ✗ case opCos:
5225 ✗ stacktop = std::cos(stacktop);
5226 ✗ break;
5227 ✗ case opTan:
5228 ✗ stacktop = std::tan(stacktop);
5229 ✗ break;
5230 ✗ case opAsin:
5231 ✗ stacktop = std::asin(stacktop);
5232 ✗ break;
5233 ✗ case opAcos:
5234 ✗ stacktop = std::acos(stacktop);
5235 ✗ break;
5236 ✗ case opAtan:
5237 ✗ stacktop = std::atan(stacktop);
5238 ✗ break;
5239 ✗ case opAtan2:
5240 ✗ --si;
5241 ✗ stacktop = std::atan2(stack[si], stacktop); // y, x -> -Pi..+Pi
5242 ✗ break;
5243 ✗ case opGt:
5244 ✗ --si;
5245 ✗ stacktop = (stack[si] > stacktop) ? 1.0f : 0.0f;
5246 ✗ break;
5247 ✗ case opLt:
5248 ✗ --si;
5249 ✗ stacktop = (stack[si] < stacktop) ? 1.0f : 0.0f;
5250 ✗ break;
5251 ✗ case opEq:
5252 ✗ --si;
5253 ✗ stacktop = (stack[si] == stacktop) ? 1.0f : 0.0f;
5254 ✗ break;
5255 ✗ case opNotEq:
5256 ✗ --si;
5257 ✗ stacktop = (stack[si] != stacktop) ? 1.0f : 0.0f;
5258 ✗ break;
5259 ✗ case opLE:
5260 ✗ --si;
5261 ✗ stacktop = (stack[si] <= stacktop) ? 1.0f : 0.0f;
5262 ✗ break;
5263 ✗ case opGE:
5264 ✗ --si;
5265 ✗ stacktop = (stack[si] >= stacktop) ? 1.0f : 0.0f;
5266 ✗ break;
5267 ✗ case opTernary:
5268 ✗ si -= 2;
5269 ✗ stacktop = (stack[si] > 0) ? stack[si + 1] : stacktop;
5270 ✗ break;
5271 ✗ case opAnd:
5272 ✗ --si;
5273 ✗ stacktop = (stacktop > 0 && stack[si] > 0) ? 1.0f : 0.0f;
5274 ✗ break;
5275 ✗ case opOr:
5276 ✗ --si;
5277 ✗ stacktop = (stacktop > 0 || stack[si] > 0) ? 1.0f : 0.0f;
5278 ✗ break;
5279 ✗ case opXor:
5280 ✗ --si;
5281 ✗ stacktop = ((stacktop > 0) != (stack[si] > 0)) ? 1.0f : 0.0f;
5282 ✗ break;
5283 ✗ case opNeg:
5284 ✗ stacktop = (stacktop > 0) ? 0.0f : 1.0f;
5285 ✗ break;
5286 ✗ case opNegSign:
5287 ✗ stacktop = -stacktop;
5288 ✗ break;
5289 ✗ case opStore8:
5290 ✗ dstp[x] = (uint8_t)(std::max(0.0f, std::min(stacktop, 255.0f)) + 0.5f);
5291 ✗ goto loopend;
5292 ✗ case opStore10:
5293 ✗ reinterpret_cast<uint16_t*>(dstp)[x] = (uint16_t)(std::max(0.0f, std::min(stacktop, 1023.0f)) + 0.5f);
5294 ✗ goto loopend;
5295 ✗ case opStore12:
5296 ✗ reinterpret_cast<uint16_t*>(dstp)[x] = (uint16_t)(std::max(0.0f, std::min(stacktop, 4095.0f)) + 0.5f);
5297 ✗ goto loopend;
5298 ✗ case opStore14:
5299 ✗ reinterpret_cast<uint16_t*>(dstp)[x] = (uint16_t)(std::max(0.0f, std::min(stacktop, 16383.0f)) + 0.5f);
5300 ✗ goto loopend;
5301 ✗ case opStore16:
5302 ✗ reinterpret_cast<uint16_t*>(dstp)[x] = (uint16_t)(std::max(0.0f, std::min(stacktop, 65535.0f)) + 0.5f);
5303 ✗ goto loopend;
5304 ✗ case opStoreF32:
5305 ✗ reinterpret_cast<float*>(dstp)[x] = stacktop;
5306 ✗ goto loopend;
5307 ✗ case opStoreVar:
5308 ✗ variable_area[vops[i].e.ival] = stacktop;
5309 ✗ break;
5310 ✗ case opStoreVarAndDrop1:
5311 ✗ variable_area[vops[i].e.ival] = stacktop;
5312 ✗ --si;
5313 ✗ if (si >= 0)
5314 ✗ stacktop = stack[si];
5315 ✗ break;
5316 }
5317 ✗ }
5318 ✗ loopend:;
5319 }
5320 ✗ dstp += dst_stride;
5321 ✗ if (d.lutmode == 0) {
5322 ✗ for (int i = 0; i < numInputs; i++)
5323 ✗ srcp[i] += src_stride[i];
5324 }
5325 }
5326 ✗ }
5327 ✗ }
5328
5329 ✗ void Exprfilter::preReadFrameProps(int plane, std::vector<PVideoFrame>& src, IScriptEnvironment* env)
5330 {
5331 ✗ for (auto& framePropToRead : d.frameprops[plane]) {
5332 ✗ int srcIndex = framePropToRead.srcIndex;
5333 ✗ auto fpname = framePropToRead.name;
5334
5335 ✗ const AVSMap* avsmap = env->getFramePropsRO(src[srcIndex]);
5336
5337 // default is 0f
5338 ✗ float varToStore = 0.0f; // std::numeric_limits<float>::quiet_NaN();
5339
5340 ✗ char res = env->propGetType(avsmap, fpname.c_str());
5341 // 'u'nset, 'i'nteger, 'f'loat, 's'string, 'c'lip, 'v'ideoframe, 'm'ethod };
5342
5343 int error;
5344 // only float and int are valid
5345 // cast to float: Expr supports 32 bit float, no double,
5346 ✗ if (res == 'i') {
5347 ✗ int64_t result = env->propGetInt(avsmap, fpname.c_str(), 0, &error);
5348 ✗ if (!error) varToStore = static_cast<float>(result);
5349 }
5350 ✗ else if (res == 'f') {
5351 ✗ float result = env->propGetFloatSaturated(avsmap, fpname.c_str(), 0, &error);
5352 ✗ if (!error) varToStore = result;
5353 }
5354
5355 ✗ framePropToRead.value = varToStore;
5356 ✗ }
5357 ✗ }
5358
5359 ✗ void Exprfilter::calculate_lut(IScriptEnvironment* env)
5360 {
5361 // ExprData d class variable already filled
5362
5363 // Only when there are frame props.
5364 // frame property set from GetFrame(0) is treated as Clip prop
5365
5366 ✗ std::vector<PVideoFrame> src;
5367
5368 ✗ bool frameprops = false;
5369 ✗ for (int plane = 0; plane < d.vi.NumComponents(); plane++) {
5370 ✗ if (d.frameprops[plane].size() > 0) {
5371 ✗ frameprops = true;
5372 ✗ break;
5373 }
5374 }
5375
5376 ✗ if (frameprops) {
5377 ✗ src.reserve(children.size());
5378 // fetch 0th frame only when needed in lut
5379 ✗ for (size_t i = 0; i < children.size(); i++) {
5380 ✗ const auto& child = children[i];
5381 ✗ src.emplace_back(child->GetFrame(0, env));
5382 }
5383 }
5384
5385 ✗ std::vector<const uint8_t*> srcp(MAX_EXPR_INPUTS);
5386 ✗ std::vector<const uint8_t*> srcp_orig(MAX_EXPR_INPUTS);
5387 ✗ std::vector<int> src_stride(MAX_EXPR_INPUTS);
5388
5389 ✗ for (int plane = 0; plane < d.vi.NumComponents(); plane++) {
5390 // calculate only if plane is processed
5391 ✗ if (d.plane[plane] != poProcess)
5392 ✗ continue;
5393
5394 // read actually needed frame properties into the variable storage area
5395 ✗ preReadFrameProps(plane, src, env);
5396
5397 uint8_t* dstp;
5398 int dst_stride;
5399 int h, w;
5400
5401 // no buffer allocated yet, prepare lut target buffer, and fake input frame dimensions
5402 ✗ const int bits_per_pixel = d.vi.BitsPerComponent();
5403 ✗ const int pixelsize = d.vi.ComponentSize();
5404 ✗ const auto lut1d_size = (1 << bits_per_pixel); // 1 or 2 bytes per entry
5405 ✗ const auto lut1d_bytesize = lut1d_size * pixelsize;
5406 ✗ const auto lut_size = d.lutmode == 1 ? lut1d_bytesize : lut1d_bytesize * lut1d_bytesize;
5407 // buffer start must be aligned to at least 32 bytes for avx2.
5408 // Size must be mod64 but it is fulfilled always.
5409 ✗ d.luts[plane] = (uint8_t *)avs_malloc(lut_size, 32); // 256 lut_x 65536: lut_xy (8 bit)
5410 ✗ dstp = d.luts[plane];
5411 ✗ dst_stride = lut1d_bytesize;
5412 ✗ h = lutmode == 1 ? 1 : lut1d_size; // 1x256, 256x256. 10 bit: 1024, 1024x1024
5413 ✗ w = lut1d_size;
5414
5415 // for simd:
5416 // same as in GetFrame
5417 ✗ const int pixels_per_iter = (optAvx2 && d.planeOptAvx2[plane]) ? (optSingleMode ? 8 : 16) : (optSingleMode ? 4 : 8);
5418 ✗ std::vector<intptr_t> ptroffsets(1 + 1 + MAX_EXPR_INPUTS);
5419 ✗ ptroffsets[RWPTR_START_OF_OUTPUT] = d.vi.ComponentSize() * pixels_per_iter; // stepping for output pointer
5420 ✗ ptroffsets[RWPTR_START_OF_XCOUNTER] = pixels_per_iter; // stepping for xcounter
5421
5422 // no srcp pointers in lut. Technically only inputless sx,sy relative coordinates are in there
5423 ✗ for (int i = 0; i < d.numInputs; i++) {
5424 ✗ srcp[i] = nullptr;
5425 ✗ srcp_orig[i] = nullptr;
5426 ✗ src_stride[i] = 0;
5427 ✗ ptroffsets[RWPTR_START_OF_INPUTS + i] = 0;
5428 }
5429
5430 ✗ const int dummy_framecount = 0;
5431 ✗ const int dummy_relative_time = 0;
5432 ✗ processFrame(plane, w, h, pixels_per_iter, dummy_framecount, dummy_relative_time, d.numInputs, dstp, dst_stride, srcp, src_stride, ptroffsets, srcp_orig);
5433 ✗ } // for planes
5434 ✗ }
5435
5436 template<typename pixel_t, int bits_per_pixel>
5437 ✗ static void do_lut_xy(const uint8_t* lut8, uint8_t* dstp, int dst_stride, const uint8_t** srcp, const int* src_stride, int w, int h)
5438 {
5439 ✗ const int max_pixel_value = (1 << bits_per_pixel) - 1;
5440 ✗ const pixel_t* lut = reinterpret_cast<const pixel_t*>(lut8);
5441 ✗ const uint8_t* src0 = srcp[0];
5442 ✗ const uint8_t* src1 = srcp[1];
5443 ✗ const auto pitch0 = src_stride[0];
5444 ✗ const auto pitch1 = src_stride[1];
5445 ✗ for (auto y = 0; y < h; y++) {
5446 ✗ for (auto x = 0; x < w; x++) {
5447 if constexpr (bits_per_pixel == 8 || bits_per_pixel == 16) {
5448 // no limit check
5449 ✗ const int pixel0 = reinterpret_cast<const pixel_t*>(src0)[x];
5450 ✗ const int pixel1 = reinterpret_cast<const pixel_t*>(src1)[x];
5451 ✗ reinterpret_cast<pixel_t*>(dstp)[x] = lut[(pixel1 << bits_per_pixel) + pixel0];
5452 }
5453 else {
5454 ✗ const int pixel0 = min((int)reinterpret_cast<const pixel_t*>(src0)[x], max_pixel_value);
5455 ✗ const int pixel1 = min((int)reinterpret_cast<const pixel_t*>(src1)[x], max_pixel_value);
5456 ✗ reinterpret_cast<pixel_t*>(dstp)[x] = lut[(pixel1 << bits_per_pixel) + pixel0];
5457 }
5458 }
5459 ✗ src0 += pitch0;
5460 ✗ src1 += pitch1;
5461 ✗ dstp += dst_stride;
5462 }
5463 ✗ }
5464
5465 ✗ PVideoFrame __stdcall Exprfilter::GetFrame(int n, IScriptEnvironment *env) {
5466 // ExprData d class variable already filled
5467
5468 ✗ std::vector<PVideoFrame> src;
5469 ✗ src.reserve(children.size());
5470
5471 ✗ int first_used_clip_index = -1;
5472 ✗ for (size_t i = 0; i < children.size(); i++) {
5473 ✗ const auto &child = children[i];
5474 ✗ src.emplace_back(d.clipsUsed[i] ? child->GetFrame(n, env) : nullptr); // GetFrame only when really referenced
5475 ✗ if (first_used_clip_index < 0) {
5476 ✗ if (d.clipsUsed[i])
5477 ✗ first_used_clip_index = (int)i; // inherit frameprop from
5478 }
5479 }
5480
5481 ✗ PVideoFrame dst;
5482 ✗ if (first_used_clip_index >= 0)
5483 ✗ dst = env->NewVideoFrameP(d.vi, &src[first_used_clip_index]);
5484 else
5485 ✗ dst = env->NewVideoFrame(d.vi);
5486
5487 ✗ std::vector<const uint8_t*> srcp(MAX_EXPR_INPUTS);
5488 ✗ std::vector<const uint8_t*> srcp_orig(MAX_EXPR_INPUTS);
5489 ✗ std::vector<int> src_stride(MAX_EXPR_INPUTS);
5490
5491 ✗ const float framecount = (float)n; // max precision: 2^24 (16M) frames (32 bit float precision)
5492 ✗ const float relative_time = vi.num_frames > 1 ? (float)((double)n / (vi.num_frames - 1)) : 0.0f; // 0 <= time <= 1
5493 ✗ const int planes_y[4] = { PLANAR_Y, PLANAR_U, PLANAR_V, PLANAR_A };
5494 ✗ const int planes_r[4] = { PLANAR_R, PLANAR_G, PLANAR_B, PLANAR_A }; // expression string order is R G B unlike internal G B R plane order
5495 ✗ const int *plane_enums_d = (d.vi.IsYUV() || d.vi.IsYUVA()) ? planes_y : planes_r;
5496
5497 ✗ for (int plane = 0; plane < d.vi.NumComponents(); plane++) {
5498
5499 ✗ const int plane_enum_d = plane_enums_d[plane];
5500
5501 ✗ if (d.plane[plane] == poProcess) {
5502
5503 // read actually needed frame properties into the variable storage area
5504 ✗ preReadFrameProps(plane, src, env);
5505
5506 uint8_t* dstp;
5507 int dst_stride;
5508 int h, w;
5509
5510 ✗ dstp = dst->GetWritePtr(plane_enum_d);
5511 ✗ dst_stride = dst->GetPitch(plane_enum_d);
5512 ✗ h = d.vi.height >> d.vi.GetPlaneHeightSubsampling(plane_enum_d);
5513 ✗ w = d.vi.width >> d.vi.GetPlaneWidthSubsampling(plane_enum_d);
5514
5515 // for simd:
5516 ✗ const int pixels_per_iter = (optAvx2 && d.planeOptAvx2[plane]) ? (optSingleMode ? 8 : 16) : (optSingleMode ? 4 : 8);
5517 ✗ std::vector<intptr_t> ptroffsets(1 + 1 + MAX_EXPR_INPUTS);
5518 ✗ ptroffsets[RWPTR_START_OF_OUTPUT] = d.vi.ComponentSize() * pixels_per_iter; // stepping for output pointer
5519 ✗ ptroffsets[RWPTR_START_OF_XCOUNTER] = pixels_per_iter; // stepping for xcounter
5520
5521 ✗ for (int i = 0; i < d.numInputs; i++) {
5522 ✗ if (d.clips[i]) {
5523 ✗ if (d.clipsUsed[i]) {
5524 // when input is a single Y, use PLANAR_Y instead of the plane matching to the output
5525 ✗ const VideoInfo& vi_src = d.clips[i]->GetVideoInfo();
5526 ✗ const int* plane_enums_s = (vi_src.IsYUV() || d.vi.IsYUVA()) ? planes_y : planes_r;
5527 ✗ const int plane_enum_s = vi_src.IsY() ? PLANAR_Y : plane_enums_s[plane];
5528
5529 ✗ srcp[i] = src[i]->GetReadPtr(plane_enum_s);
5530 // C only:
5531 ✗ srcp_orig[i] = srcp[i];
5532 ✗ src_stride[i] = src[i]->GetPitch(plane_enum_s);
5533 // SIMD only
5534 ✗ ptroffsets[RWPTR_START_OF_INPUTS + i] = d.clips[i]->GetVideoInfo().ComponentSize() * pixels_per_iter; // 1..Nth: inputs
5535 }
5536 else {
5537 ✗ srcp[i] = nullptr;
5538 ✗ srcp_orig[i] = nullptr;
5539 ✗ src_stride[i] = 0;
5540 ✗ ptroffsets[RWPTR_START_OF_INPUTS + i] = 0;
5541 }
5542 }
5543 }
5544
5545 ✗ if (lutmode == 0) {
5546 ✗ processFrame(plane, w, h, pixels_per_iter, framecount, relative_time, d.numInputs, dstp, dst_stride, srcp, src_stride, ptroffsets, srcp_orig);
5547 } else {
5548 // lut table for plane is filled, do lookup now
5549 ✗ const int bits_per_pixel = d.vi.BitsPerComponent();
5550
5551 ✗ if (d.lutmode == 1) {
5552 // lut_x
5553 ✗ if (bits_per_pixel == 8)
5554 {
5555 ✗ uint8_t* lut = d.luts[plane];
5556 ✗ const uint8_t* src0 = srcp[0];
5557 ✗ const auto pitch0 = src_stride[0];
5558 ✗ for (auto y = 0; y < h; y++) {
5559 ✗ for (auto x = 0; x < w; x++) {
5560 ✗ const int pixel = src0[x];
5561 ✗ dstp[x] = lut[pixel];
5562 }
5563 ✗ src0 += pitch0;
5564 ✗ dstp += dst_stride;
5565 }
5566 }
5567 else {
5568 ✗ const int max_pixel_value = (1 << bits_per_pixel) - 1;
5569 ✗ uint16_t* lut = reinterpret_cast<uint16_t*>(d.luts[plane]);
5570 ✗ const uint8_t* src0 = srcp[0];
5571 ✗ const auto pitch0 = src_stride[0];
5572 ✗ if (bits_per_pixel == 16) {
5573 // no limit check
5574 ✗ for (auto y = 0; y < h; y++) {
5575 ✗ for (auto x = 0; x < w; x++) {
5576 ✗ const int pixel = reinterpret_cast<const uint16_t*>(src0)[x];
5577 ✗ reinterpret_cast<uint16_t*>(dstp)[x] = lut[pixel];
5578 }
5579 ✗ src0 += pitch0;
5580 ✗ dstp += dst_stride;
5581 }
5582 }
5583 else {
5584 ✗ for (auto y = 0; y < h; y++) {
5585 ✗ for (auto x = 0; x < w; x++) {
5586 ✗ const int pixel = reinterpret_cast<const uint16_t*>(src0)[x];
5587 ✗ reinterpret_cast<uint16_t*>(dstp)[x] = lut[min(pixel, max_pixel_value)]; // e.g. 10 bits in 2 byte safety
5588 }
5589 ✗ src0 += pitch0;
5590 ✗ dstp += dst_stride;
5591 }
5592 }
5593 }
5594 }
5595 ✗ else if (d.lutmode == 2) {
5596 // lut_xy
5597 // templates for speed: bitshift with immediate constant
5598 ✗ const uint8_t* lut = d.luts[plane];
5599 ✗ if (bits_per_pixel == 8)
5600 ✗ do_lut_xy<uint8_t, 8>(lut, dstp, dst_stride, srcp_orig.data(), src_stride.data(), w, h);
5601 ✗ else if (bits_per_pixel == 10)
5602 ✗ do_lut_xy<uint16_t, 10>(lut, dstp, dst_stride, srcp_orig.data(), src_stride.data(), w, h);
5603 ✗ else if (bits_per_pixel == 12)
5604 ✗ do_lut_xy<uint16_t, 12>(lut, dstp, dst_stride, srcp_orig.data(), src_stride.data(), w, h);
5605 ✗ else if (bits_per_pixel == 14)
5606 ✗ do_lut_xy<uint16_t, 14>(lut, dstp, dst_stride, srcp_orig.data(), src_stride.data(), w, h);
5607 ✗ else if (bits_per_pixel == 16) // well, this is not enabled 16bit lutxy would take a 8GB table
5608 ✗ do_lut_xy<uint16_t, 16>(lut, dstp, dst_stride, srcp_orig.data(), src_stride.data(), w, h);
5609 else
5610 ✗ assert(0);
5611 }
5612 else { // 1d lut, 2d lut
5613 ✗ assert(0); // no lut_xyz
5614 }
5615 } // lut branch
5616 ✗ }
5617 // avs+: copy plane here
5618 ✗ else if (d.plane[plane] == poCopy) {
5619 // avs+ copy from Nth clip
5620 ✗ const int copySource = d.planeCopySourceClip[plane];
5621 // when input is a single Y, use PLANAR_Y instead of the plane matching to the output
5622 ✗ const VideoInfo& vi_src = d.clips[copySource]->GetVideoInfo();
5623 ✗ const int plane_enum_s = vi_src.IsY() ? PLANAR_Y : plane_enums_d[plane];
5624
5625 ✗ env->BitBlt(dst->GetWritePtr(plane_enum_d), dst->GetPitch(plane_enum_d),
5626 ✗ src[copySource]->GetReadPtr(plane_enum_s),
5627 ✗ src[copySource]->GetPitch(plane_enum_s),
5628 ✗ src[copySource]->GetRowSize(plane_enum_s),
5629 ✗ src[copySource]->GetHeight(plane_enum_s)
5630 );
5631 }
5632 ✗ else if (d.plane[plane] == poFill) { // avs+
5633 ✗ uint8_t *dstp = dst->GetWritePtr(plane_enum_d);
5634 ✗ const int dst_rowsize = dst->GetRowSize(plane_enum_d);
5635 ✗ const int dst_stride = dst->GetPitch(plane_enum_d);
5636 ✗ const int h = dst->GetHeight(plane_enum_d);
5637
5638 ✗ const int bits_per_pixel = vi.BitsPerComponent();
5639
5640 ✗ float val = d.planeFillValue[plane];
5641 ✗ int val_i = 0;
5642 ✗ if (bits_per_pixel <= 16) {
5643 ✗ const int max_pixel_value = (1 << bits_per_pixel) - 1;
5644 ✗ val_i = (int)(std::max(0.0f, std::min(val, (float)max_pixel_value)) + 0.5f);
5645 }
5646
5647 ✗ if(bits_per_pixel == 8)
5648 ✗ fill_plane<BYTE>(dstp, h, dst_rowsize, dst_stride, val_i);
5649 ✗ else if(bits_per_pixel <= 16)
5650 ✗ fill_plane<uint16_t>(dstp, h, dst_rowsize, dst_stride, val_i);
5651 else // 32 bit float
5652 ✗ fill_plane<float>(dstp, h, dst_rowsize, dst_stride, val);
5653
5654 } // plane modes
5655 } // for planes
5656
5657 ✗ return dst;
5658 ✗ }
5659
5660 ✗ Exprfilter::~Exprfilter() {
5661 ✗ for (int i = 0; i < MAX_EXPR_INPUTS; i++)
5662 ✗ d.clips[i] = nullptr;
5663 ✗ for (int i = 0; i < 4; i++)
5664 ✗ if(d.luts[i]) avs_free(d.luts[i]); // aligned free
5665 ✗ }
5666
5667 ✗ static SOperation getLoadOp(const VideoInfo *vi, bool relativeKind) {
5668 ✗ if (!vi)
5669 ✗ return relativeKind ? opLoadRelSrcF32 : opLoadSrcF32;
5670 ✗ if (vi->BitsPerComponent() == 32) // float, avs has no f16c float
5671 ✗ return relativeKind ? opLoadRelSrcF32 : opLoadSrcF32;
5672 ✗ else if (vi->BitsPerComponent() == 8)
5673 ✗ return relativeKind ? opLoadRelSrc8 : opLoadSrc8;
5674 else
5675 ✗ return relativeKind ? opLoadRelSrc16 : opLoadSrc16; // 10..16 bits common
5676 }
5677
5678 ✗ static SOperation getStoreOp(const VideoInfo *vi) {
5679 // avs+ has no f16c float
5680 ✗ switch (vi->BitsPerComponent()) {
5681 ✗ case 8: return opStore8;
5682 ✗ case 10: return opStore10; // avs+
5683 ✗ case 12: return opStore12; // avs+
5684 ✗ case 14: return opStore14; // avs+
5685 ✗ case 16: return opStore16;
5686 ✗ case 32: return opStoreF32;
5687 ✗ default: return opStoreF32;
5688 }
5689 }
5690
5691 #define LOAD_OP(op,v,req) do { if (stackSize < req) env->ThrowError("Expr: Not enough elements on stack to perform operation %s", tokens[i].c_str()); ops.push_back(ExprOp(op, (v))); maxStackSize = std::max(++stackSize, maxStackSize); } while(0)
5692 #define LOAD_REL_OP(op,v,req,dx,dy) do { if (stackSize < req) env->ThrowError("Expr: Not enough elements on stack to perform operation %s", tokens[i].c_str()); ops.push_back(ExprOp(op, (v), (dx), (dy))); maxStackSize = std::max(++stackSize, maxStackSize); } while(0)
5693 #define GENERAL_OP(op, v, req, dec) do { if (stackSize < req) env->ThrowError("Expr: Not enough elements on stack to perform operation %s", tokens[i].c_str()); ops.push_back(ExprOp(op, (v))); stackSize-=(dec); } while(0)
5694 #define ONE_ARG_OP(op) GENERAL_OP(op, 0, 1, 0)
5695 #define VAR_STORE_OP(op,v) GENERAL_OP(op, v, 1, 0)
5696 #define VAR_STORE_SPEC_OP(op,v) GENERAL_OP(op, v, 1, 1)
5697 #define TWO_ARG_OP(op) GENERAL_OP(op, 0, 2, 1)
5698 #define THREE_ARG_OP(op) GENERAL_OP(op, 0, 3, 2)
5699 // defines for special scale-back-before-store where no token is in context:
5700 #define LOAD_OP_NOTOKEN(op,v,req) do { if (stackSize < req) env->ThrowError("Expr: Not enough elements on stack to perform a load operation"); ops.push_back(ExprOp(op, (v))); maxStackSize = std::max(++stackSize, maxStackSize); } while(0)
5701 #define GENERAL_OP_NOTOKEN(op, v, req, dec) do { if (stackSize < req) env->ThrowError("Expr: Not enough elements on stack to perform an operation"); ops.push_back(ExprOp(op, (v))); stackSize-=(dec); } while(0)
5702 #define TWO_ARG_OP_NOTOKEN(op) GENERAL_OP_NOTOKEN(op, 0, 2, 1)
5703
5704 ✗ static inline bool isAlphaUnderscore(char c) {
5705 ✗ return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_';
5706 }
5707
5708 ✗ static inline bool isAlphaNumUnderscore(char c) {
5709 ✗ return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_';
5710 }
5711
5712 ✗ static bool isValidVarName(const std::string& s) {
5713 ✗ size_t len = s.length();
5714 ✗ if (!len)
5715 ✗ return false;
5716
5717 ✗ if (!isAlphaUnderscore(s[0]))
5718 ✗ return false;
5719 ✗ for (size_t i = 1; i < len; i++)
5720 ✗ if (!isAlphaNumUnderscore(s[i]))
5721 ✗ return false;
5722 ✗ return true;
5723 }
5724
5725
5726 // finds _X suffix (clip letter) and returns 0..25 for x,y,z,a,b,...w
5727 // no suffix means 0
5728 ✗ static int getSuffix(std::string token, std::string base) {
5729 ✗ size_t len = base.length();
5730
5731 ✗ if (token.substr(0, len) != base)
5732 ✗ return -1; // no match
5733
5734 ✗ if (token.length() == len)
5735 ✗ return 0; // no suffix, treat as _x
5736
5737 // find _X suffix, where X is x,y,z,a..w
5738 ✗ if (token.length() != len + 2 || token[len] != '_')
5739 ✗ return -2; // no proper suffix
5740
5741 ✗ char srcChar = token[len + 1]; // last char
5742 int loadIndex;
5743 ✗ if (srcChar >= 'x')
5744 ✗ loadIndex = srcChar - 'x';
5745 else
5746 ✗ loadIndex = srcChar - 'a' + 3;
5747 ✗ return loadIndex;
5748 }
5749
5750 // if automatic source bit depth conversion takes place, ymax, ymin, range_xxx, etc.. constants are changed accordingly
5751 ✗ static int getEffectiveBitsPerComponent(int bitsPerComponent, bool autoconv_conv_float, bool autoconv_conv_int, int autoScaleSourceBitDepth)
5752 {
5753 ✗ if ((autoconv_conv_float && bitsPerComponent == 32) || (autoconv_conv_int && bitsPerComponent != 32))
5754 ✗ return autoScaleSourceBitDepth;
5755 ✗ return bitsPerComponent;
5756 }
5757
5758 ✗ static size_t parseExpression(const std::string &expr, std::vector<ExprOp> &ops, std::vector<ExprFramePropData>& fp, const VideoInfo **vi, const VideoInfo *vi_output, const SOperation storeOp, int numInputs, int planewidth, int planeheight, bool chroma,
5759 const bool autoconv_full_scale, const bool autoconv_conv_int, const bool autoconv_conv_float, const int clamp_float_i, const bool shift_float, const int lutmode,
5760 IScriptEnvironment *env)
5761 {
5762 // vi_output is new in avs+, and is not used yet
5763
5764 // optional scaling, scale_from bit depth, default scale_to bitdepth, used in scaleb and scalef (yscaleb, yscalef)
5765 ✗ int targetBitDepth = vi[0]->BitsPerComponent(); // avs+
5766 ✗ int autoScaleSourceBitDepth = 8; // avs+ scalable constants are in 8 bit range by default
5767
5768 ✗ std::vector<std::string> tokens;
5769 ✗ split(tokens, expr, " \r\n\t", split1::no_empties);
5770
5771 ✗ std::unordered_map<std::string, int> varnames;
5772 ✗ int varindex = 0;
5773 ✗ std::unordered_map<std::string, int> fpnames;
5774 ✗ int fpindex = 0;
5775
5776 ✗ size_t maxStackSize = 0;
5777 ✗ size_t stackSize = 0;
5778
5779 ✗ for (size_t i = 0; i < tokens.size(); i++) {
5780 ✗ const size_t tokenlen = tokens[i].length();
5781 ✗ if (tokens[i] == "+")
5782 ✗ TWO_ARG_OP(opAdd);
5783 ✗ else if (tokens[i] == "-")
5784 ✗ TWO_ARG_OP(opSub);
5785 ✗ else if (tokens[i] == "*")
5786 ✗ TWO_ARG_OP(opMul);
5787 ✗ else if (tokens[i] == "/")
5788 ✗ TWO_ARG_OP(opDiv);
5789 ✗ else if (tokens[i] == "%")
5790 ✗ TWO_ARG_OP(opFmod);
5791 ✗ else if (tokens[i] == "max")
5792 ✗ TWO_ARG_OP(opMax);
5793 ✗ else if (tokens[i] == "min")
5794 ✗ TWO_ARG_OP(opMin);
5795 ✗ else if (tokens[i] == "exp")
5796 ✗ ONE_ARG_OP(opExp);
5797 ✗ else if (tokens[i] == "log")
5798 ✗ ONE_ARG_OP(opLog);
5799 ✗ else if (tokens[i] == "pow" || tokens[i] == "^") // avs+: ^ can be used for power
5800 ✗ TWO_ARG_OP(opPow);
5801 ✗ else if (tokens[i] == "sqrt")
5802 ✗ ONE_ARG_OP(opSqrt);
5803 ✗ else if (tokens[i] == "abs")
5804 ✗ ONE_ARG_OP(opAbs);
5805 ✗ else if (tokens[i] == "sgn")
5806 ✗ ONE_ARG_OP(opSgn);
5807 ✗ else if (tokens[i] == "sin")
5808 ✗ ONE_ARG_OP(opSin);
5809 ✗ else if (tokens[i] == "cos")
5810 ✗ ONE_ARG_OP(opCos);
5811 ✗ else if (tokens[i] == "tan")
5812 ✗ ONE_ARG_OP(opTan);
5813 ✗ else if (tokens[i] == "asin")
5814 ✗ ONE_ARG_OP(opAsin);
5815 ✗ else if (tokens[i] == "acos")
5816 ✗ ONE_ARG_OP(opAcos);
5817 ✗ else if (tokens[i] == "atan")
5818 ✗ ONE_ARG_OP(opAtan);
5819 ✗ else if (tokens[i] == "atan2")
5820 ✗ TWO_ARG_OP(opAtan2);
5821 ✗ else if (tokens[i] == "clip")
5822 ✗ THREE_ARG_OP(opClip);
5823 ✗ else if (tokens[i] == "round")
5824 ✗ ONE_ARG_OP(opRound);
5825 ✗ else if (tokens[i] == "floor")
5826 ✗ ONE_ARG_OP(opFloor);
5827 ✗ else if (tokens[i] == "ceil")
5828 ✗ ONE_ARG_OP(opCeil);
5829 ✗ else if (tokens[i] == "trunc")
5830 ✗ ONE_ARG_OP(opTrunc);
5831 ✗ else if (tokens[i] == ">")
5832 ✗ TWO_ARG_OP(opGt);
5833 ✗ else if (tokens[i] == "<")
5834 ✗ TWO_ARG_OP(opLt);
5835 ✗ else if (tokens[i] == "=" || tokens[i] == "==") // avs+: == can be used to equality check
5836 ✗ TWO_ARG_OP(opEq);
5837 ✗ else if (tokens[i] == "!=") // avs+: not equal
5838 ✗ TWO_ARG_OP(opNotEq);
5839 ✗ else if (tokens[i] == ">=")
5840 ✗ TWO_ARG_OP(opGE);
5841 ✗ else if (tokens[i] == "<=")
5842 ✗ TWO_ARG_OP(opLE);
5843 ✗ else if (tokens[i] == "?")
5844 ✗ THREE_ARG_OP(opTernary);
5845 ✗ else if (tokens[i] == "and" || tokens[i] == "&") // avs+: & alias for and
5846 ✗ TWO_ARG_OP(opAnd);
5847 ✗ else if (tokens[i] == "or" || tokens[i] == "|") // avs+: | alias for or
5848 ✗ TWO_ARG_OP(opOr);
5849 ✗ else if (tokens[i] == "xor")
5850 ✗ TWO_ARG_OP(opXor);
5851 ✗ else if (tokens[i] == "not")
5852 ✗ ONE_ARG_OP(opNeg);
5853 ✗ else if (tokens[i] == "neg")
5854 ✗ ONE_ARG_OP(opNegSign);
5855 ✗ else if (tokens[i].substr(0, 3) == "dup")
5856 ✗ if (tokens[i].size() == 3) {
5857 ✗ LOAD_OP(opDup, 0, 1);
5858 }
5859 else {
5860 try {
5861 ✗ int tmp = std::stoi(tokens[i].substr(3));
5862 ✗ if (tmp < 0)
5863 ✗ env->ThrowError("Expr: Dup suffix can't be less than 0 '%s'", tokens[i].c_str());
5864 ✗ LOAD_OP(opDup, tmp, (size_t)(tmp + 1));
5865 }
5866 ✗ catch (std::logic_error &) {
5867 ✗ env->ThrowError("Expr: Failed to convert dup suffix '%s' to valid index", tokens[i].c_str());
5868 ✗ }
5869 }
5870 ✗ else if (tokens[i].substr(0, 4) == "swap")
5871 ✗ if (tokens[i].size() == 4) {
5872 ✗ GENERAL_OP(opSwap, 1, 2, 0);
5873 }
5874 else {
5875 try {
5876 ✗ int tmp = std::stoi(tokens[i].substr(4));
5877 ✗ if (tmp < 1)
5878 ✗ env->ThrowError("Expr: Swap suffix can't be less than 1 '%s'", tokens[i].c_str());
5879 ✗ GENERAL_OP(opSwap, tmp, (size_t)(tmp + 1), 0);
5880 }
5881 ✗ catch (std::logic_error &) {
5882 ✗ env->ThrowError("Expr: Failed to convert swap suffix '%s' to valid index", tokens[i].c_str());
5883 ✗ }
5884 }
5885 ✗ else if (tokens[i] == "sx") { // avs+
5886 // spatial
5887 ✗ if (lutmode > 0)
5888 ✗ env->ThrowError("Expr: 'sx' is forbidden in lut mode");
5889 ✗ LOAD_OP(opLoadSpatialX, 0, 0);
5890 }
5891 ✗ else if (tokens[i] == "sy") { // avs+
5892 // spatial
5893 ✗ if (lutmode > 0)
5894 ✗ env->ThrowError("Expr: 'sy' is forbidden in lut mode");
5895 ✗ LOAD_OP(opLoadSpatialY, 0, 0);
5896 }
5897 ✗ else if (tokens[i] == "sxr") { // avs+
5898 // spatial X relative 0..1
5899 ✗ if (lutmode > 0)
5900 ✗ env->ThrowError("Expr: 'sxr' is forbidden in lut mode");
5901 ✗ LOAD_OP(opLoadSpatialX, 0, 0);
5902 /* Paranoia: precision at rightmost position? Ensure that sxr == 1.0 there
5903 Multiply by 1/x is different.
5904 Test:
5905 constexpr float x = 1919.0f;
5906 constexpr float y = 1 / 1919.0f;
5907 constexpr float zz = x * y;
5908 constexpr bool b = zz == 1.0f; // false!
5909
5910 const float p = planewidth > 1 ? 1.0f / ((float)planewidth - 1.0f) : 1.0f;
5911 LOAD_OP(opLoadConst, p, 0);
5912 TWO_ARG_OP(opMul);
5913 */
5914 ✗ const float p = planewidth > 1 ? (float)planewidth - 1.0f : 1.0f;
5915 ✗ LOAD_OP(opLoadConst, p, 0);
5916 ✗ TWO_ARG_OP(opDiv);
5917 }
5918 ✗ else if (tokens[i] == "syr") { // avs+
5919 // spatial Y relative 0..1
5920 ✗ if (lutmode > 0)
5921 ✗ env->ThrowError("Expr: 'syr' is forbidden in lut mode");
5922 ✗ LOAD_OP(opLoadSpatialY, 0, 0);
5923 /* Multiply by 1/x is different
5924 const float p = planeheight > 1 ? 1.0f / ((float)planeheight - 1.0f) : 1.0f;
5925 LOAD_OP(opLoadConst, p, 0);
5926 TWO_ARG_OP(opMul);
5927 */
5928 ✗ const float p = planeheight > 1 ? (float)planeheight - 1.0f : 1.0f;
5929 ✗ LOAD_OP(opLoadConst, p, 0);
5930 ✗ TWO_ARG_OP(opDiv);
5931 }
5932 ✗ else if (tokens[i] == "frameno") { // avs+
5933 ✗ if (lutmode > 0)
5934 ✗ env->ThrowError("Expr: 'frameno' is forbidden in lut mode");
5935 ✗ LOAD_OP(opLoadInternalVar, INTERNAL_VAR_CURRENT_FRAME, 0);
5936 }
5937 ✗ else if (tokens[i] == "time") { // avs+
5938 ✗ if (lutmode > 0)
5939 ✗ env->ThrowError("Expr: 'time' is forbidden in lut mode");
5940 ✗ LOAD_OP(opLoadInternalVar, INTERNAL_VAR_RELTIME, 0);
5941 }
5942 ✗ else if (tokens[i] == "width") { // avs+
5943 ✗ LOAD_OP(opLoadConst, (float)planewidth, 0);
5944 }
5945 ✗ else if (tokens[i] == "height") { // avs+
5946 ✗ LOAD_OP(opLoadConst, (float)planeheight, 0);
5947 }
5948 ✗ else if ((tokens[i].length() == 1 || (tokens[i].length() > 1 && tokens[i][1] == '[')) && tokens[i][0] >= 'a' && tokens[i][0] <= 'z') {
5949 ✗ const bool rel = tokens[i].length() > 1; // relative pixel addressing; indexed clips e.g. x[-1,-2]
5950 // loading source clip pixels
5951 ✗ char srcChar = tokens[i][0];
5952 int loadIndex;
5953 ✗ if (srcChar >= 'x')
5954 ✗ loadIndex = srcChar - 'x';
5955 else
5956 ✗ loadIndex = srcChar - 'a' + 3;
5957 ✗ if (loadIndex >= numInputs)
5958 ✗ env->ThrowError("Expr: Too few input clips supplied to reference '%s'", tokens[i].c_str());
5959
5960 ✗ if (rel) {
5961 int dx, dy;
5962 ✗ std::string s;
5963 ✗ std::istringstream numStream(tokens[i].substr(2)); // after '['
5964 ✗ numStream.imbue(std::locale::classic());
5965 // first coord
5966 ✗ if (!(numStream >> dx))
5967 ✗ env->ThrowError("Expr: Failed to convert '%s' to integer, relative index dx", tokens[i].c_str());
5968 // separator ','
5969 ✗ if (numStream.get() != ',')
5970 ✗ env->ThrowError("Expr: Failed to convert '%s', character ',' expected between the coordinates", tokens[i].c_str());
5971 // second coord
5972 ✗ if (!(numStream >> dy))
5973 ✗ env->ThrowError("Expr: Failed to convert '%s' to integer, relative index dy", tokens[i].c_str());
5974 // ending ']'
5975 ✗ if (numStream.get() != ']')
5976 ✗ env->ThrowError("Expr: Failed to convert '%s' to [x,y], closing ']' expected ", tokens[i].c_str());
5977 ✗ if (numStream >> s)
5978 ✗ env->ThrowError("Expr: Failed to convert '%s' to [x,y], invalid character after ']'", tokens[i].c_str());
5979
5980 ✗ if (lutmode > 0)
5981 ✗ env->ThrowError("Expr: relative pixel addressing is forbidden in lut mode");
5982
5983 ✗ if (dx <= -vi_output->width || dx >= vi_output->width)
5984 ✗ env->ThrowError("Expr: dx must be between +/- (width-1) in '%s'", tokens[i].c_str());
5985 ✗ if (dy <= -vi_output->height || dy >= vi_output->height)
5986 ✗ env->ThrowError("Expr: dy must be between +/- (height-1) in '%s'", tokens[i].c_str());
5987 ✗ LOAD_REL_OP(getLoadOp(vi[loadIndex], true), loadIndex, 0, dx, dy);
5988 ✗ }
5989 else {
5990 // not relative, single clip letter
5991 ✗ if (lutmode == 0) {
5992 ✗ LOAD_OP(getLoadOp(vi[loadIndex], false), loadIndex, 0);
5993 }
5994 else {
5995 // for lut we replace x and y to sx and sy to make the initialization
5996 ✗ if (loadIndex >= lutmode) // lutx
5997 ✗ env->ThrowError("Expr: more input clips than lut's dimension. Problematic clip: '%s'", tokens[i].c_str());
5998 // spatial
5999 ✗ if (loadIndex == 0)
6000 ✗ LOAD_OP(opLoadSpatialX, 0, 0);
6001 else // if (loadIndex == 1)
6002 ✗ LOAD_OP(opLoadSpatialY, 0, 0);
6003 }
6004 }
6005
6006 // avs+: 'scale_inputs': converts input pixels to a common specified range
6007 // Apply full or limited conversion to integer and/or float bit depths.
6008 // For integers bit-shift or full-scale-stretch method can be chosen
6009 // There is no precision loss, since the multiplication/division occurs when original pixels
6010 // are already loaded as float
6011 ✗ const int srcBitDepth = vi[loadIndex]->BitsPerComponent();
6012 ✗ const int dstBitDepth = autoScaleSourceBitDepth; // internal precision
6013
6014 ✗ const bool use_chroma = chroma; // && !forceNonUV;
6015 ✗ const bool isfull = autoconv_full_scale;
6016
6017 ✗ if (autoconv_conv_int || autoconv_conv_float) {
6018
6019 ✗ if ((srcBitDepth != 32 && autoconv_conv_int) ||
6020 ✗ ((srcBitDepth == 32 && autoconv_conv_float))) {
6021
6022 ✗ bits_conv_constants d;
6023 ✗ get_bits_conv_constants(d, use_chroma, isfull, isfull, srcBitDepth, dstBitDepth);
6024 // chroma is spec: signed. Limited:16-240 is really 128 +/-112. Full:1-255 is really 128+/-127
6025
6026 ✗ if (d.src_offset != 0) {
6027 ✗ LOAD_OP(opLoadConst, (float)d.src_offset, 0);
6028 ✗ TWO_ARG_OP(opSub);
6029 }
6030 ✗ if (d.mul_factor != 1.0f) {
6031 ✗ LOAD_OP(opLoadConst, d.mul_factor, 0);
6032 ✗ TWO_ARG_OP(opMul);
6033 }
6034 ✗ if (d.dst_offset != 0) {
6035 ✗ LOAD_OP(opLoadConst, (float)d.dst_offset, 0);
6036 ✗ TWO_ARG_OP(opAdd);
6037 }
6038 }
6039 }
6040
6041 // "floatUV" - shifts -0.5 .. +0.5 chroma range to 0 .. 1.0 only when no other autoscaling is active
6042 // the effect of this pre-shift is reversed at the storage phase
6043 ✗ if ((!autoconv_conv_float || dstBitDepth == 32) && srcBitDepth == 32) {
6044 ✗ if (chroma && shift_float) {
6045 ✗ LOAD_OP(opLoadConst, 0.5f, 0);
6046 ✗ TWO_ARG_OP(opAdd); // at the end pixel exit: opSub
6047 }
6048 }
6049 }
6050 ✗ else if (tokens[i] == "pi") // avs+
6051 {
6052 ✗ float pi = 3.141592653589793f;
6053 ✗ LOAD_OP(opLoadConst, pi, 0);
6054 }
6055 // avs+
6056 // bitdepth: automatic silent parameter of the lut expression(clip bit depth)
6057 // clip-specific bitdepths: bitdepth_x, bitdepth_y, bitdepth_z, bitdepth_a, .. bitdepth_w,
6058 // sbitdepth : automatic silent parameter of the lut expression(bit depth of values to scale)
6059 //
6060 // pre-defined, bit depth aware constants
6061 // range_half : autoscaled 128 or 0.5 for float, (or 0.0 for chroma with zero-base float chroma version)
6062 // range_max : 255 / 1023 / 4095 / 16383 / 65535 or 1.0 for float
6063 // 0.5 for float chroma (new zero-based style)
6064 // range_min : 0 for 8-16bits, or 0 for float luma, or -0.5 for float chroma
6065 // -0.5 for float chroma (new zero-based style)
6066 // range_size : 256 / 1024...65536
6067 // ymin, ymax : 16 / 235 autoscaled.
6068 // cmin, cmax : 16 / 240 autoscaled. For 32bits zero based chroma: (16-128)/255.0, (240-128)/255.0
6069
6070 // ymin or ymin_x, ymin_y, ymin_y, ymin_a....
6071 // similarly: ymax, range_max, cmin, cmax, range_half
6072 // without clip index specifier, or with '_'+letter suffix
6073 ✗ else if (tokens[i].substr(0, 8) == "bitdepth") // avs+
6074 {
6075 ✗ int loadIndex = -1;
6076 ✗ std::string toFind = "bitdepth";
6077 ✗ if (tokens[i].substr(0, toFind.length()) == toFind)
6078 ✗ loadIndex = getSuffix(tokens[i], toFind);
6079 ✗ if (loadIndex < 0)
6080 ✗ env->ThrowError("Expr: Error in built-in constant expression '%s'", tokens[i].c_str());
6081 ✗ if (loadIndex >= numInputs)
6082 ✗ env->ThrowError("Expr: Too few input clips supplied for reference '%s'", tokens[i].c_str());
6083
6084 ✗ int bitsPerComponent = getEffectiveBitsPerComponent(vi[loadIndex]->BitsPerComponent(), autoconv_conv_float, autoconv_conv_int, autoScaleSourceBitDepth);
6085 ✗ float q = (float)bitsPerComponent;
6086 ✗ LOAD_OP(opLoadConst, q, 0);
6087 ✗ }
6088 ✗ else if (tokens[i] == "sbitdepth") // avs+
6089 {
6090 ✗ float q = (float)autoScaleSourceBitDepth;
6091 ✗ LOAD_OP(opLoadConst, q, 0);
6092 }
6093 ✗ else if (tokens[i].substr(0, 4) == "ymin") // avs+
6094 {
6095 ✗ int loadIndex = -1;
6096 ✗ std::string toFind = "ymin";
6097 ✗ if (tokens[i].substr(0, toFind.length()) == toFind)
6098 ✗ loadIndex = getSuffix(tokens[i], toFind);
6099 ✗ if (loadIndex < 0)
6100 ✗ env->ThrowError("Expr: Error in built-in constant expression '%s'", tokens[i].c_str());
6101 ✗ if (loadIndex >= numInputs)
6102 ✗ env->ThrowError("Expr: Too few input clips supplied for reference '%s'", tokens[i].c_str());
6103
6104 ✗ int bitsPerComponent = getEffectiveBitsPerComponent(vi[loadIndex]->BitsPerComponent(), autoconv_conv_float, autoconv_conv_int, autoScaleSourceBitDepth);
6105 ✗ float q = bitsPerComponent == 32 ? 16.0f / 255 : (16 << (bitsPerComponent - 8)); // scale luma min 16
6106 ✗ LOAD_OP(opLoadConst, q, 0);
6107 ✗ }
6108 ✗ else if (tokens[i].substr(0, 4) == "ymax") // avs+
6109 {
6110 ✗ int loadIndex = -1;
6111 ✗ std::string toFind = "ymax";
6112 ✗ if (tokens[i].substr(0, toFind.length()) == toFind)
6113 ✗ loadIndex = getSuffix(tokens[i], toFind);
6114 ✗ if (loadIndex < 0)
6115 ✗ env->ThrowError("Expr: Error in built-in constant expression '%s'", tokens[i].c_str());
6116 ✗ if (loadIndex >= numInputs)
6117 ✗ env->ThrowError("Expr: Too few input clips supplied for reference '%s'", tokens[i].c_str());
6118
6119 ✗ int bitsPerComponent = getEffectiveBitsPerComponent(vi[loadIndex]->BitsPerComponent(), autoconv_conv_float, autoconv_conv_int, autoScaleSourceBitDepth);
6120 ✗ float q = bitsPerComponent == 32 ? 235.0f / 255 : (235 << (bitsPerComponent - 8)); // scale luma max 235
6121 ✗ LOAD_OP(opLoadConst, q, 0);
6122 ✗ }
6123 ✗ else if (tokens[i].substr(0, 4) == "cmin") // avs+
6124 {
6125 ✗ int loadIndex = -1;
6126 ✗ std::string toFind = "cmin";
6127 ✗ if (tokens[i].substr(0, toFind.length()) == toFind)
6128 ✗ loadIndex = getSuffix(tokens[i], toFind);
6129 ✗ if (loadIndex < 0)
6130 ✗ env->ThrowError("Expr: Error in built-in constant expression '%s'", tokens[i].c_str());
6131 ✗ if (loadIndex >= numInputs)
6132 ✗ env->ThrowError("Expr: Too few input clips supplied for reference '%s'", tokens[i].c_str());
6133
6134 ✗ int bitsPerComponent = getEffectiveBitsPerComponent(vi[loadIndex]->BitsPerComponent(), autoconv_conv_float, autoconv_conv_int, autoScaleSourceBitDepth);
6135
6136 ✗ float q = bitsPerComponent == 32 ? uv8tof(16) : (16 << (bitsPerComponent - 8)); // scale chroma min 16
6137 ✗ if (shift_float && bitsPerComponent) q += 0.5f;
6138 ✗ LOAD_OP(opLoadConst, q, 0);
6139 ✗ }
6140 ✗ else if (tokens[i].substr(0, 4) == "cmax") // avs+
6141 {
6142 ✗ int loadIndex = -1;
6143 ✗ std::string toFind = "cmax";
6144 ✗ if (tokens[i].substr(0, toFind.length()) == toFind)
6145 ✗ loadIndex = getSuffix(tokens[i], toFind);
6146 ✗ if (loadIndex < 0)
6147 ✗ env->ThrowError("Expr: Error in built-in constant expression '%s'", tokens[i].c_str());
6148 ✗ if (loadIndex >= numInputs)
6149 ✗ env->ThrowError("Expr: Too few input clips supplied for reference '%s'", tokens[i].c_str());
6150
6151 ✗ int bitsPerComponent = getEffectiveBitsPerComponent(vi[loadIndex]->BitsPerComponent(), autoconv_conv_float, autoconv_conv_int, autoScaleSourceBitDepth);
6152 ✗ float q = bitsPerComponent == 32 ? uv8tof(240) : (240 << (bitsPerComponent - 8)); // scale chroma max 240
6153 ✗ if (shift_float && bitsPerComponent == 32) q += 0.5f;
6154 ✗ LOAD_OP(opLoadConst, q, 0);
6155 ✗ }
6156 ✗ else if (tokens[i].substr(0, 10) == "range_size") // avs+
6157 {
6158 ✗ int loadIndex = -1;
6159 ✗ std::string toFind = "range_size";
6160 ✗ if (tokens[i].substr(0, toFind.length()) == toFind)
6161 ✗ loadIndex = getSuffix(tokens[i], toFind);
6162 ✗ if (loadIndex < 0)
6163 ✗ env->ThrowError("Expr: Error in built-in constant expression '%s'", tokens[i].c_str());
6164 ✗ if (loadIndex >= numInputs)
6165 ✗ env->ThrowError("Expr: Too few input clips supplied for reference '%s'", tokens[i].c_str());
6166
6167 ✗ int bitsPerComponent = getEffectiveBitsPerComponent(vi[loadIndex]->BitsPerComponent(), autoconv_conv_float, autoconv_conv_int, autoScaleSourceBitDepth);
6168 ✗ float q = bitsPerComponent == 32 ? 1.0f : (1 << bitsPerComponent); // 1.0, 256, 1024,... 65536
6169 ✗ LOAD_OP(opLoadConst, q, 0);
6170 ✗ }
6171 ✗ else if (tokens[i].substr(0, 9) == "range_min") // avs+ > r2636
6172 {
6173 ✗ int loadIndex = -1;
6174 ✗ std::string toFind = "range_min";
6175 ✗ if (tokens[i].substr(0, toFind.length()) == toFind)
6176 ✗ loadIndex = getSuffix(tokens[i], toFind);
6177 ✗ if (loadIndex < 0)
6178 ✗ env->ThrowError("Expr: Error in built-in constant expression '%s'", tokens[i].c_str());
6179 ✗ if (loadIndex >= numInputs)
6180 ✗ env->ThrowError("Expr: Too few input clips supplied for reference '%s'", tokens[i].c_str());
6181
6182 ✗ int bitsPerComponent = getEffectiveBitsPerComponent(vi[loadIndex]->BitsPerComponent(), autoconv_conv_float, autoconv_conv_int, autoScaleSourceBitDepth);
6183 // 0.0 (or -0.5 for 32bit float chroma)
6184 ✗ float q = bitsPerComponent == 32 ? (chroma ? -0.5f : 0.0f) : 0;
6185 ✗ if (chroma && shift_float && bitsPerComponent == 32) q += 0.5f;
6186 ✗ LOAD_OP(opLoadConst, q, 0);
6187 ✗ }
6188 ✗ else if (tokens[i].substr(0, 10) == "yrange_min")
6189 {
6190 ✗ int loadIndex = -1;
6191 ✗ std::string toFind = "yrange_min";
6192 ✗ if (tokens[i].substr(0, toFind.length()) == toFind)
6193 ✗ loadIndex = getSuffix(tokens[i], toFind);
6194 ✗ if (loadIndex < 0)
6195 ✗ env->ThrowError("Expr: Error in built-in constant expression '%s'", tokens[i].c_str());
6196 ✗ if (loadIndex >= numInputs)
6197 ✗ env->ThrowError("Expr: Too few input clips supplied for reference '%s'", tokens[i].c_str());
6198
6199 ✗ int bitsPerComponent = getEffectiveBitsPerComponent(vi[loadIndex]->BitsPerComponent(), autoconv_conv_float, autoconv_conv_int, autoScaleSourceBitDepth);
6200 ✗ float q = bitsPerComponent == 32 ? 0.0f : 0;
6201 ✗ LOAD_OP(opLoadConst, q, 0);
6202 ✗ }
6203 ✗ else if (tokens[i].substr(0, 9) == "range_max") // avs+
6204 {
6205 ✗ int loadIndex = -1;
6206 ✗ std::string toFind = "range_max";
6207 ✗ if (tokens[i].substr(0, toFind.length()) == toFind)
6208 ✗ loadIndex = getSuffix(tokens[i], toFind);
6209 ✗ if (loadIndex < 0)
6210 ✗ env->ThrowError("Expr: Error in built-in constant expression '%s'", tokens[i].c_str());
6211 ✗ if (loadIndex >= numInputs)
6212 ✗ env->ThrowError("Expr: Too few input clips supplied for reference '%s'", tokens[i].c_str());
6213
6214 ✗ int bitsPerComponent = getEffectiveBitsPerComponent(vi[loadIndex]->BitsPerComponent(), autoconv_conv_float, autoconv_conv_int, autoScaleSourceBitDepth);
6215 // 1.0 (or 0.5 for 32bit float chroma), 255, 1023,... 65535
6216 ✗ float q = bitsPerComponent == 32 ? (chroma ? + 0.5f : 1.0f) : ((1 << bitsPerComponent) - 1);
6217 ✗ if (chroma && shift_float && bitsPerComponent == 32) q += 0.5f;
6218 ✗ LOAD_OP(opLoadConst, q, 0);
6219 ✗ }
6220 ✗ else if (tokens[i].substr(0, 10) == "yrange_max")
6221 {
6222 ✗ int loadIndex = -1;
6223 ✗ std::string toFind = "yrange_max";
6224 ✗ if (tokens[i].substr(0, toFind.length()) == toFind)
6225 ✗ loadIndex = getSuffix(tokens[i], toFind);
6226 ✗ if (loadIndex < 0)
6227 ✗ env->ThrowError("Expr: Error in built-in constant expression '%s'", tokens[i].c_str());
6228 ✗ if (loadIndex >= numInputs)
6229 ✗ env->ThrowError("Expr: Too few input clips supplied for reference '%s'", tokens[i].c_str());
6230
6231 ✗ int bitsPerComponent = getEffectiveBitsPerComponent(vi[loadIndex]->BitsPerComponent(), autoconv_conv_float, autoconv_conv_int, autoScaleSourceBitDepth);
6232 // 1.0, 255, 1023,... 65535
6233 ✗ float q = bitsPerComponent == 32 ? 1.0f : ((1 << bitsPerComponent) - 1);
6234 ✗ LOAD_OP(opLoadConst, q, 0);
6235 ✗ }
6236 ✗ else if (tokens[i].substr(0, 10) == "range_half") // avs+
6237 {
6238 ✗ int loadIndex = -1;
6239 ✗ std::string toFind = "range_half";
6240 ✗ if (tokens[i].substr(0, toFind.length()) == toFind)
6241 ✗ loadIndex = getSuffix(tokens[i], toFind);
6242 ✗ if (loadIndex < 0)
6243 ✗ env->ThrowError("Expr: Error in built-in constant expression '%s'", tokens[i].c_str());
6244 ✗ if (loadIndex >= numInputs)
6245 ✗ env->ThrowError("Expr: Too few input clips supplied for reference '%s'", tokens[i].c_str());
6246
6247 // for chroma: range_half is 0.0 for 32bit float (or 0.5 for old float chroma representation)
6248 ✗ int bitsPerComponent = getEffectiveBitsPerComponent(vi[loadIndex]->BitsPerComponent(), autoconv_conv_float, autoconv_conv_int, autoScaleSourceBitDepth);
6249 ✗ float q = bitsPerComponent == 32 ? (chroma ? uv8tof(128) : 0.5f) : (1 << (bitsPerComponent - 1)); // 0.5f, 128, 512, ... 32768
6250 ✗ if (chroma && shift_float && bitsPerComponent == 32) q += 0.5f;
6251 ✗ LOAD_OP(opLoadConst, q, 0);
6252 ✗ }
6253 ✗ else if (tokens[i].substr(0, 11) == "yrange_half") // avs+
6254 {
6255 ✗ int loadIndex = -1;
6256 ✗ std::string toFind = "yrange_half";
6257 ✗ if (tokens[i].substr(0, toFind.length()) == toFind)
6258 ✗ loadIndex = getSuffix(tokens[i], toFind);
6259 ✗ if (loadIndex < 0)
6260 ✗ env->ThrowError("Expr: Error in built-in constant expression '%s'", tokens[i].c_str());
6261 ✗ if (loadIndex >= numInputs)
6262 ✗ env->ThrowError("Expr: Too few input clips supplied for reference '%s'", tokens[i].c_str());
6263
6264 ✗ int bitsPerComponent = getEffectiveBitsPerComponent(vi[loadIndex]->BitsPerComponent(), autoconv_conv_float, autoconv_conv_int, autoScaleSourceBitDepth);
6265 ✗ float q = bitsPerComponent == 32 ? 0.5f : (1 << (bitsPerComponent - 1)); // 0.5f, 128, 512, ... 32768
6266 ✗ LOAD_OP(opLoadConst, q, 0);
6267 ✗ }
6268 // "scaleb" and "scalef" functions scale their operand from 8 bit to the bit depth of the first clip.
6269 // "i8", "i10", "i14", "i16" and "f32" (typically at the beginning of the expression) sets the scale-base to 8..16 bits or float, respectively.
6270 // "i8".."f32" keywords can appear anywhere in the expression, but only the last occurence will be effective for the whole expression.
6271 ✗ else if (tokens[i] == "scaleb" || tokens[i] == "yscaleb" || tokens[i] == "scalef" || tokens[i] == "yscalef") // avs+, scale by bit shift
6272 {
6273 // Note: if 'scale_float' is used then all float input is automatically converted to integer
6274 // in this case the targetBitDepth is not 32 for float clips but the actual autoscaleSourceBitDepth
6275 ✗ int effectivetargetBitDepth = getEffectiveBitsPerComponent(targetBitDepth, autoconv_conv_float, autoconv_conv_int, autoScaleSourceBitDepth);
6276 // number to scale is not chroma-related one even if we are in chroma (U,V) plane
6277 ✗ const bool forceNonUV = (tokens[i] == "yscaleb") || (tokens[i] == "yscalef");
6278
6279 ✗ const int srcBitDepth = autoScaleSourceBitDepth;
6280 ✗ const int dstBitDepth = effectivetargetBitDepth;
6281 ✗ const bool use_chroma = chroma && !forceNonUV;
6282
6283 ✗ const bool isfull = tokens[i] == "scalef" || tokens[i] == "yscalef";
6284
6285 ✗ bits_conv_constants d;
6286 ✗ get_bits_conv_constants(d, use_chroma, isfull, isfull, srcBitDepth, dstBitDepth);
6287 // chroma is spec: signed. Limited:16-240 is really 128 +/-112. Full:1-255 is really 128+/-127
6288
6289 // floatUV
6290 ✗ if (use_chroma && shift_float && dstBitDepth == 32 && srcBitDepth < 32) {
6291 ✗ d.dst_offset = 0.5f; // get out from -0.5..0.5 to 0..1
6292 }
6293
6294 ✗ if (d.src_offset != 0) {
6295 ✗ LOAD_OP(opLoadConst, (float)d.src_offset, 0);
6296 ✗ TWO_ARG_OP(opSub);
6297 }
6298 ✗ if (d.mul_factor != 1.0f) {
6299 ✗ LOAD_OP(opLoadConst, d.mul_factor, 0);
6300 ✗ TWO_ARG_OP(opMul);
6301 }
6302 ✗ if (d.dst_offset != 0) {
6303 ✗ LOAD_OP(opLoadConst, (float)d.dst_offset, 0);
6304 ✗ TWO_ARG_OP(opAdd);
6305 }
6306 }
6307 ✗ else if (tokens[i] == "i8") // avs+
6308 {
6309 ✗ autoScaleSourceBitDepth = 8;
6310 }
6311 ✗ else if (tokens[i] == "i10") // avs+
6312 {
6313 ✗ autoScaleSourceBitDepth = 10;
6314 }
6315 ✗ else if (tokens[i] == "i12") // avs+
6316 {
6317 ✗ autoScaleSourceBitDepth = 12;
6318 }
6319 ✗ else if (tokens[i] == "i14") // avs+
6320 {
6321 ✗ autoScaleSourceBitDepth = 14;
6322 }
6323 ✗ else if (tokens[i] == "i16") // avs+
6324 {
6325 ✗ autoScaleSourceBitDepth = 16;
6326 }
6327 ✗ else if (tokens[i] == "f32") // avs+
6328 {
6329 ✗ autoScaleSourceBitDepth = 32;
6330 }
6331 ✗ else if (tokens[i].length() > 1 && tokens[i][0] >= 'a' && tokens[i][0] <= 'z' && tokens[i][1] == '.') {
6332 // frame property access: x.framePropName syntax
6333 ✗ char srcChar = tokens[i][0];
6334 int srcIndex;
6335 ✗ if (srcChar >= 'x')
6336 ✗ srcIndex = srcChar - 'x';
6337 else
6338 ✗ srcIndex = srcChar - 'a' + 3;
6339 ✗ if (srcIndex >= numInputs)
6340 ✗ env->ThrowError("Expr: Too few input clips supplied to reference '%s'", tokens[i].c_str());
6341
6342 ✗ auto fullname = tokens[i];
6343 ✗ auto fpname = tokens[i].substr(2); // after '.'
6344 ✗ if(fpname.length() == 0)
6345 ✗ env->ThrowError("Expr: no frame property name is specified");
6346 ✗ if (!isValidVarName(fpname))
6347 ✗ env->ThrowError("Expr: invalid frame property name '%s'", fpname.c_str());
6348
6349 // check if frame property already existed in a variable slots
6350 ✗ auto key = fullname;
6351 ✗ auto it = fpnames.find(key);
6352 int loadIndex;
6353 ✗ if (it == fpnames.end()) {
6354 // first occurance, insert name and actual index
6355 ✗ if (fpindex >= MAX_FRAMEPROP_VARIABLES)
6356 ✗ env->ThrowError("Expr: too many frame property references, maximum reached (%d)", MAX_FRAMEPROP_VARIABLES);
6357 ✗ loadIndex = fpindex++;
6358 ✗ fpnames[key] = loadIndex;
6359 // Register into the list of frame properties to read
6360 // input clip index: srcIndex
6361 // name of frame property: fpname
6362 // variable index to fill: loadindex
6363 ✗ ExprFramePropData fpData;
6364 ✗ fpData.name = fpname;
6365 ✗ fpData.srcIndex = srcIndex;
6366 ✗ fpData.var_index = loadIndex;
6367 ✗ fp.push_back(fpData);
6368 ✗ }
6369 else {
6370 ✗ loadIndex = it->second;
6371 }
6372 ✗ LOAD_OP(opLoadFramePropVar, loadIndex, 0);
6373 ✗ }
6374 ✗ else if (tokenlen >= 2 && (tokens[i][tokenlen - 1] == '^' || tokens[i][tokenlen - 1] == '@'))
6375 {
6376 // storing a variable: A@ .. Z@
6377 // storing a variable and remove from stack: A^..Z^
6378 ✗ auto key = tokens[i].substr(0, tokenlen - 1);
6379 ✗ auto opchar = tokens[i][tokenlen - 1];
6380 ✗ auto it = varnames.find(key);
6381 int loadIndex;
6382 ✗ if (it == varnames.end()) {
6383 ✗ if(!isValidVarName(key))
6384 ✗ env->ThrowError("Expr: invalid variable name '%s'", key.c_str());
6385 // first occurance, insert name and actual index
6386 ✗ if (varindex >= MAX_USER_VARIABLES)
6387 ✗ env->ThrowError("Expr: too many variables, maximum reached (%d)", MAX_USER_VARIABLES);
6388 ✗ loadIndex = varindex++;
6389 ✗ varnames[key] = loadIndex;
6390 }
6391 else {
6392 ✗ loadIndex = it->second;
6393 }
6394 ✗ if (opchar == '^')
6395 ✗ VAR_STORE_SPEC_OP(opStoreVarAndDrop1, loadIndex);
6396 else // if (opchar == '@')
6397 ✗ VAR_STORE_OP(opStoreVar, loadIndex);
6398 ✗ }
6399 ✗ else if (isValidVarName(tokens[i]))
6400 {
6401 // variable names
6402 // the very end, all reserved expr words were processed
6403 ✗ auto key = tokens[i];
6404 ✗ auto it = varnames.find(key);
6405 ✗ if (it == varnames.end())
6406 ✗ env->ThrowError("Expr: keyword or variable not found: '%s'", key.c_str());
6407 ✗ auto loadIndex = it->second;
6408 ✗ LOAD_OP(opLoadVar, loadIndex, 0);
6409 ✗ }
6410 else {
6411 // parse a number
6412 float f;
6413 ✗ std::string s;
6414 ✗ std::istringstream numStream(tokens[i]);
6415 ✗ numStream.imbue(std::locale::classic());
6416 ✗ if (!(numStream >> f))
6417 ✗ env->ThrowError("Expr: Failed to convert '%s' to float", tokens[i].c_str());
6418 ✗ if (numStream >> s)
6419 ✗ env->ThrowError("Expr: Failed to convert '%s' to float, not the whole token could be converted", tokens[i].c_str());
6420 ✗ LOAD_OP(opLoadConst, f, 0);
6421 ✗ }
6422 }
6423
6424 ✗ if (tokens.size() > 0) {
6425 ✗ if (stackSize != 1)
6426 ✗ env->ThrowError("Expr: Stack unbalanced at end of expression (size=%zu). Need to have exactly one value on the stack to return.", stackSize);
6427
6428 // When scale_inputs option was used for scaling input to a common internal range,
6429 // we have to scale pixels before storing them back
6430 // need any conversion?
6431 // or use effectiveTargetBitDepth instead of autoScaleSourceBitDepth
6432 ✗ const int srcBitDepth = autoScaleSourceBitDepth;
6433 ✗ const int dstBitDepth = targetBitDepth;
6434 ✗ const bool use_chroma = chroma; // && !forceNonUV;
6435
6436 ✗ const bool isfull = autoconv_full_scale;
6437
6438 ✗ if (autoconv_conv_int || autoconv_conv_float) {
6439
6440 ✗ if ((targetBitDepth != 32 && autoconv_conv_int) ||
6441 ✗ ((targetBitDepth == 32 && autoconv_conv_float))) {
6442
6443 ✗ bits_conv_constants d;
6444 ✗ get_bits_conv_constants(d, use_chroma, isfull, isfull, srcBitDepth, dstBitDepth);
6445 // chroma is spec: signed. Limited:16-240 is really 128 +/-112. Full:1-255 is really 128+/-127
6446
6447 ✗ if (d.src_offset != 0) {
6448 ✗ LOAD_OP_NOTOKEN(opLoadConst, (float)d.src_offset, 0);
6449 ✗ TWO_ARG_OP_NOTOKEN(opSub);
6450 }
6451 ✗ if (d.mul_factor != 1.0f) {
6452 ✗ LOAD_OP_NOTOKEN(opLoadConst, d.mul_factor, 0);
6453 ✗ TWO_ARG_OP_NOTOKEN(opMul);
6454 }
6455 ✗ if (d.dst_offset != 0) {
6456 ✗ LOAD_OP_NOTOKEN(opLoadConst, (float)d.dst_offset, 0);
6457 ✗ TWO_ARG_OP_NOTOKEN(opAdd);
6458 }
6459 }
6460 }
6461 // "floatUV" -
6462 // this reverses the effect of the 0.5 float-type pre-shift is the pixel load phase
6463 // pre-shifts chroma pixels by +0.5 before applying the expression (see at pixel load),
6464 // then here the result is shifted back by -0.5
6465 // Thus expressions, which rely on a working range of 0..1.0 will work transparently
6466 ✗ if ((!autoconv_conv_float || srcBitDepth == 32) && targetBitDepth == 32) {
6467 ✗ if (chroma && shift_float) {
6468 ✗ LOAD_OP_NOTOKEN(opLoadConst, 0.5f, 0);
6469 ✗ TWO_ARG_OP_NOTOKEN(opSub); // at pixel load it was opAdd
6470 }
6471 }
6472
6473 ✗ if (clamp_float_i > 0 && targetBitDepth == 32) {
6474 ✗ if (chroma) {
6475 // clamp_float clamp_float_uv -> clamp_float_i clamp range for Y clamp range for UV
6476 // false x 0 0..1 -0.5..+0.5
6477 // true false 1 0..1 -0.5..+0.5
6478 // true true 2 0..1 0..1
6479 ✗ LOAD_OP_NOTOKEN(opLoadConst, clamp_float_i == 2 ? 0.0f : -0.5f, 0);
6480 ✗ TWO_ARG_OP_NOTOKEN(opMax);
6481 ✗ LOAD_OP_NOTOKEN(opLoadConst, clamp_float_i == 2 ? 1.0f : 0.5f, 0);
6482 ✗ TWO_ARG_OP_NOTOKEN(opMin);
6483 }
6484 else
6485 { // luma
6486 ✗ LOAD_OP_NOTOKEN(opLoadConst, 0.0f, 0);
6487 ✗ TWO_ARG_OP_NOTOKEN(opMax);
6488 ✗ LOAD_OP_NOTOKEN(opLoadConst, 1.0f, 0);
6489 ✗ TWO_ARG_OP_NOTOKEN(opMin);
6490 }
6491 }
6492
6493 // and finally store it
6494 ✗ ops.push_back(storeOp);
6495 }
6496
6497 ✗ return maxStackSize;
6498 ✗ }
6499
6500 ✗ static float calculateOneOperand(uint32_t op, float a) {
6501 ✗ switch (op) {
6502 ✗ case opSqrt:
6503 ✗ return std::sqrt(a);
6504 ✗ case opAbs:
6505 ✗ return std::abs(a);
6506 ✗ case opSgn:
6507 ✗ return (a < 0) ? -1.0f : (a > 0) ? 1.0f : 0.0f;
6508 ✗ case opNeg: // Expr "boolean": not
6509 ✗ return (a > 0) ? 0.0f : 1.0f;
6510 ✗ case opNegSign:
6511 ✗ return -a;
6512 ✗ case opExp:
6513 ✗ return std::exp(a);
6514 ✗ case opLog:
6515 ✗ return std::log(a);
6516 ✗ case opSin:
6517 ✗ return std::sin(a);
6518 ✗ case opCos:
6519 ✗ return std::cos(a);
6520 ✗ case opTan:
6521 ✗ return std::tan(a);
6522 ✗ case opAsin:
6523 ✗ return std::asin(a);
6524 ✗ case opAcos:
6525 ✗ return std::acos(a);
6526 ✗ case opAtan:
6527 ✗ return std::atan(a);
6528 ✗ case opRound:
6529 ✗ return std::round(a);
6530 ✗ case opFloor:
6531 ✗ return std::floor(a);
6532 ✗ case opCeil:
6533 ✗ return std::ceil(a);
6534 ✗ case opTrunc:
6535 ✗ return std::trunc(a);
6536 }
6537
6538 ✗ return 0.0f;
6539 }
6540
6541 ✗ static float calculateTwoOperands(uint32_t op, float a, float b) {
6542 ✗ switch (op) {
6543 ✗ case opAdd:
6544 ✗ return a + b;
6545 ✗ case opSub:
6546 ✗ return a - b;
6547 ✗ case opMul:
6548 ✗ return a * b;
6549 ✗ case opDiv:
6550 ✗ return a / b;
6551 ✗ case opFmod:
6552 ✗ return std::fmod(a, b);
6553 ✗ case opMax:
6554 ✗ return std::max(a, b);
6555 ✗ case opMin:
6556 ✗ return std::min(a, b);
6557 ✗ case opGt:
6558 ✗ return (a > b) ? 1.0f : 0.0f;
6559 ✗ case opLt:
6560 ✗ return (a < b) ? 1.0f : 0.0f;
6561 ✗ case opEq:
6562 ✗ return (a == b) ? 1.0f : 0.0f;
6563 ✗ case opNotEq:
6564 ✗ return (a != b) ? 1.0f : 0.0f;
6565 ✗ case opLE:
6566 ✗ return (a <= b) ? 1.0f : 0.0f;
6567 ✗ case opGE:
6568 ✗ return (a >= b) ? 1.0f : 0.0f;
6569 ✗ case opAnd:
6570 ✗ return (a > 0 && b > 0) ? 1.0f : 0.0f;
6571 ✗ case opOr:
6572 ✗ return (a > 0 || b > 0) ? 1.0f : 0.0f;
6573 ✗ case opXor:
6574 ✗ return ((a > 0) != (b > 0)) ? 1.0f : 0.0f;
6575 ✗ case opPow:
6576 ✗ return std::pow(a, b);
6577 ✗ case opAtan2:
6578 ✗ return std::atan2(a, b);
6579 }
6580
6581 ✗ return 0.0f;
6582 }
6583
6584 ✗ static int numOperands(uint32_t op) {
6585 ✗ switch (op) {
6586 ✗ case opLoadConst:
6587 case opLoadSrc8:
6588 case opLoadSrc16:
6589 case opLoadSrcF32:
6590 case opLoadSrcF16:
6591 case opLoadRelSrc8:
6592 case opLoadRelSrc16:
6593 case opLoadRelSrcF32:
6594 case opDup:
6595 case opLoadSpatialX:
6596 case opLoadSpatialY:
6597 case opLoadVar:
6598 case opLoadFramePropVar:
6599 case opLoadInternalVar:
6600 case opSwap:
6601 case opStoreVar:
6602 case opStoreVarAndDrop1:
6603 ✗ return 0;
6604
6605 ✗ case opSqrt:
6606 case opAbs:
6607 case opSgn:
6608 case opNeg:
6609 case opNegSign:
6610 case opExp:
6611 case opLog:
6612 case opSin:
6613 case opCos:
6614 case opTan:
6615 case opAsin:
6616 case opAcos:
6617 case opAtan:
6618 case opRound:
6619 case opFloor:
6620 case opCeil:
6621 case opTrunc:
6622 ✗ return 1;
6623
6624 ✗ case opAdd:
6625 case opSub:
6626 case opMul:
6627 case opDiv:
6628 case opFmod:
6629 case opMax:
6630 case opMin:
6631 case opGt:
6632 case opLt:
6633 case opEq:
6634 case opNotEq:
6635 case opLE:
6636 case opGE:
6637 case opAnd:
6638 case opOr:
6639 case opXor:
6640 case opPow:
6641 case opAtan2:
6642 ✗ return 2;
6643
6644 ✗ case opTernary:
6645 case opClip:
6646 ✗ return 3;
6647 }
6648
6649 ✗ return 0;
6650 }
6651
6652 ✗ static bool isLoadOp(uint32_t op) {
6653 ✗ switch (op) {
6654 ✗ case opLoadConst:
6655 case opLoadSrc8:
6656 case opLoadSrc16:
6657 case opLoadSrcF32:
6658 case opLoadSrcF16:
6659 case opLoadRelSrc8:
6660 case opLoadRelSrc16:
6661 case opLoadRelSrcF32:
6662 case opLoadSpatialX:
6663 case opLoadSpatialY:
6664 case opLoadVar:
6665 case opLoadFramePropVar:
6666 case opLoadInternalVar:
6667 ✗ return true;
6668 }
6669
6670 ✗ return false;
6671 }
6672
6673 ✗ static void findBranches(std::vector<ExprOp>& ops, size_t pos, size_t* start1, size_t* start2, size_t* start3) {
6674 ✗ int operands = numOperands(ops[pos].op);
6675
6676 size_t temp1, temp2, temp3;
6677
6678 ✗ if (operands == 0) {
6679 // dup loadsrc loadrel loadconst swap, storevar
6680 ✗ if (ops[pos].op == opSwap || ops[pos].op == opStoreVar || ops[pos].op == opStoreVarAndDrop1)
6681 {
6682 ✗ findBranches(ops, pos - 1, &temp1, &temp2, &temp3);
6683 ✗ *start1 = temp1;
6684 ✗ if (ops[pos].op == opStoreVarAndDrop1) {
6685 // StoreAndPopVar: opStoreVar + Ignore topmost stack.
6686 // Branch was calculated only for storing its result into var.
6687 // Leaves no trace on stack.
6688 // So we go on with next branch
6689 ✗ pos = *start1;
6690 ✗ findBranches(ops, pos - 1, &temp1, &temp2, &temp3);
6691 ✗ *start1 = temp1;
6692 }
6693 }
6694 else
6695 ✗ *start1 = pos;
6696 }
6697 ✗ else if (operands == 1) {
6698 ✗ if (isLoadOp(ops[pos - 1].op)) {
6699 ✗ *start1 = pos - 1;
6700 }
6701 else {
6702 ✗ findBranches(ops, pos - 1, &temp1, &temp2, &temp3);
6703 ✗ *start1 = temp1;
6704 }
6705 }
6706 ✗ else if (operands == 2) {
6707 ✗ if (isLoadOp(ops[pos - 1].op)) {
6708 ✗ *start2 = pos - 1;
6709 }
6710 else {
6711 ✗ findBranches(ops, pos - 1, &temp1, &temp2, &temp3);
6712 ✗ *start2 = temp1;
6713 }
6714
6715 ✗ if (isLoadOp(ops[*start2 - 1].op)) {
6716 ✗ *start1 = *start2 - 1;
6717 }
6718 else {
6719 ✗ findBranches(ops, *start2 - 1, &temp1, &temp2, &temp3);
6720 ✗ *start1 = temp1;
6721 }
6722 }
6723 ✗ else if (operands == 3) {
6724 ✗ if (isLoadOp(ops[pos - 1].op)) {
6725 ✗ *start3 = pos - 1;
6726 }
6727 else {
6728 ✗ findBranches(ops, pos - 1, &temp1, &temp2, &temp3);
6729 ✗ *start3 = temp1;
6730 }
6731
6732 ✗ if (isLoadOp(ops[*start3 - 1].op)) {
6733 ✗ *start2 = *start3 - 1;
6734 }
6735 else {
6736 ✗ findBranches(ops, *start3 - 1, &temp1, &temp2, &temp3);
6737 ✗ *start2 = temp1;
6738 }
6739
6740 ✗ if (isLoadOp(ops[*start2 - 1].op)) {
6741 ✗ *start1 = *start2 - 1;
6742 }
6743 else {
6744 ✗ findBranches(ops, *start2 - 1, &temp1, &temp2, &temp3);
6745 ✗ *start1 = temp1;
6746 }
6747 }
6748 ✗ }
6749
6750
6751 ✗ static void foldConstants(std::vector<ExprOp> &ops) {
6752 ✗ for (size_t i = 0; i < ops.size(); i++) {
6753 ✗ switch (ops[i].op) {
6754 // optimize pow
6755 ✗ case opPow:
6756 ✗ if (ops[i - 1].op == opLoadConst) {
6757 ✗ if (ops[i - 1].e.fval == 0.5f) {
6758 // replace pow 0.5 with sqrt
6759 ✗ ops[i].op = opSqrt;
6760 ✗ ops.erase(ops.begin() + i - 1);
6761 ✗ i--;
6762 }
6763 ✗ else if (ops[i - 1].e.fval == 1.0f) {
6764 // replace pow 1 with nothing
6765 ✗ ops.erase(ops.begin() + i - 1, ops.begin() + i + 1);
6766 ✗ i -= 2;
6767 }
6768 ✗ else if (ops[i - 1].e.fval == 2.0f) {
6769 // replace pow 2 with dup *
6770 ✗ ops[i].op = opMul;
6771 ✗ ops[i - 1].op = opDup; ops[i - 1].e.ival = 0; // dup 0
6772 ✗ i--;
6773 }
6774 ✗ else if (ops[i - 1].e.fval == 3.0f) {
6775 // replace pow 3 with dup dup * *
6776 ✗ ops[i].op = opMul;
6777 ✗ ops[i - 1].op = opMul;
6778 ✗ ExprOp extraDup(opDup, 0);
6779 ✗ ops.insert(ops.begin() + i - 1, extraDup);
6780 ✗ ops.insert(ops.begin() + i - 1, extraDup);
6781 ✗ i--;
6782 }
6783 ✗ else if (ops[i - 1].e.fval == 4.0f) {
6784 // replace pow 4 with dup * dup *
6785 ✗ ops[i].op = opMul;
6786 ✗ ops[i - 1].op = opDup; ops[i - 1].e.ival = 0; // dup 0
6787 ✗ ExprOp extraMul(opMul);
6788 ✗ ExprOp extraDup(opDup, 0);
6789 ✗ ops.insert(ops.begin() + i - 1, extraMul);
6790 ✗ ops.insert(ops.begin() + i - 1, extraDup);
6791 ✗ i--;
6792 }
6793 }
6794 ✗ break;
6795 // optimize Mul 1 Div 1, Mul -1, Div -1
6796 ✗ case opMul: case opDiv:
6797 ✗ if (ops[i - 1].op == opLoadConst) {
6798 ✗ if (ops[i - 1].e.fval == 1.0f) {
6799 // replace mul 1 or div 1 with nothing
6800 ✗ ops.erase(ops.begin() + i - 1, ops.begin() + i + 1);
6801 ✗ i -= 2;
6802 }
6803 ✗ else if (ops[i - 1].e.fval == -1.0f) {
6804 // replace mul -1 or div -1 with neg
6805 ✗ ops[i].op = opNegSign;
6806 ✗ ops.erase(ops.begin() + i - 1);
6807 ✗ i--;
6808 }
6809 }
6810 ✗ break;
6811 // optimize Add 0 or Sub 0
6812 ✗ case opAdd: case opSub:
6813 ✗ if (ops[i - 1].op == opLoadConst) {
6814 ✗ if (ops[i - 1].e.fval == 0.0f) {
6815 // replace add 0 or sub 0 with nothing
6816 ✗ ops.erase(ops.begin() + i - 1, ops.begin() + i + 1);
6817 ✗ i -= 2;
6818 }
6819 }
6820 ✗ break;
6821
6822 }
6823
6824 // fold constant
6825 ✗ switch (ops[i].op) {
6826 ✗ case opDup:
6827 ✗ if (ops[i - 1].op == opLoadConst && ops[i].e.ival == 0) {
6828 ✗ ops[i] = ops[i - 1];
6829 }
6830 ✗ break;
6831
6832 ✗ case opSqrt:
6833 case opAbs:
6834 case opSgn:
6835 case opNeg:
6836 case opNegSign:
6837 case opExp:
6838 case opLog:
6839 case opSin:
6840 case opCos:
6841 case opTan:
6842 case opAsin:
6843 case opAcos:
6844 case opAtan:
6845 case opRound:
6846 case opFloor:
6847 case opCeil:
6848 case opTrunc:
6849 ✗ if (ops[i - 1].op == opLoadConst) {
6850 ✗ ops[i].e.fval = calculateOneOperand(ops[i].op, ops[i - 1].e.fval);
6851 ✗ ops[i].op = opLoadConst;
6852 ✗ ops.erase(ops.begin() + i - 1);
6853 ✗ i--;
6854 }
6855 ✗ break;
6856
6857 ✗ case opSwap:
6858 ✗ if (ops[i - 2].op == opLoadConst && ops[i - 1].op == opLoadConst && ops[i].e.ival == 1) {
6859 ✗ const float temp = ops[i - 2].e.fval;
6860 ✗ ops[i - 2].e.fval = ops[i - 1].e.fval;
6861 ✗ ops[i - 1].e.fval = temp;
6862 ✗ ops.erase(ops.begin() + i);
6863 ✗ i--;
6864 }
6865 ✗ break;
6866
6867 ✗ case opAdd:
6868 case opSub:
6869 case opMul:
6870 case opDiv:
6871 case opFmod:
6872 case opMax:
6873 case opMin:
6874 case opGt:
6875 case opLt:
6876 case opEq:
6877 case opNotEq:
6878 case opLE:
6879 case opGE:
6880 case opAnd:
6881 case opOr:
6882 case opXor:
6883 case opPow:
6884 case opAtan2:
6885 ✗ if (ops[i - 2].op == opLoadConst && ops[i - 1].op == opLoadConst) {
6886 ✗ ops[i].e.fval = calculateTwoOperands(ops[i].op, ops[i - 2].e.fval, ops[i - 1].e.fval);
6887 ✗ ops[i].op = opLoadConst;
6888 ✗ ops.erase(ops.begin() + i - 2, ops.begin() + i);
6889 ✗ i -= 2;
6890 }
6891 ✗ break;
6892
6893 ✗ case opTernary:
6894 size_t start1, start2, start3;
6895 ✗ findBranches(ops, i, &start1, &start2, &start3);
6896 // start1/2/3: condition/true/false branch
6897 ✗ if (ops[start2 - 1].op == opLoadConst) { // condition expression is a single constant
6898 ✗ ops.erase(ops.begin() + i); // erase ternary op
6899 ✗ if (ops[start1].e.fval > 0.0f) { // condition is constant 'true'
6900 // start1 is start2 - 1
6901 ✗ ops.erase(ops.begin() + start3, ops.begin() + i); // erase 'false' branch
6902 ✗ i = start3;
6903 } else {
6904 ✗ ops.erase(ops.begin() + start2, ops.begin() + start3); // erase 'true' branch
6905 ✗ i -= start3 - start2;
6906 }
6907 ✗ ops.erase(ops.begin() + start1); // erase constant
6908 ✗ i -= 2;
6909 }
6910 ✗ break;
6911 }
6912 }
6913 ✗ }
6914
6915 ✗ Exprfilter::Exprfilter(const std::vector<PClip>& _child_array, const std::vector<std::string>& _expr_array, const char *_newformat, const bool _optAvx2,
6916 ✗ const bool _optSingleMode, const bool _optSSE2, const bool _optVectorC, const std::string _scale_inputs, const int _clamp_float_i, const int _lutmode, IScriptEnvironment *env) :
6917 ✗ children(_child_array), expressions(_expr_array), optAvx2(_optAvx2), optSingleMode(_optSingleMode), optSSE2(_optSSE2),
6918 ✗ optVectorC(_optVectorC), scale_inputs(_scale_inputs), clamp_float_i(_clamp_float_i), lutmode(_lutmode) {
6919
6920 ✗ vi = children[0]->GetVideoInfo();
6921 ✗ d.vi = vi;
6922 ✗ if (lutmode < 0 || lutmode>2)
6923 ✗ env->ThrowError("'Expr: 'lut' can be 0 (no lut), 1 (lut_x) or 2 (lut_xy)");
6924 ✗ if (lutmode == 1 && vi.BitsPerComponent() == 32)
6925 ✗ lutmode = 0; // fallback to realtime
6926 ✗ if (lutmode == 2 && vi.BitsPerComponent() > 14)
6927 ✗ lutmode = 0; // fallback to realtime
6928 ✗ d.lutmode = lutmode;
6929
6930 ✗ for (int i = 0; i < 4; i++)
6931 ✗ d.luts[i] = nullptr;
6932
6933 // parse "scale_inputs"
6934 ✗ autoconv_full_scale = false;
6935 ✗ autoconv_conv_float = false;
6936 ✗ autoconv_conv_int = false;
6937 ✗ shift_float = false;
6938
6939 ✗ if (scale_inputs == "allf") {
6940 ✗ autoconv_full_scale = true;
6941 ✗ autoconv_conv_int = true;
6942 ✗ autoconv_conv_float = true;
6943 }
6944 ✗ else if (scale_inputs == "intf") {
6945 ✗ autoconv_full_scale = true;
6946 ✗ autoconv_conv_int = true;
6947 }
6948 ✗ else if (scale_inputs == "floatf") {
6949 ✗ autoconv_full_scale = true;
6950 ✗ autoconv_conv_float = true;
6951 }
6952 ✗ else if (scale_inputs == "all") {
6953 ✗ autoconv_conv_int = true;
6954 ✗ autoconv_conv_float = true;
6955 }
6956 ✗ else if (scale_inputs == "int") {
6957 ✗ autoconv_conv_int = true;
6958 }
6959 ✗ else if (scale_inputs == "float") {
6960 ✗ autoconv_conv_float = true;
6961 }
6962 ✗ else if (scale_inputs == "floatuv") {
6963 ✗ autoconv_conv_float = false; // !! really
6964 // like in masktools2 2.2.20+
6965 ✗ shift_float = true; // !!
6966 }
6967 ✗ else if (scale_inputs != "none") {
6968 ✗ env->ThrowError("Expr: scale_inputs must be 'all','allf','int','intf','float','floatf','floatUV' or 'none'");
6969 }
6970
6971 try {
6972 ✗ d.numInputs = (int)children.size(); // d->numInputs = vsapi->propNumElements(in, "clips");
6973 ✗ if (d.numInputs > 26)
6974 ✗ env->ThrowError("Expr: More than 26 input clips provided");
6975
6976 ✗ for (int i = 0; i < d.numInputs; i++)
6977 ✗ d.clips[i] = children[i];
6978
6979 ✗ if (d.lutmode > 0) {
6980 ✗ if(d.numInputs != d.lutmode)
6981 ✗ env->ThrowError("Expr lut: number of input clips must be the same as LUT's dimension. LUT is %dD. Passed clip(s): %d", d.lutmode, d.numInputs);
6982 }
6983
6984 // checking formats
6985 ✗ const VideoInfo* vi_array[MAX_EXPR_INPUTS] = {};
6986 ✗ for (int i = 0; i < d.numInputs; i++)
6987 ✗ if (d.clips[i])
6988 ✗ vi_array[i] = &d.clips[i]->GetVideoInfo();
6989
6990
6991 ✗ int planes_y[4] = { PLANAR_Y, PLANAR_U, PLANAR_V, PLANAR_A };
6992 ✗ int planes_r[4] = { PLANAR_G, PLANAR_B, PLANAR_R, PLANAR_A }; // for checking GBR order is OK
6993 ✗ int *plane_enums = (d.vi.IsYUV() || d.vi.IsYUVA()) ? planes_y : planes_r;
6994 ✗ const int plane_enum = plane_enums[1]; // for subsampling check U only
6995
6996 // check all clips against first one
6997 ✗ for (int i = 0; i < d.numInputs; i++) {
6998
6999 ✗ int *plane_enums_i = (vi_array[i]->IsYUV() || vi_array[i]->IsYUVA()) ? planes_y : planes_r;
7000 ✗ const int plane_enum_i = plane_enums_i[1];
7001
7002 ✗ if (vi_array[0]->NumComponents() != vi_array[i]->NumComponents() // number of planes should match
7003 ✗ ||
7004 ✗ ( !vi_array[0]->IsY() && ( // no subsampling for Y
7005 ✗ vi_array[0]->GetPlaneWidthSubsampling(plane_enum) != vi_array[i]->GetPlaneWidthSubsampling(plane_enum_i)
7006 ✗ || vi_array[0]->GetPlaneHeightSubsampling(plane_enum) != vi_array[i]->GetPlaneHeightSubsampling(plane_enum_i)
7007 )
7008 )
7009 ✗ || vi_array[0]->width != vi_array[i]->width
7010 ✗ || vi_array[0]->height != vi_array[i]->height)
7011 ✗ env->ThrowError("Expr: All inputs must have the same number of planes and the same dimensions, subsampling included");
7012
7013 ✗ if (vi_array[i]->IsRGB() && !vi_array[i]->IsPlanar())
7014 ✗ env->ThrowError("Expr: No packed RGB format allowed for clip #%d, use planar RGB instead",i+1);
7015 ✗ if (vi_array[i]->IsYUY2())
7016 ✗ env->ThrowError("Expr: YUY2 format not allowed for clip #%d", i+1);
7017 }
7018
7019 // format override
7020 ✗ if (_newformat != nullptr) {
7021 ✗ int pixel_type = GetPixelTypeFromName(_newformat);
7022 ✗ if (pixel_type == VideoInfo::CS_UNKNOWN)
7023 ✗ env->ThrowError("Expr: Invalid video format string parameter");
7024 ✗ d.vi.pixel_type = pixel_type;
7025 ✗ if(d.vi.IsRGB() && !d.vi.IsPlanar())
7026 ✗ env->ThrowError("Expr: No packed RGB format allowed");
7027 ✗ if (d.vi.IsYUY2())
7028 ✗ env->ThrowError("Expr: YUY2 format not allowed");
7029
7030 ✗ const bool isSinglePlaneInput = vi_array[0]->IsY();
7031
7032 // input number of planes >= output planes
7033 ✗ if (!isSinglePlaneInput && vi_array[0]->NumComponents() < d.vi.NumComponents())
7034 ✗ env->ThrowError("Expr: number of planes in input should be greater than or equal than of output");
7035
7036 // subsampling should match
7037 ✗ int *plane_enums_s = (vi_array[0]->IsYUV() || vi_array[0]->IsYUVA()) ? planes_y : planes_r;
7038 ✗ int *plane_enums_d = (d.vi.IsYUV() || d.vi.IsYUVA()) ? planes_y : planes_r;
7039 ✗ for (int p = 0; p < d.vi.NumComponents(); p++) {
7040 ✗ const int plane_enum_s = isSinglePlaneInput ? plane_enums_s[0] : plane_enums_s[p]; // for Y inputs, reference is Y for each output plane
7041 ✗ const int plane_enum_d = plane_enums_d[p];
7042 ✗ if (vi_array[0]->GetPlaneWidthSubsampling(plane_enum_s) != d.vi.GetPlaneWidthSubsampling(plane_enum_d)
7043 ✗ || vi_array[0]->GetPlaneHeightSubsampling(plane_enum_s) != d.vi.GetPlaneHeightSubsampling(plane_enum_d)) {
7044 ✗ if(isSinglePlaneInput)
7045 ✗ env->ThrowError("Expr: output must not be a subsampled format for Y-only input(s)");
7046 else
7047 ✗ env->ThrowError("Expr: inputs and output must have the same subsampling");
7048 }
7049 }
7050
7051 ✗ vi = d.vi;
7052 }
7053
7054 // check expression count, duplicate omitted expressions from previous one
7055 ✗ int nexpr = (int)expressions.size();
7056 ✗ if (nexpr > d.vi.NumComponents()) // ->numPlanes)
7057 ✗ env->ThrowError("Expr: More expressions given than there are planes");
7058
7059 ✗ std::string expr[4]; // 4th: alpha
7060 ✗ for (int i = 0; i < nexpr; i++)
7061 ✗ expr[i] = expressions[i];
7062 ✗ if (nexpr == 1) {
7063 ✗ expr[1] = expr[0];
7064 ✗ expr[2] = expr[0]; // e.g. exprU = exprV = exprY
7065 }
7066 ✗ else if (nexpr == 2) {
7067 ✗ expr[2] = expr[1]; // e.g. exprV = exprU
7068 }
7069 ✗ if(nexpr <= 3)
7070 ✗ expr[3] = ""; // do not use previous expression to alpha expr. Default: "" (copy)
7071
7072 // default: all clips unused
7073 ✗ for (int i = 0; i < MAX_EXPR_INPUTS; i++) {
7074 ✗ d.clipsUsed[i] = false;
7075 }
7076
7077 ✗ for (int i = 0; i < d.vi.NumComponents(); i++) {
7078 ✗ if (!expr[i].empty()) {
7079 ✗ d.plane[i] = poProcess;
7080 }
7081 else {
7082 ✗ if (d.vi.BitsPerComponent() == vi_array[0]->BitsPerComponent()) {
7083 ✗ d.plane[i] = poCopy; // copy only when target clip format bit depth == 1st clip's bit depth
7084 ✗ d.planeCopySourceClip[i] = 0; // default source clip from empty expression: first one
7085 ✗ d.clipsUsed[0] = true; // mark clip to have its GetFrame
7086 }
7087 else
7088 ✗ d.plane[i] = poUndefined;
7089 }
7090 }
7091
7092 ✗ int* plane_enums_d = (d.vi.IsYUV() || d.vi.IsYUVA()) ? planes_y : planes_r;
7093
7094 ✗ d.maxStackSize = 0;
7095 ✗ for (int i = 0; i < d.vi.NumComponents(); i++) {
7096 ✗ const int plane_enum_s = plane_enums[i];
7097 ✗ const int plane_enum = plane_enums_d[i];
7098 ✗ const int planewidth = d.vi.width >> d.vi.GetPlaneWidthSubsampling(plane_enum);
7099 ✗ const int planeheight = d.vi.height >> d.vi.GetPlaneHeightSubsampling(plane_enum);
7100 ✗ const bool chroma = (plane_enum_s == PLANAR_U || plane_enum_s == PLANAR_V);
7101 ✗ d.maxStackSize = std::max(parseExpression(expr[i], d.ops[i], d.frameprops[i], vi_array, &d.vi, getStoreOp(&d.vi), d.numInputs, planewidth, planeheight, chroma,
7102 ✗ autoconv_full_scale, autoconv_conv_int, autoconv_conv_float, clamp_float_i, shift_float, d.lutmode,
7103 ✗ env), d.maxStackSize);
7104 ✗ foldConstants(d.ops[i]);
7105
7106 // optimize constant store, change operation to "fill"
7107 ✗ if (d.plane[i] == poProcess && d.ops[i].size() == 2 && d.ops[i][0].op == opLoadConst) {
7108 ✗ uint32_t op = d.ops[i][1].op;
7109 ✗ if (op == opStore8 || op == opStore10 || op == opStore12 || op == opStore14 || op == opStore16 || op == opStoreF32)
7110 {
7111 ✗ d.plane[i] = poFill;
7112 ✗ d.planeFillValue[i] = d.ops[i][0].e.fval;
7113 }
7114 }
7115
7116 // optimize single clip letter in expression: Load-Store. Change operation to "copy"
7117 // no relative loads here
7118 ✗ if (d.plane[i] == poProcess && d.ops[i].size() == 2 &&
7119 ✗ (d.ops[i][0].op == opLoadSrc8 || d.ops[i][0].op == opLoadSrc16 || d.ops[i][0].op == opLoadSrcF16 || d.ops[i][0].op == opLoadSrcF32) &&
7120 ✗ (d.ops[i][1].op == opStore8 || d.ops[i][1].op == opStore10 || d.ops[i][1].op == opStore12 || d.ops[i][1].op == opStore14 || d.ops[i][1].op == opStore16 || d.ops[i][1].op == opStoreF16 || d.ops[i][1].op == opStoreF32))
7121 {
7122 ✗ const int sourceClip = d.ops[i][0].e.ival;
7123 // check target vs source bit depth
7124 ✗ if(d.vi.BitsPerComponent() == vi_array[sourceClip]->BitsPerComponent()) // no 16bit float in avs+
7125 {
7126 ✗ d.plane[i] = poCopy;
7127 ✗ d.planeCopySourceClip[i] = sourceClip;
7128 ✗ d.clipsUsed[sourceClip] = true; // mark clip to have its GetFrame
7129 }
7130 }
7131
7132 // optimize: mark referenced input clips in order to not call GetFrame for unused inputs
7133 ✗ if (lutmode == 0) {
7134 ✗ for (size_t j = 0; j < d.ops[i].size(); j++) {
7135 ✗ const uint32_t op = d.ops[i][j].op;
7136 ✗ if (op == opLoadSrc8 || op == opLoadSrc16 || op == opLoadSrcF16 || op == opLoadSrcF32 ||
7137 ✗ op == opLoadRelSrc8 || op == opLoadRelSrc16 || op == opLoadRelSrcF32)
7138 {
7139 ✗ const int sourceClip = d.ops[i][j].e.ival;
7140 ✗ d.clipsUsed[sourceClip] = true;
7141 }
7142 }
7143 // input clips with frame property access are used as well
7144 ✗ for (auto framePropToRead : d.frameprops[i]) {
7145 ✗ const int sourceClip = framePropToRead.srcIndex;
7146 ✗ d.clipsUsed[sourceClip] = true;
7147 ✗ }
7148 }
7149
7150 ✗ if (lutmode > 0) {
7151 ✗ for (int i = 0; i < lutmode; i++) // lut: always get. Needed for the init
7152 ✗ d.clipsUsed[i] = true;
7153 // * bit depth of input clip(s) and output must match
7154 bool lut_ok =
7155 ✗ (d.numInputs == 1
7156 ✗ && d.vi.BitsPerComponent() == vi_array[0]->BitsPerComponent())
7157 ✗ ||
7158 (
7159 ✗ d.numInputs == 2
7160 ✗ && d.vi.BitsPerComponent() == vi_array[0]->BitsPerComponent()
7161 ✗ && d.vi.BitsPerComponent() == vi_array[1]->BitsPerComponent()
7162 ✗ );
7163 ✗ if(!lut_ok)
7164 ✗ env->ThrowError("Expr: error in lut mode: input bit depths and output bit depth must be the same");
7165 }
7166
7167
7168 #ifdef INTEL_INTRINSICS
7169 // Check CPU instuction level constraints:
7170 // opLoadRel8/16/32: minimum SSSE3 (pshufb, alignr) for SIMD, and no AVX2 support
7171 // round, floor, ceil, trunc: minimun SSE4.1
7172 // Trig.func: C only
7173 ✗ d.planeOptAvx2[i] = optAvx2;
7174 ✗ d.planeOptSSE2[i] = optSSE2;
7175 ✗ for (size_t j = 0; j < d.ops[i].size(); j++) {
7176 ✗ const uint32_t op = d.ops[i][j].op;
7177 ✗ if (op == opLoadRelSrc8 || op == opLoadRelSrc16 || op == opLoadRelSrcF32)
7178 {
7179 ✗ d.planeOptAvx2[i] = false; // avx2 path not implemented
7180 ✗ if(!(env->GetCPUFlags() & CPUF_SSSE3)) // required minimum (pshufb, alignr)
7181 ✗ d.planeOptSSE2[i] = false;
7182 }
7183 // some trig.functions C only, except Sin and Cos and Atan2 and tan
7184 ✗ if (op == opAsin || op == opAcos || op == opAtan) {
7185 ✗ d.planeOptAvx2[i] = false;
7186 ✗ d.planeOptSSE2[i] = false;
7187 ✗ break;
7188 }
7189 // round, trunc, ceil: minimum of SSE4.1
7190 ✗ if (op == opRound || op == opFloor || op == opCeil || op == opTrunc ) {
7191 ✗ if (!(env->GetCPUFlags() & CPUF_SSE4_1)) // required minimum (_mm_round_ps...)
7192 ✗ d.planeOptSSE2[i] = false;
7193 ✗ break;
7194 }
7195 }
7196 #endif
7197
7198 }
7199
7200 #ifdef VS_TARGET_CPU_X86
7201 // optAvx2 can only disable avx2 when available
7202
7203 ✗ for (int i = 0; i < d.vi.NumComponents(); i++) {
7204 ✗ if (d.plane[i] == poProcess) {
7205
7206 ✗ const int plane_enum = plane_enums_d[i];
7207 ✗ int planewidth = d.vi.width >> d.vi.GetPlaneWidthSubsampling(plane_enum);
7208 ✗ int planeheight = d.vi.height >> d.vi.GetPlaneHeightSubsampling(plane_enum);
7209
7210 ✗ const int planewidth_real_or_lut = (lutmode == 0) ? planewidth: (1 << d.vi.BitsPerComponent());
7211 // to decide if partial chunk is left from the width at the end of the 4/8/16 pixel processing unit big main loops
7212 // when lut: fake width (x size of lut table) of the lut-init
7213
7214 ✗ if (optAvx2 && d.planeOptAvx2[i]) {
7215
7216 // avx2
7217 ✗ ExprEvalAvx2 ExprObj(d.ops[i], d.numInputs, env->GetCPUFlags(), planewidth_real_or_lut, planeheight, optSingleMode);
7218 ✗ if (ExprObj.GetCode(true) && ExprObj.GetCodeSize()) { // PF modded jitasm. true: epilog with vmovaps, and vzeroupper
7219 #ifdef VS_TARGET_OS_WINDOWS
7220 d.proc[i] = (ExprData::ProcessLineProc)VirtualAlloc(nullptr, ExprObj.GetCodeSize(), MEM_COMMIT, PAGE_EXECUTE_READWRITE);
7221 #else
7222 ✗ d.proc[i] = (ExprData::ProcessLineProc)mmap(nullptr, ExprObj.GetCodeSize(), PROT_READ | PROT_WRITE | PROT_EXEC, MAP_ANON | MAP_PRIVATE, 0, 0);
7223 #endif
7224 ✗ memcpy((void *)d.proc[i], ExprObj.GetCode(), ExprObj.GetCodeSize());
7225 }
7226 ✗ }
7227 ✗ else if (optSSE2 && d.planeOptSSE2[i]) {
7228 // sse2, sse4
7229 ✗ ExprEval ExprObj(d.ops[i], d.numInputs, env->GetCPUFlags(), planewidth_real_or_lut, planeheight, optSingleMode);
7230 ✗ if (ExprObj.GetCode() && ExprObj.GetCodeSize()) {
7231 #ifdef VS_TARGET_OS_WINDOWS
7232 d.proc[i] = (ExprData::ProcessLineProc)VirtualAlloc(nullptr, ExprObj.GetCodeSize(), MEM_COMMIT, PAGE_EXECUTE_READWRITE);
7233 #else
7234 ✗ d.proc[i] = (ExprData::ProcessLineProc)mmap(nullptr, ExprObj.GetCodeSize(), PROT_READ | PROT_WRITE | PROT_EXEC, MAP_ANON | MAP_PRIVATE, 0, 0);
7235 #endif
7236 ✗ memcpy((void *)d.proc[i], ExprObj.GetCode(), ExprObj.GetCodeSize());
7237 }
7238 ✗ }
7239
7240 } // if plane is to be processed
7241 }
7242 #ifdef VS_TARGET_OS_WINDOWS
7243 if (optSSE2)
7244 FlushInstructionCache(GetCurrentProcess(), nullptr, 0);
7245 #endif
7246 #endif
7247
7248 ✗ if (lutmode > 0)
7249 ✗ calculate_lut(env);
7250 ✗ }
7251 ✗ catch (std::runtime_error &e) {
7252 ✗ for (int i = 0; i < MAX_EXPR_INPUTS; i++)
7253 ✗ d.clips[i] = nullptr; // vsapi->freeNode(d->node[i]);
7254 ✗ std::string s = "Expr: ";
7255 ✗ s += e.what();
7256 ✗ env->ThrowError(s.c_str());
7257 ✗ return;
7258 ✗ }
7259
7260 ✗ }
7261
7262 #ifdef XP_TLS
7263 #ifdef MSVC_PURE
7264 // end of v141_xp toolset ultra slow build workaround
7265 #pragma optimize("", on)
7266 #endif
7267 #endif
7268