GCC Code Coverage Report

Directory: avs_core/
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 28.4% 71 / 0 / 250
Functions: 50.0% 6 / 0 / 12
Branches: 19.7% 41 / 0 / 208
core/parser/tokenizer.cpp
Line Branch Exec Source
1 // Avisynth v2.5. Copyright 2002 Ben Rudiak-Gould et al.
2 // http://avisynth.nl
3
4 // This program is free software; you can redistribute it and/or modify
5 // it under the terms of the GNU General Public License as published by
6 // the Free Software Foundation; either version 2 of the License, or
7 // (at your option) any later version.
8 //
9 // This program is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 // GNU General Public License for more details.
13 //
14 // You should have received a copy of the GNU General Public License
15 // along with this program; if not, write to the Free Software
16 // Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA, or visit
17 // http://www.gnu.org/copyleft/gpl.html .
18 //
19 // Linking Avisynth statically or dynamically with other modules is making a
20 // combined work based on Avisynth. Thus, the terms and conditions of the GNU
21 // General Public License cover the whole combination.
22 //
23 // As a special exception, the copyright holders of Avisynth give you
24 // permission to link Avisynth with independent modules that communicate with
25 // Avisynth solely through the interfaces defined in avisynth.h, regardless of the license
26 // terms of these independent modules, and to copy and distribute the
27 // resulting combined work under terms of your choice, provided that
28 // every copy of the combined work is accompanied by a complete copy of
29 // the source code of Avisynth (the version of Avisynth used to produce the
30 // combined work), being distributed under the terms of the GNU General
31 // Public License plus this exception. An independent module is a module
32 // which is not derived from or based on Avisynth, such as 3rd-party filters,
33 // import and export plugins, or graphical user interfaces.
34
35
36 #include "tokenizer.h"
37 #include "../InternalEnvironment.h"
38 #ifdef AVS_WINDOWS
39 #include <avs/win.h>
40 #else
41 #include <avs/posix.h>
42 #endif
43
44 #include <cfloat>
45 #include <climits>
46
47
48
49 /****************************
50 ******* Tokenizer ******
51 ***************************/
52
53 6 Tokenizer::Tokenizer(const char* pc, IScriptEnvironment* _env)
54 6 : env(_env)
55 {
56 6 this->pc = pc;
57 6 this->line = 1;
58 6 this->type = 0;
59 6 NextToken();
60 6 }
61
62 ✗ Tokenizer::Tokenizer(Tokenizer* old)
63 ✗ : env(old->env)
64 {
65 ✗ pc = old->pc;
66 ✗ line = old->line;
67 ✗ type = old->type;
68 ✗ NextToken();
69 ✗ }
70
71 54 bool Tokenizer::IsIdentifier(const char* id) const
72 {
73
3/4
✓ Branch 3 → 4 taken 27 times.
✓ Branch 3 → 6 taken 27 times.
✗ Branch 4 → 5 not taken.
✓ Branch 4 → 6 taken 27 times.
 54 return IsIdentifier() && !lstrcmpi(id, identifier);
74 }
75
76
77 12 void Tokenizer::NextToken() {
78
79
1/2
✗ Branch 3 → 4 not taken.
✓ Branch 3 → 5 taken 12 times.
 12 if (IsNewline())
80 ✗ line++;
81
82 // skip whitespace, comments, and escaped newlines
83
84 for (;;) {
85 12 SkipWhitespace();
86
1/2
✗ Branch 7 → 8 not taken.
✓ Branch 7 → 14 taken 12 times.
 12 if (*pc == '\\') {
87 // eat backslash followed by newline
88 ✗ const char* const old_pc = pc;
89 ✗ pc++;
90 ✗ SkipWhitespace();
91 ✗ if (*pc == '\n' || *pc == '\r') {
92 ✗ SkipNewline();
93 ✗ continue;
94 } else {
95 ✗ token_start = old_pc;
96 ✗ env->ThrowError("Script error: `\\' can only appear at the beginning or end of a line");
97 }
98
2/4
✓ Branch 14 → 15 taken 12 times.
✗ Branch 14 → 16 not taken.
✗ Branch 15 → 16 not taken.
✓ Branch 15 → 27 taken 12 times.
 12 } else if (*pc == '\n' || *pc == '\r') {
99 // skip newline if it's followed by backslash or `{' }
100 ✗ const char* const old_pc = pc;
101 ✗ const int old_line = line;
102 do {
103 ✗ SkipNewline();
104 ✗ SkipWhitespace();
105 ✗ } while (*pc == '\n' || *pc == '\r');
106 ✗ if (*pc == '\\') {
107 ✗ pc++;
108 ✗ continue;
109 ✗ } else if (*pc == '{') {
110 ✗ break;
111 } else {
112 ✗ pc = old_pc;
113 ✗ line = old_line;
114 ✗ break;
115 }
116
1/2
✗ Branch 27 → 28 not taken.
✓ Branch 27 → 34 taken 12 times.
 12 } else if (*pc == '#') {
117 // skip from # to end of line (comment)
118 ✗ while (*pc != 0 && *pc != '\n' && *pc != '\r')
119 ✗ pc++;
120 ✗ break;
121
1/4
✗ Branch 34 → 35 not taken.
✓ Branch 34 → 44 taken 12 times.
✗ Branch 35 → 36 not taken.
✗ Branch 35 → 44 not taken.
 12 } else if (pc[0] == '/' && pc[1] == '*') { // Block comment /* */
122 ✗ const char *end = strstr(pc+2, "*/");
123 ✗ if (!end)
124 ✗ env->ThrowError("Parse error: block comment missing closing */");
125
126 ✗ for (const char *cp = pc+2; cp < end; cp++) {
127 ✗ if (*cp == '\n') { line++; }
128 }
129 ✗ pc = end+2;
130 ✗ continue;
131
1/4
✗ Branch 44 → 45 not taken.
✓ Branch 44 → 47 taken 12 times.
✗ Branch 45 → 46 not taken.
✗ Branch 45 → 47 not taken.
 12 } else if (pc[0] == '*' && pc[1] == '/') {
132 ✗ env->ThrowError("Parse error: orphan block comment closing */");
133
134
1/4
✗ Branch 47 → 48 not taken.
✓ Branch 47 → 61 taken 12 times.
✗ Branch 48 → 49 not taken.
✗ Branch 48 → 61 not taken.
 12 } else if (pc[0] == '[' && pc[1] == '*') { // Nestable block comment [* *]
135 ✗ const char *end = strstr(pc+2, "*]");
136 ✗ const char *nest = strstr(pc+2, "[*");
137
138 ✗ while (nest && nest+1 < end) {
139 ✗ end = strstr(end+2, "*]");
140 ✗ nest = strstr(nest+2, "[*");
141 }
142 ✗ if (!end)
143 ✗ env->ThrowError("Parse error: nestable block comment missing closing *]");
144
145 ✗ for (const char *cp = pc+2; cp < end; cp++) {
146 ✗ if (*cp == '\n') { line++; }
147 }
148 ✗ pc = end+2;
149 ✗ continue;
150
1/4
✗ Branch 61 → 62 not taken.
✓ Branch 61 → 65 taken 12 times.
✗ Branch 62 → 63 not taken.
✗ Branch 62 → 65 not taken.
 12 } else if (pc[0] == '*' && pc[1] == ']') {
151 ✗ env->ThrowError("Parse error: orphan nestable block comment closing *]");
152
153 } else {
154 break;
155 }
156 ✗ }
157
158 12 token_start = pc;
159
160
2/9
✓ Branch 65 → 66 taken 6 times.
✗ Branch 65 → 67 not taken.
✗ Branch 65 → 69 not taken.
✗ Branch 65 → 73 not taken.
✗ Branch 65 → 79 not taken.
✗ Branch 65 → 84 not taken.
✗ Branch 65 → 89 not taken.
✗ Branch 65 → 91 not taken.
✓ Branch 65 → 109 taken 6 times.
 12 switch (*pc) {
161
162 6 case 0:
163 6 type = 0;
164 6 break;
165
166 ✗ case '\n': case '\r':
167 ✗ SkipNewline();
168 ✗ type = 'n';
169 ✗ line--;
170 ✗ break;
171
172 ✗ case '.':
173 // a '.' followed by a digit is a number, otherwise it's an operator
174 ✗ if (isdigit(pc[1])) {
175 ✗ GetNumber();
176 } else {
177 ✗ ++pc;
178 ✗ SetToOperator('.');
179 }
180 ✗ break;
181
182 ✗ case '<': // these operators have versions followed by '='
183 ✗ if ((pc[1] == '=') || (pc[1] == '>')) {
184 ✗ SetToOperator(pc[0] * 256 + pc[1]);
185 ✗ pc += 2;
186 } else {
187 ✗ SetToOperator(*pc++);
188 }
189 ✗ break;
190
191 ✗ case '>': // these operators have versions followed by '='
192 case '!':
193 case '=':
194 ✗ if (pc[1] != '=') {
195 ✗ SetToOperator(*pc++);
196 } else {
197 ✗ SetToOperator(pc[0] * 256 + pc[1]);
198 ✗ pc += 2;
199 }
200 ✗ break;
201
202 ✗ case '+': // these operators have single and double (++, &&, ||, ==) versions
203 case '&':
204 case '|':
205 ✗ if (pc[1] != pc[0]) {
206 ✗ SetToOperator(*pc++);
207 } else {
208 ✗ SetToOperator(pc[0] * 256 + pc[1]);
209 ✗ pc += 2;
210 }
211 ✗ break;
212
213 ✗ case '{': // these operators are always lone characters
214 case '}':
215 case '(':
216 case ')':
217 case '[': // variable capture for function definition
218 case ']':
219 case ',':
220 case '?':
221 case ':':
222 case '-':
223 case '*':
224 case '/':
225 case '%':
226 ✗ SetToOperator(*pc++);
227 ✗ break;
228
229 ✗ case '$':
230 // hexadecimal constant $FFFFFFFF remains -1 int;
231 // or suffixed with L for 64 bit data: 0xFFFFFFFFL becomes 0x00000000FFFFFFFF
232 {
233 ✗ bool long64bit = false;
234 ✗ type = 'l';
235 ✗ longlong = 0;
236 ✗ ++pc;
237 do {
238 ✗ if (*pc >= '0' && *pc <= '9')
239 ✗ longlong = longlong * 16 + (*pc - '0');
240 ✗ else if (*pc >= 'a' && *pc <= 'f')
241 ✗ longlong = longlong * 16 + (*pc - 'a' + 10);
242 ✗ else if (*pc >= 'A' && *pc <= 'F')
243 ✗ longlong = longlong * 16 + (*pc - 'A' + 10);
244 ✗ else if (*pc == 'L' || *pc == 'l') {
245 ✗ long64bit = true;
246 ✗ ++pc;
247 ✗ break; // nothing allowed after L cast
248 }
249 else
250 ✗ env->ThrowError("$ must be followed by a hexadecimal number");
251 ✗ } while (isalnum(*++pc));
252 ✗ if (!long64bit) {
253 // Default is integer.
254 // Thus we can avoid the color (like $FFFFFFFF) constants
255 // to be casted to 64 bit.
256 ✗ type = 'i';
257 ✗ integer = (int)longlong;
258 }
259 }
260 ✗ break;
261
262 6 default:
263
2/6
✓ Branch 109 → 110 taken 6 times.
✗ Branch 109 → 112 not taken.
✗ Branch 110 → 111 not taken.
✓ Branch 110 → 148 taken 6 times.
✗ Branch 111 → 112 not taken.
✗ Branch 111 → 148 not taken.
 6 if(*pc == '"' || (*pc == 'e' && pc[1] == '"')) { // string
264 ✗ bool escape = false;
265 ✗ if (*pc == 'e') {
266 ✗ escape = true;
267 ✗ ++pc;
268 }
269 const char *start, *end;
270 ✗ if (pc[1] == '"' && pc[2] == '"') {
271 // """..."""
272 ✗ start = pc+3;
273 ✗ bool escaped_quotation = false;
274 ✗ auto tmp_start = start;
275 ✗ do {
276 ✗ end = strstr(tmp_start, "\"\"\"");
277 ✗ if (!end)
278 ✗ env->ThrowError("Parse error: string missing closing quotation marks");
279 ✗ escaped_quotation = escape && (*(end - 1) == '\\');
280 ✗ if (escaped_quotation)
281 ✗ tmp_start = end + 1;
282 } while (escaped_quotation);
283
284 ✗ while (end[3] == '"')
285 ✗ end++;
286 ✗ pc = end+3;
287 ✗ } else {
288 // "..."
289 ✗ start = pc+1;
290 ✗ bool escaped_quotation = false;
291 ✗ auto tmp_start = start;
292 ✗ do {
293 ✗ end = strchr(tmp_start, '"');
294 ✗ if (!end)
295 ✗ env->ThrowError("Parse error: string missing closing quotation mark");
296 ✗ escaped_quotation = escape && (*(end - 1) == '\\');
297 ✗ if (escaped_quotation)
298 ✗ tmp_start = end + 1;
299 } while (escaped_quotation);
300
301 /* I like the ability to have newlines in strings, thanks */
302 // const char *cr = strchr(start, '\r'), *lf = strchr(start, '\n');
303 // if ((cr && cr < end) || (lf && lf < end))
304 // env->ThrowError("Parse error: newline found in string");
305
306 ✗ pc = end+1;
307 }
308 ✗ for (const char *cp = start; cp < end; cp++) {
309 ✗ if (*cp == '\n') { line++; }
310 ✗ } type = 's';
311 ✗ string = static_cast<InternalEnvironment*>(env)->SaveString(start, int(end-start), escape);
312 ✗ }
313
2/2
✓ Branch 148 → 149 taken 3 times.
✓ Branch 148 → 150 taken 3 times.
 6 else if (isdigit(*pc)) {
314 // number
315 3 GetNumber();
316
2/4
✓ Branch 150 → 151 taken 3 times.
✗ Branch 150 → 152 not taken.
✓ Branch 151 → 152 taken 3 times.
✗ Branch 151 → 160 not taken.
 3 } else if (*pc == '_' || isalpha(*pc)) {
317 // identifier
318 do {
319 33 pc++;
320
3/4
✗ Branch 154 → 153 not taken.
✓ Branch 154 → 155 taken 33 times.
✓ Branch 155 → 153 taken 30 times.
✓ Branch 155 → 156 taken 3 times.
 33 } while (*pc == '_' || isalnum(*pc));
321 3 type = 'I';
322 3 identifier = env->SaveString(token_start, int(pc - token_start));
323
1/2
✗ Branch 157 → 158 not taken.
✓ Branch 157 → 159 taken 3 times.
 3 if (!lstrcmpi(identifier, "__END__")) {
324 ✗ type = 0;
325 }
326 } else {
327 ✗ env->ThrowError("unexpected character \"%c\"", *pc);
328 }
329 6 break;
330 }
331 12 }
332
333
334 ✗ int Tokenizer::GetColumn(const char* start_of_string) const
335 {
336 ✗ const char* x = pc;
337 ✗ while (x > start_of_string && x[-1] != '\n' && x[-1] != '\r')
338 ✗ x--;
339 ✗ return int(pc-x);
340 }
341
342
343 12 void Tokenizer::SkipWhitespace()
344 {
345
2/4
✗ Branch 4 → 3 not taken.
✓ Branch 4 → 5 taken 12 times.
✗ Branch 5 → 3 not taken.
✓ Branch 5 → 6 taken 12 times.
 12 while (*pc == ' ' || *pc == '\t')
346 ✗ pc++;
347 12 }
348
349 ✗ void Tokenizer::SkipNewline()
350 {
351 ✗ if (*pc == '\n' || *pc == '\r')
352 {
353 ✗ pc++;
354 ✗ line++;
355 ✗ if ((*pc == '\n' || *pc == '\r') && *pc != *(pc-1))
356 ✗ pc++;
357 }
358 ✗ }
359
360 6 void Tokenizer::AssertType(char expected_type) const
361 {
362
1/2
✗ Branch 2 → 3 not taken.
✓ Branch 2 → 4 taken 6 times.
 6 if (type != expected_type)
363 ✗ ThrowTypeMismatch(expected_type, type, env);
364 6 }
365
366 3 void Tokenizer::GetNumber()
367 {
368 // old: start by assuming an int and switch to float if necessary
369 // v11: default precision is 64 bit
370 // start by assuming long and switch to double if necessary
371 3 constexpr int64_t INT64_MAX_DIV_10 = INT64_MAX / 10;
372 3 constexpr int64_t INT64_MAX_MOD_10 = INT64_MAX % 10;
373 3 type = 'l';
374 3 longlong = 0;
375 3 double dtemp = 0;
376 3 double place = 1;
377
378 do {
379
2/2
✓ Branch 4 → 5 taken 1 time.
✓ Branch 4 → 9 taken 3 times.
 4 if (*pc == '.') {
380 // first, gather into double
381 1 type = 'f';
382 1 ++pc;
383
2/2
✓ Branch 7 → 6 taken 1 time.
✓ Branch 7 → 8 taken 1 time.
 2 while (isdigit(*pc)) {
384 1 place *= 10;
385 1 dtemp = dtemp * 10 + (*pc - '0');
386 1 ++pc;
387 }
388 1 break;
389 }
390 else {
391
1/2
✓ Branch 9 → 10 taken 3 times.
✗ Branch 9 → 16 not taken.
 3 if (type != 'f') {
392 // Check for potential overflow before performing the operation, but go on anyway
393
2/6
✓ Branch 10 → 11 taken 3 times.
✗ Branch 10 → 13 not taken.
✗ Branch 11 → 12 not taken.
✓ Branch 11 → 14 taken 3 times.
✗ Branch 12 → 13 not taken.
✗ Branch 12 → 14 not taken.
 3 if (longlong > INT64_MAX_DIV_10 || (longlong == INT64_MAX_DIV_10 && (*pc - '0') > INT64_MAX_MOD_10))
394 ✗ type = 'f';
395 else
396 3 longlong = longlong * 10 + (*pc - '0');
397 }
398 3 dtemp = dtemp * 10 + (*pc - '0'); // gather always
399 }
400 3 ++pc;
401
3/4
✗ Branch 16 → 3 not taken.
✓ Branch 16 → 17 taken 3 times.
✓ Branch 17 → 3 taken 1 time.
✓ Branch 17 → 18 taken 2 times.
 3 } while (isdigit(*pc) || *pc == '.');
402
403 3 dtemp /= place;
404
1/2
✗ Branch 18 → 19 not taken.
✓ Branch 18 → 20 taken 3 times.
 3 if (dtemp >= DBL_MAX)
405 ✗ env->ThrowError("Tokenizer: Number is to big.");
406
407
2/2
✓ Branch 20 → 21 taken 1 time.
✓ Branch 20 → 22 taken 2 times.
 3 if (type == 'f') {
408 1 double_pt = dtemp; // v11: type name remained 'f' but content is double
409 }
410
1/2
✓ Branch 22 → 23 taken 2 times.
✗ Branch 22 → 24 not taken.
 2 else if (longlong <= INT_MAX) {
411 // store into a more simple type
412 2 type = 'i';
413 2 integer = (int)longlong;
414 }
415 3 }
416
417
418 ✗ void Tokenizer::SetToOperator(int o)
419 {
420 ✗ type = 'o';
421 ✗ op = o;
422 ✗ }
423
424
425
426 /**** Helper Functions ****/
427
428 ✗ static const char* GetTypeName(char type)
429 {
430 ✗ switch (type)
431 {
432 ✗ case 0: return "undefined";
433 ✗ case 'a': return "array";
434 ✗ case 'b': return "boolean";
435 ✗ case 'c': return "clip";
436 ✗ case 'I': return "identifier";
437 ✗ case 'f': return "floating-point"; // v11: double
438 ✗ case 'i': return "integer";
439 ✗ case 'l': return "long";
440 ✗ case 'o': return "operator";
441 ✗ case 's': return "string";
442 ✗ default: return "unknown";
443 }
444 }
445
446
447 ✗ void ThrowTypeMismatch(char expected, char actual, IScriptEnvironment* env)
448 {
449 ✗ env->ThrowError("Tokenizer: expected type '%s' doesn't match actual type '%s' (this is a bug)",
450 GetTypeName(expected), GetTypeName(actual));
451 ✗ }
452
453