core/strings.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // This program is free software; you can redistribute it and/or modify | ||
| 2 | // it under the terms of the GNU General Public License as published by | ||
| 3 | // the Free Software Foundation; either version 2 of the License, or | ||
| 4 | // (at your option) any later version. | ||
| 5 | // | ||
| 6 | // This program is distributed in the hope that it will be useful, | ||
| 7 | // but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| 8 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| 9 | // GNU General Public License for more details. | ||
| 10 | // | ||
| 11 | // You should have received a copy of the GNU General Public License | ||
| 12 | // along with this program; if not, write to the Free Software | ||
| 13 | // Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA, or visit | ||
| 14 | // http://www.gnu.org/copyleft/gpl.html . | ||
| 15 | // | ||
| 16 | // Linking Avisynth statically or dynamically with other modules is making a | ||
| 17 | // combined work based on Avisynth. Thus, the terms and conditions of the GNU | ||
| 18 | // General Public License cover the whole combination. | ||
| 19 | // | ||
| 20 | // As a special exception, the copyright holders of Avisynth give you | ||
| 21 | // permission to link Avisynth with independent modules that communicate with | ||
| 22 | // Avisynth solely through the interfaces defined in avisynth.h, regardless of the license | ||
| 23 | // terms of these independent modules, and to copy and distribute the | ||
| 24 | // resulting combined work under terms of your choice, provided that | ||
| 25 | // every copy of the combined work is accompanied by a complete copy of | ||
| 26 | // the source code of Avisynth (the version of Avisynth used to produce the | ||
| 27 | // combined work), being distributed under the terms of the GNU General | ||
| 28 | // Public License plus this exception. An independent module is a module | ||
| 29 | // which is not derived from or based on Avisynth, such as 3rd-party filters, | ||
| 30 | // import and export plugins, or graphical user interfaces. | ||
| 31 | |||
| 32 | #include <avs/config.h> | ||
| 33 | #ifdef AVS_WINDOWS | ||
| 34 | #include <io.h> | ||
| 35 | #include <avs/win.h> | ||
| 36 | #else | ||
| 37 | #include <avs/posix.h> | ||
| 38 | #include "parser/os/win32_string_compat.h" | ||
| 39 | #endif | ||
| 40 | #include <avs/filesystem.h> | ||
| 41 | |||
| 42 | #include "strings.h" | ||
| 43 | #include <cassert> | ||
| 44 | #include <string> | ||
| 45 | #include <algorithm> | ||
| 46 | #include <codecvt> | ||
| 47 | #include <locale> | ||
| 48 | #include <vector> | ||
| 49 | |||
| 50 | |||
| 51 | 1199072 | static inline char tolower(char c) | |
| 52 | { | ||
| 53 | // Works for letters of the english alphabet in ASCII | ||
| 54 |
4/4✓ Branch 2 → 3 taken 999713 times.
✓ Branch 2 → 5 taken 199359 times.
✓ Branch 3 → 4 taken 216518 times.
✓ Branch 3 → 5 taken 783195 times.
|
1199072 | return ((c >=65) && (c <=90)) ? c + 32 : c; |
| 55 | } | ||
| 56 | |||
| 57 | 26686 | bool streqi(const char* s1, const char* s2) | |
| 58 | { | ||
| 59 | // Why we dont use Windows's lstrcmpi? It is by multiple orders of magnitude slower and non-portable. | ||
| 60 | // lstrcmpi handles locales and UTF and whatnot, but because variable and function names in Avisynth | ||
| 61 | // are limited to ASCII, we don't need that funcationality. | ||
| 62 | |||
| 63 | while(1) | ||
| 64 | { | ||
| 65 |
3/4✓ Branch 3 → 4 taken 25393 times.
✓ Branch 3 → 6 taken 599536 times.
✓ Branch 4 → 5 taken 25393 times.
✗ Branch 4 → 6 not taken.
|
624929 | if ((*s1 == 0) && (*s2 == 0)) |
| 66 | 25393 | return true; | |
| 67 | |||
| 68 |
2/2✓ Branch 8 → 9 taken 1293 times.
✓ Branch 8 → 10 taken 598243 times.
|
599536 | if (tolower(*s1) != tolower(*s2)) |
| 69 | 1293 | return false; | |
| 70 | |||
| 71 | 598243 | ++s1; | |
| 72 | 598243 | ++s2; | |
| 73 | } | ||
| 74 | |||
| 75 | assert(0); | ||
| 76 | return false; | ||
| 77 | } | ||
| 78 | |||
| 79 | ✗ | std::string concat(const std::string &s1, const std::string &s2) | |
| 80 | { | ||
| 81 | ✗ | std::string ret(s1); | |
| 82 | ✗ | return ret.append(s2); | |
| 83 | ✗ | } | |
| 84 | |||
| 85 | 1812 | bool replace(std::string &haystack, const std::string &needle, const std::string &newStr) | |
| 86 | { | ||
| 87 | 1812 | bool replaced = false; | |
| 88 | 1812 | for(size_t pos = 0; ; pos += newStr.length()) | |
| 89 | { | ||
| 90 | // Locate the substring to replace | ||
| 91 | 1812 | pos = haystack.find(needle, pos); | |
| 92 |
1/2✓ Branch 4 → 5 taken 1812 times.
✗ Branch 4 → 6 not taken.
|
1812 | if(pos == std::string::npos) break; |
| 93 | // Replace by erasing and inserting | ||
| 94 | ✗ | haystack.erase(pos, needle.length()); | |
| 95 | ✗ | haystack.insert(pos, newStr); | |
| 96 | ✗ | replaced = true; | |
| 97 | } | ||
| 98 | |||
| 99 | 1812 | return replaced; | |
| 100 | } | ||
| 101 | |||
| 102 | 5436 | bool replace_beginning(std::string &haystack, const std::string &needle, const std::string &newStr) | |
| 103 | { | ||
| 104 | // Locate the substring to replace | ||
| 105 | 5436 | size_t pos = haystack.find(needle); | |
| 106 |
1/2✓ Branch 3 → 4 taken 5436 times.
✗ Branch 3 → 5 not taken.
|
5436 | if(pos == std::string::npos) return false; |
| 107 | ✗ | if(pos != 0) return false; | |
| 108 | |||
| 109 | // Replace by erasing and inserting | ||
| 110 | ✗ | haystack.erase( pos, needle.length() ); | |
| 111 | ✗ | haystack.insert( pos, newStr ); | |
| 112 | |||
| 113 | ✗ | return true; | |
| 114 | } | ||
| 115 | |||
| 116 | 3624 | bool replace(std::string &haystack, char needle, char newChar) | |
| 117 | { | ||
| 118 |
1/2✓ Branch 2 → 3 taken 3624 times.
✗ Branch 2 → 13 not taken.
|
3624 | std::string haystack_bck = haystack; |
| 119 | 3624 | std::replace(haystack.begin(),haystack.end(), needle, newChar); | |
| 120 |
1/2✓ Branch 6 → 7 taken 3624 times.
✗ Branch 6 → 11 not taken.
|
7248 | return haystack.compare(haystack_bck) != 0; |
| 121 | 3624 | } | |
| 122 | |||
| 123 | const char* ws = " \t\n\r\f\v"; | ||
| 124 | |||
| 125 | ✗ | std::string trim(const std::string& s) | |
| 126 | { | ||
| 127 | ✗ | auto wsfront = std::find_if_not(s.begin(), s.end(), [](int c) {return isspace(c); }); | |
| 128 | ✗ | auto wsback = std::find_if_not(s.rbegin(), s.rend(), [](int c) {return isspace(c); }).base(); | |
| 129 | ✗ | return (wsback <= wsfront ? std::string() : std::string(wsfront, wsback)); | |
| 130 | } | ||
| 131 | |||
| 132 | /*********************************** | ||
| 133 | ******* wchar_t-utf-ansi ****** | ||
| 134 | **********************************/ | ||
| 135 | |||
| 136 | #ifdef AVS_WINDOWS | ||
| 137 | std::unique_ptr<char[]> AnsiToUtf8(const char* input) | ||
| 138 | { | ||
| 139 | int wlen = MultiByteToWideChar(AreFileApisANSI() ? CP_ACP : CP_OEMCP, 0, input, -1, NULL, 0); | ||
| 140 | wchar_t* wstr = new wchar_t[wlen]; | ||
| 141 | MultiByteToWideChar(AreFileApisANSI() ? CP_ACP : CP_OEMCP, 0, input, -1, wstr, wlen); | ||
| 142 | const auto utf8len = WideCharToMultiByte(CP_UTF8, 0, wstr, -1, NULL, 0, NULL, NULL); | ||
| 143 | // The length comes with the \0 terminator | ||
| 144 | // Allocate a buffer for the UTF-8 string | ||
| 145 | auto s_utf8 = std::make_unique<char[]>(utf8len); | ||
| 146 | // Convert the UTF-16 string from UTF-8 | ||
| 147 | WideCharToMultiByte(CP_UTF8, 0, wstr, -1, s_utf8.get(), (int)utf8len, NULL, NULL); | ||
| 148 | delete[] wstr; | ||
| 149 | return s_utf8; | ||
| 150 | } | ||
| 151 | |||
| 152 | std::unique_ptr<char[]> WideCharToUtf8(const wchar_t* w_string) | ||
| 153 | { | ||
| 154 | const auto utf8len = WideCharToMultiByte(CP_UTF8, 0, w_string, -1, NULL, 0, 0, 0) - 1; // w/o the \0 terminator | ||
| 155 | auto s_utf8 = std::make_unique<char[]>(utf8len + 1); | ||
| 156 | WideCharToMultiByte(CP_UTF8, 0, w_string, -1, s_utf8.get(), (int)utf8len + 1, 0, 0); | ||
| 157 | return s_utf8; | ||
| 158 | } | ||
| 159 | |||
| 160 | std::unique_ptr<char[]> WideCharToAnsi(const wchar_t* w_string) | ||
| 161 | { | ||
| 162 | const auto len = wcslen(w_string); | ||
| 163 | auto s_ansi = std::make_unique<char[]>(len + 1); | ||
| 164 | WideCharToMultiByte(AreFileApisANSI() ? CP_ACP : CP_OEMCP, 0, w_string, -1, s_ansi.get(), (int)len + 1, NULL, NULL); // replaces out-of-CP chars by ? | ||
| 165 | // int succ = wcstombs(s_ansi, w_string, len +1); | ||
| 166 | // no good, stops at non-replacable unicode chars. If wcstombs encounters a wide character it cannot convert to a multibyte character, it returns 1 cast to type size_t and sets errno to EILSEQ. | ||
| 167 | return s_ansi; | ||
| 168 | } | ||
| 169 | |||
| 170 | std::unique_ptr<char[]> WideCharToAnsiACP(const wchar_t* w_string) | ||
| 171 | { | ||
| 172 | const auto len = wcslen(w_string); | ||
| 173 | auto s_ansi = std::make_unique<char[]>(len + 1); | ||
| 174 | WideCharToMultiByte(CP_ACP, 0, w_string, -1, s_ansi.get(), (int)len + 1, NULL, NULL); // replaces out-of-CP chars by ? | ||
| 175 | // int succ = wcstombs(s_ansi, w_string, len +1); | ||
| 176 | // no good, stops at non-replacable unicode chars. If wcstombs encounters a wide character it cannot convert to a multibyte character, it returns 1 cast to type size_t and sets errno to EILSEQ. | ||
| 177 | return s_ansi; | ||
| 178 | } | ||
| 179 | |||
| 180 | std::unique_ptr<char[]> WideCharToUtf8_maxn(const wchar_t* w_string, size_t maxn) | ||
| 181 | { | ||
| 182 | const auto utf8len = WideCharToMultiByte(CP_UTF8, 0, w_string, (int)maxn, NULL, 0, 0, 0); // no \0 terminator check requested here | ||
| 183 | auto s_utf8 = std::make_unique<char[]>(utf8len + 1); | ||
| 184 | WideCharToMultiByte(CP_UTF8, 0, w_string, -1, s_utf8.get(), utf8len, 0, 0); | ||
| 185 | s_utf8[utf8len] = 0; | ||
| 186 | return s_utf8; | ||
| 187 | } | ||
| 188 | |||
| 189 | std::unique_ptr<char[]> WideCharToAnsi_maxn(const wchar_t* w_string, size_t maxn) | ||
| 190 | { | ||
| 191 | auto s_ansi = std::make_unique<char[]>(maxn + 1); | ||
| 192 | WideCharToMultiByte(AreFileApisANSI() ? CP_ACP : CP_OEMCP, 0, w_string, -1, s_ansi.get(), (int)maxn, NULL, NULL); // replaces out-of-CP chars by ? | ||
| 193 | s_ansi[maxn] = 0; | ||
| 194 | return s_ansi; | ||
| 195 | } | ||
| 196 | |||
| 197 | std::unique_ptr<wchar_t[]> AnsiToWideChar(const char* s_ansi) | ||
| 198 | { | ||
| 199 | const size_t bufsize = strlen(s_ansi) + 1; | ||
| 200 | auto w_string = std::make_unique<wchar_t[]>(bufsize); | ||
| 201 | MultiByteToWideChar(AreFileApisANSI() ? CP_ACP : CP_OEMCP, 0, s_ansi, -1, w_string.get(), (int)bufsize); | ||
| 202 | //mbstowcs(script_name_w, script_name, len); // ansi to wchar_t, does not convert properly out-of-the box | ||
| 203 | return w_string; | ||
| 204 | } | ||
| 205 | |||
| 206 | std::unique_ptr<wchar_t[]> AnsiToWideCharACP(const char* s_ansi) | ||
| 207 | { | ||
| 208 | const size_t bufsize = strlen(s_ansi) + 1; | ||
| 209 | auto w_string = std::make_unique<wchar_t[]>(bufsize); | ||
| 210 | MultiByteToWideChar(CP_ACP, 0, s_ansi, -1, w_string.get(), (int)bufsize); | ||
| 211 | //mbstowcs(script_name_w, script_name, len); // ansi to wchar_t, does not convert properly out-of-the box | ||
| 212 | return w_string; | ||
| 213 | } | ||
| 214 | |||
| 215 | std::unique_ptr<wchar_t[]> Utf8ToWideChar(const char* s_utf8) | ||
| 216 | { | ||
| 217 | const size_t wchars_count = MultiByteToWideChar(CP_UTF8, 0, s_utf8, -1, NULL, 0); | ||
| 218 | const size_t bufsize = wchars_count + 1; | ||
| 219 | auto w_string = std::make_unique<wchar_t[]>(bufsize); | ||
| 220 | MultiByteToWideChar(CP_UTF8, 0, s_utf8, -1, w_string.get(), (int)bufsize); | ||
| 221 | return w_string; | ||
| 222 | } | ||
| 223 | |||
| 224 | std::string Utf8ToAnsi(const char* s_utf8) | ||
| 225 | { | ||
| 226 | std::string s; | ||
| 227 | auto w = Utf8ToWideChar(s_utf8); | ||
| 228 | auto ansi = WideCharToAnsi(w.get()); // replaces out-of-CP chars by ? | ||
| 229 | s = ansi.get(); | ||
| 230 | return s; | ||
| 231 | } | ||
| 232 | |||
| 233 | #endif | ||
| 234 | |||
| 235 | ✗ | std::string double_to_string(double d) | |
| 236 | { | ||
| 237 | char s[50]; // safe size for a double | ||
| 238 | #ifdef MSVC | ||
| 239 | _locale_t locale = _create_locale(LC_NUMERIC, "C"); // force '.' as decimal separator | ||
| 240 | _sprintf_l(s, "%lf", locale, d); | ||
| 241 | _free_locale(locale); | ||
| 242 | #else | ||
| 243 | ✗ | sprintf(s, "%lf", d); | |
| 244 | #endif | ||
| 245 | ✗ | return s; | |
| 246 | } | ||
| 247 | |||
| 248 | 1812 | std::string GetFullPathNameWrapUtf8(const std::string& f) | |
| 249 | { | ||
| 250 | #ifdef AVS_WINDOWS | ||
| 251 | // assume f is utf-8 | ||
| 252 | auto f_wide = Utf8ToWideChar(f.c_str()); | ||
| 253 | auto p = fs::absolute(fs::path(f_wide.get()).lexically_normal()); | ||
| 254 | auto w = p.wstring(); // UTF-16 on Windows | ||
| 255 | auto utf8 = WideCharToUtf8(w.c_str()); | ||
| 256 | return std::string(utf8.get()); | ||
| 257 | #else | ||
| 258 |
4/8✓ Branch 2 → 3 taken 1812 times.
✗ Branch 2 → 18 not taken.
✓ Branch 3 → 4 taken 1812 times.
✗ Branch 3 → 16 not taken.
✓ Branch 4 → 5 taken 1812 times.
✗ Branch 4 → 14 not taken.
✓ Branch 5 → 6 taken 1812 times.
✗ Branch 5 → 12 not taken.
|
1812 | return fs::absolute(fs::path(f).lexically_normal()).generic_string(); |
| 259 | #endif | ||
| 260 | } | ||
| 261 | |||
| 262 | 7 | size_t str_utf8_size(const std::string& s) { | |
| 263 | // Does not handle combined codepoints, e.g. diacritic mark modifications | ||
| 264 | // Leading UTF8 byte: | ||
| 265 | // 1 byte 0xxxxxxx | ||
| 266 | // 2 bytes 110xxxxx | ||
| 267 | // 3 bytes 1110xxxx | ||
| 268 | // 4 bytes 11110xxx | ||
| 269 | 7 | size_t len = 0; | |
| 270 |
2/2✓ Branch 17 → 4 taken 369 times.
✓ Branch 17 → 18 taken 7 times.
|
383 | for (char c : s) { |
| 271 |
1/2✓ Branch 6 → 7 taken 369 times.
✗ Branch 6 → 8 not taken.
|
369 | if ((c & 0xc0) != 0x80) // Not a trailing byte 10xxxxxx |
| 272 | 369 | len++; | |
| 273 | } | ||
| 274 | 7 | return len; | |
| 275 | } | ||
| 276 | |||
| 277 | // converts a 16 bit unicode codepoint to its utf8 string representation | ||
| 278 | 9478 | std::string U16_to_utf8(uint16_t u16) | |
| 279 | { | ||
| 280 | uint8_t bytes[3]; | ||
| 281 | 9478 | int size = 0; | |
| 282 | // check the range of the UTF-16 code point | ||
| 283 |
2/2✓ Branch 2 → 3 taken 672 times.
✓ Branch 2 → 4 taken 8806 times.
|
9478 | if (u16 <= 0x007F) { |
| 284 | // one byte, 0xxxxxxx | ||
| 285 | 672 | bytes[0] = static_cast<uint8_t>(u16); | |
| 286 | 672 | size = 1; | |
| 287 | } | ||
| 288 |
2/2✓ Branch 4 → 5 taken 4081 times.
✓ Branch 4 → 6 taken 4725 times.
|
8806 | else if (u16 <= 0x07FF) { |
| 289 | // two bytes, 110xxxxx 10xxxxxx | ||
| 290 | 4081 | bytes[0] = static_cast<uint8_t>(0xC0 | ((u16 >> 6) & 0x1F)); | |
| 291 | 4081 | bytes[1] = static_cast<uint8_t>(0x80 | (u16 & 0x3F)); | |
| 292 | 4081 | size = 2; | |
| 293 | } | ||
| 294 | else { | ||
| 295 | // three bytes, 1110xxxx 10xxxxxx 10xxxxxx | ||
| 296 | 4725 | bytes[0] = static_cast<uint8_t>(0xE0 | ((u16 >> 12) & 0x0F)); | |
| 297 | 4725 | bytes[1] = static_cast<uint8_t>(0x80 | ((u16 >> 6) & 0x3F)); | |
| 298 | 4725 | bytes[2] = static_cast<uint8_t>(0x80 | (u16 & 0x3F)); | |
| 299 | 4725 | size = 3; | |
| 300 | } | ||
| 301 | // convert the array of bytes to a string | ||
| 302 |
1/2✓ Branch 9 → 10 taken 9478 times.
✗ Branch 9 → 13 not taken.
|
9478 | std::string u8str(bytes, bytes + size); |
| 303 | 9478 | return u8str; | |
| 304 | } | ||
| 305 | |||
| 306 | // for posix: always assume utf8 | ||
| 307 | 1 | std::string charToUtf8(const char* text, bool utf8) | |
| 308 | { | ||
| 309 | 1 | std::string s; | |
| 310 | // AVS_POSIX: utf8 is always true, no ANSI here | ||
| 311 | #ifdef AVS_POSIX | ||
| 312 | 1 | utf8 = true; | |
| 313 | #endif | ||
| 314 |
1/2✓ Branch 3 → 4 taken 1 time.
✗ Branch 3 → 5 not taken.
|
1 | if (utf8) { |
| 315 |
1/2✓ Branch 4 → 5 taken 1 time.
✗ Branch 4 → 7 not taken.
|
1 | s = text; // no change |
| 316 | } | ||
| 317 | #ifdef AVS_WINDOWS | ||
| 318 | else { | ||
| 319 | // ANSI (or system local Active CP, since Win10 it can be UTF8), Windows | ||
| 320 | auto source = AnsiToUtf8(text); | ||
| 321 | s = source.get(); | ||
| 322 | } | ||
| 323 | #endif | ||
| 324 | 1 | return s; | |
| 325 | ✗ | } | |
| 326 |