/* ** unicode.cpp ** handling for conversion / comparison of filenames ** **--------------------------------------------------------------------------- ** Copyright 2019 Christoph Oelckers ** All rights reserved. ** ** Redistribution and use in source and binary forms, with or without ** modification, are permitted provided that the following conditions ** are met: ** ** 1. Redistributions of source code must retain the above copyright ** notice, this list of conditions and the following disclaimer. ** 2. Redistributions in binary form must reproduce the above copyright ** notice, this list of conditions and the following disclaimer in the ** documentation and/or other materials provided with the distribution. ** 3. The name of the author may not be used to endorse or promote products ** derived from this software without specific prior written permission. ** ** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR ** IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES ** OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. ** IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, ** INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT ** NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, ** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY ** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF ** THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **--------------------------------------------------------------------------- ** */ #include "unicode.h" #include "utf8proc.h" namespace FileSys { //========================================================================== // // // //========================================================================== static void utf8_encode(int32_t codepoint, std::vector& buffer) { if (codepoint < 0 || codepoint > 0x10FFFF || (codepoint >= 0xD800 && codepoint < 0xDFFF)) { codepoint = 0xFFFD; } uint8_t buf[4]; auto size = utf8proc_encode_char(codepoint, buf); for(int i = 0; i < size; i++) buffer.push_back(buf[i]); } //========================================================================== // // convert UTF16 to UTF8 (needed for 7z) // //========================================================================== void utf16_to_utf8(const unsigned short* in, std::vector& buffer) { buffer.clear(); if (!*in) return; while (int char1 = *in++) { if (char1 >= 0xD800 && char1 <= 0xDBFF) { int char2 = *in; if (char2 >= 0xDC00 && char2 <= 0xDFFF) { in++; char1 -= 0xD800; char2 -= 0xDC00; char1 <<= 10; char1 += char2; char1 += 0x010000; } else { // invalid code point - replace with placeholder char1 = 0xFFFD; } } else if (char1 >= 0xDC00 && char1 <= 0xDFFF) { // invalid code point - replace with placeholder char1 = 0xFFFD; } utf8_encode(char1, buffer); } buffer.push_back(0); } //========================================================================== // // convert UTF16 to UTF8 (needed for Zip) // //========================================================================== void ibm437_to_utf8(const char* in, std::vector& buffer) { static const uint16_t ibm437map[] = { 0x00c7, 0x00fc, 0x00e9, 0x00e2, 0x00e4, 0x00e0, 0x00e5, 0x00e7, 0x00ea, 0x00eb, 0x00e8, 0x00ef, 0x00ee, 0x00ec, 0x00c4, 0x00c5, 0x00c9, 0x00e6, 0x00c6, 0x00f4, 0x00f6, 0x00f2, 0x00fb, 0x00f9, 0x00ff, 0x00d6, 0x00dc, 0x00a2, 0x00a3, 0x00a5, 0x20a7, 0x0192, 0x00e1, 0x00ed, 0x00f3, 0x00fa, 0x00f1, 0x00d1, 0x00aa, 0x00ba, 0x00bf, 0x2310, 0x00ac, 0x00bd, 0x00bc, 0x00a1, 0x00ab, 0x00bb, 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561, 0x2562, 0x2556, 0x2555, 0x2563, 0x2551, 0x2557, 0x255d, 0x255c, 0x255b, 0x2510, 0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x255e, 0x255f, 0x255a, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256c, 0x2567, 0x2568, 0x2564, 0x2565, 0x2559, 0x2558, 0x2552, 0x2553, 0x256b, 0x256a, 0x2518, 0x250c, 0x2588, 0x2584, 0x258c, 0x2590, 0x2580, 0x03b1, 0x00df, 0x0393, 0x03c0, 0x03a3, 0x03c3, 0x00b5, 0x03c4, 0x03a6, 0x0398, 0x03a9, 0x03b4, 0x221e, 0x03c6, 0x03b5, 0x2229, 0x2261, 0x00b1, 0x2265, 0x2264, 0x2320, 0x2321, 0x00f7, 0x2248, 0x00b0, 0x2219, 0x00b7, 0x221a, 0x207f, 0x00b2, 0x25a0, 0x00a0, }; buffer.clear(); if (!*in) return; while (int char1 = (uint8_t)*in++) { if (char1 >= 0x80) char1 = ibm437map[char1 - 0x80]; utf8_encode(char1, buffer); } buffer.push_back(0); } //========================================================================== // // create a normalized lowercase version of a string. // //========================================================================== char *tolower_normalize(const char *str) { utf8proc_uint8_t *retval; utf8proc_map((const uint8_t*)str, 0, &retval, (utf8proc_option_t)(UTF8PROC_NULLTERM | UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_CASEFOLD)); return (char*)retval; } //========================================================================== // // validates the string for proper UTF-8 // //========================================================================== bool unicode_validate(const char* str) { while (*str != 0) { int cp; int result = utf8proc_iterate((const uint8_t*)str, -1, &cp); if (result < 0) return false; } return true; } }