#include "../src/unicode.h" #include #include #include #include int main() { bool success = true; const std::vector, std::vector>> normalization = { {{}, {}}, {{0x65, 0x301}, {0xE9}}, {{0xE9}, {0xE9}}, {{0x212B}, {0xC5}}, {{0x61, 0x315, 0x300}, {0xE0, 0x315}}, {{0x61, 0x305, 0x301}, {0x61, 0x305, 0x301}}, {{0x301, 0x327}, {0x327, 0x301}}, {{0x1100, 0x1161, 0x11A8}, {0xAC01}}, {{0xAC01}, {0xAC01}}, {{0xFB01, 0xFF21}, {0xFB01, 0xFF21}}, {{0x958}, {0x915, 0x93C}}, {{0x1D15E}, {0x1D157, 0x1D165}}, }; for (const auto & test : normalization) { if (unicode_cpts_normalize_nfc(test.first) != test.second || unicode_cpts_normalize_nfc(test.second) != test.second) { fprintf(stderr, "unexpected NFC normalization\n"); success = false; break; } } const struct { std::string text; std::vector expected; } utf8_tests[] = { { "\x7f", { 0x7f } }, { "\xc2\x80", { 0x80 } }, { "\xdf\xbf", { 0x7ff } }, { "\xe0\xa0\x80", { 0x800 } }, { "\xef\xbf\xbf", { 0xffff } }, { "\xf0\x90\x80\x80", { 0x10000 } }, { "\xf4\x8f\xbf\xbf", { 0x10ffff } }, // Invalid-input cases preserve the existing recovery behavior: one U+FFFD per byte. { "\xc0\xa0", { 0xfffd, 0xfffd } }, { "\xc1\xbf", { 0xfffd, 0xfffd } }, { "\xe0\x80\xa0", { 0xfffd, 0xfffd, 0xfffd } }, { "\xe0\x9f\xbf", { 0xfffd, 0xfffd, 0xfffd } }, { "\xf0\x80\x80\xa0", { 0xfffd, 0xfffd, 0xfffd, 0xfffd } }, { "\xf0\x8f\xbf\xbf", { 0xfffd, 0xfffd, 0xfffd, 0xfffd } }, { "\xf4\x90\x80\x80", { 0xfffd, 0xfffd, 0xfffd, 0xfffd } }, { "\xf5\x80\x80\x80", { 0xfffd, 0xfffd, 0xfffd, 0xfffd } }, { "\xf7\xbf\xbf\xbf", { 0xfffd, 0xfffd, 0xfffd, 0xfffd } }, { "\xe2\x80", { 0xfffd, 0xfffd } }, }; for (const auto & test : utf8_tests) { auto expected = test.expected; expected.insert(expected.begin(), 'a'); expected.push_back('b'); if (unicode_cpts_from_utf8("a" + test.text + "b") != expected) { fprintf(stderr, "unexpected UTF-8 decoding for bytes:"); for (const unsigned char byte : test.text) { fprintf(stderr, " %02x", byte); } fprintf(stderr, "\n"); success = false; } } const std::vector regex_exprs = { "[~][A-Za-z]+| ?[\\p{S}]+|\\s+", }; const std::vector expected = { " ~", "foo" }; const auto actual = unicode_regex_split(" ~foo", regex_exprs, false); if (actual != expected) { fprintf(stderr, "unexpected split:"); for (const auto & piece : actual) { fprintf(stderr, " [%s]", piece.c_str()); } fprintf(stderr, "\n"); success = false; } return success ? 0 : 1; }