diff options
-rw-r--r-- | .gitignore | 3 | ||||
-rw-r--r-- | CMakeLists.txt | 2 | ||||
-rw-r--r-- | Makefile | 18 | ||||
-rw-r--r-- | NEWS.md | 15 | ||||
-rw-r--r-- | test/case.c | 50 | ||||
-rw-r--r-- | test/charwidth.c | 2 | ||||
-rw-r--r-- | test/graphemetest.c | 8 | ||||
-rw-r--r-- | test/iterate.c | 160 | ||||
-rw-r--r-- | test/tests.h | 9 | ||||
-rw-r--r-- | test/valid.c | 41 | ||||
-rw-r--r-- | utf8proc.c | 156 | ||||
-rw-r--r-- | utf8proc.h | 22 |
12 files changed, 421 insertions, 65 deletions
@@ -21,3 +21,6 @@ graphemetest utf8proc_data.c.new printproperty charwidth +valid +iterate +case diff --git a/CMakeLists.txt b/CMakeLists.txt index da6aa78..c4df445 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ project (utf8proc C) # Be sure to also update these in Makefile! set(SO_MAJOR 1) -set(SO_MINOR 2) +set(SO_MINOR 3) set(SO_PATCH 0) add_definitions ( @@ -16,7 +16,7 @@ CFLAGS += -std=c99 -pedantic -Wall -fpic -DUTF8PROC_EXPORTS # compatibility is broken, even if the API is backward-compatible # Be sure to also update these in CMakeLists.txt! MAJOR=1 -MINOR=2 +MINOR=3 PATCH=0 OS := $(shell uname) @@ -40,7 +40,7 @@ includedir=$(prefix)/include all: libutf8proc.a libutf8proc.$(SHLIB_EXT) clean: - rm -f utf8proc.o libutf8proc.a libutf8proc.$(SHLIB_VERS_EXT) libutf8proc.$(SHLIB_EXT) test/normtest test/graphemetest test/printproperty test/charwidth + rm -f utf8proc.o libutf8proc.a libutf8proc.$(SHLIB_VERS_EXT) libutf8proc.$(SHLIB_EXT) test/normtest test/graphemetest test/printproperty test/charwidth test/valid test/iterate $(MAKE) -C bench clean $(MAKE) -C data clean @@ -107,8 +107,20 @@ test/printproperty: test/printproperty.c utf8proc.o utf8proc.h test/tests.h test/charwidth: test/charwidth.c utf8proc.o utf8proc.h test/tests.h $(CC) test/charwidth.c utf8proc.o -o $@ -check: test/normtest data/NormalizationTest.txt test/graphemetest data/GraphemeBreakTest.txt test/printproperty test/charwidth bench/bench.c bench/util.c bench/util.h utf8proc.o +test/valid: test/valid.c utf8proc.o utf8proc.h test/tests.h + $(cc) test/valid.c utf8proc.o -o $@ + +test/iterate: test/iterate.c utf8proc.o utf8proc.h test/tests.h + $(cc) test/iterate.c utf8proc.o -o $@ + +test/case: test/case.c utf8proc.o utf8proc.h test/tests.h + $(cc) test/case.c utf8proc.o -o $@ + +check: test/normtest data/NormalizationTest.txt test/graphemetest data/GraphemeBreakTest.txt test/printproperty test/case test/charwidth test/valid test/iterate bench/bench.c bench/util.c bench/util.h utf8proc.o $(MAKE) -C bench test/normtest data/NormalizationTest.txt test/graphemetest data/GraphemeBreakTest.txt test/charwidth + test/valid + test/iterate + test/case @@ -1,5 +1,20 @@ # utf8proc release history # +## Version 1.3-dev ## + +- New `utf8proc_tolower` and `utf8proc_toupper` functions, portable + replacements for `towlower` and `towupper` in the C library ([#40]). + +- Don't treat Unicode "non-characters" as invalid, and improved + validity checking in general ([#35]). + +- Prefix all typedefs with `utf8proc_`, e.g. `utf8proc_int32_t`, + to avoid collisions with other libraries ([#32]). + +- Rename `DLLEXPORT` to `UTF8PROC_DLLEXPORT` to prevent collisions. + +- Fix build breakage in the benchmark routines. + ## Version 1.2 ## 2015-03-28: diff --git a/test/case.c b/test/case.c new file mode 100644 index 0000000..39958e3 --- /dev/null +++ b/test/case.c @@ -0,0 +1,50 @@ +#include "tests.h" +#include <wctype.h> + +int main(int argc, char **argv) +{ + int error = 0, better = 0; + utf8proc_int32_t c; + + (void) argc; /* unused */ + (void) argv; /* unused */ + + /* some simple sanity tests of the character widths */ + for (c = 0; c <= 0x110000; ++c) { + utf8proc_int32_t l = utf8proc_tolower(c); + utf8proc_int32_t u = utf8proc_toupper(c); + + check(l == c || utf8proc_codepoint_valid(l), "invalid tolower"); + check(u == c || utf8proc_codepoint_valid(u), "invalid toupper"); + + if (sizeof(wint_t) > 2 || c < (1<<16)) { + wint_t l0 = towlower(c), u0 = towupper(c); + + /* OS unicode tables may be out of date. But if they + do have a lower/uppercase mapping, hopefully it + is correct? */ + if (l0 != c && l0 != l) { + fprintf(stderr, "MISMATCH %x != towlower(%x) == %x\n", + l, c, l0); + ++error; + } + else if (l0 != l) { /* often true for out-of-date OS unicode */ + ++better; + /* printf("%x != towlower(%x) == %x\n", l, c, l0); */ + } + if (u0 != c && u0 != u) { + fprintf(stderr, "MISMATCH %x != towupper(%x) == %x\n", + u, c, u0); + ++error; + } + else if (u0 != u) { /* often true for out-of-date OS unicode */ + ++better; + /* printf("%x != towupper(%x) == %x\n", u, c, u0); */ + } + } + } + check(!error, "utf8proc case conversion FAILED %d tests.", error); + printf("More up-to-date than OS unicode tables for %d tests.\n", better); + printf("utf8proc case conversion tests SUCCEEDED.\n"); + return 0; +} diff --git a/test/charwidth.c b/test/charwidth.c index bdae9fa..28554a4 100644 --- a/test/charwidth.c +++ b/test/charwidth.c @@ -10,7 +10,7 @@ int my_isprint(int c) { int main(int argc, char **argv) { - int prevc, c, error = 0; + int c, error = 0; (void) argc; /* unused */ (void) argv; /* unused */ diff --git a/test/graphemetest.c b/test/graphemetest.c index a05b6e2..eb3645b 100644 --- a/test/graphemetest.c +++ b/test/graphemetest.c @@ -6,6 +6,7 @@ int main(int argc, char **argv) size_t bufsize = 0; FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL; utf8proc_uint8_t src[1024]; + int len; check(f != NULL, "error opening GraphemeBreakTest.txt"); while (getline(&buf, &bufsize, f) > 0) { @@ -29,9 +30,10 @@ int main(int argc, char **argv) else if (buf[bi] == '#') { /* start of comments */ break; } - else { /* hex-encoded codepoint */ - bi += encode((char*) (src + si), buf + bi) - 1; + else { /* hex-encoded codepoint */ + len = encode((char*) (src + si), buf + bi) - 1; while (src[si]) ++si; /* advance to NUL termination */ + bi += len; } } if (si && src[si-1] == '/') @@ -59,7 +61,7 @@ int main(int argc, char **argv) utf8proc_errmsg(glen)); for (i = 0; i <= glen; ++i) if (g[i] == 0xff) - g[i] = '/'; /* easier-to-read output (/ is not in test strings) */ + g[i] = '/'; /* easier-to-read output (/ is not in test strings) */ check(!strcmp((char*)g, (char*)src), "grapheme mismatch: \"%s\" instead of \"%s\"", (char*)g, (char*)src); } diff --git a/test/iterate.c b/test/iterate.c new file mode 100644 index 0000000..cd68a52 --- /dev/null +++ b/test/iterate.c @@ -0,0 +1,160 @@ +#include "tests.h" +#include <ctype.h> +#include <wchar.h> + +static int tests; +static int error; + +#define CHECKVALID(pos, val, len) buf[pos] = val; testbytes(buf,len,len,__LINE__) +#define CHECKINVALID(pos, val, len) buf[pos] = val; testbytes(buf,len,UTF8PROC_ERROR_INVALIDUTF8,__LINE__) + +void testbytes(unsigned char *buf, int len, utf8proc_ssize_t retval, int line) +{ + utf8proc_int32_t out[16]; + utf8proc_ssize_t ret; + + tests++; + if ((ret = utf8proc_iterate(buf, len, out)) != retval) { + fprintf(stderr, "Failed (%d):", line); + for (int i = 0; i < len ; i++) { + fprintf(stderr, " 0x%02x", buf[i]); + } + fprintf(stderr, " -> %zd\n", ret); + error++; + } +} + +int main(int argc, char **argv) +{ + uint32_t byt; + unsigned char buf[16]; + + tests = error = 0; + + // Check valid sequences that were considered valid erroneously before + buf[0] = 0xef; + buf[1] = 0xb7; + for (byt = 0x90; byt < 0xa0; byt++) { + CHECKVALID(2, byt, 3); + } + // Check 0xfffe and 0xffff + buf[1] = 0xbf; + CHECKVALID(2, 0xbe, 3); + CHECKVALID(2, 0xbf, 3); + // Check 0x??fffe & 0x??ffff + for (byt = 0x1fffe; byt < 0x110000; byt += 0x10000) { + buf[0] = 0xf0 | (byt >> 18); + buf[1] = 0x80 | ((byt >> 12) & 0x3f); + CHECKVALID(3, 0xbe, 4); + CHECKVALID(3, 0xbf, 4); + } + + // Continuation byte not after lead + for (byt = 0x80; byt < 0xc0; byt++) { + CHECKINVALID(0, byt, 1); + } + + // Continuation byte not after lead + for (byt = 0x80; byt < 0xc0; byt++) { + CHECKINVALID(0, byt, 1); + } + + // Test lead bytes + for (byt = 0xc0; byt <= 0xff; byt++) { + // Single lead byte at end of string + CHECKINVALID(0, byt, 1); + // Lead followed by non-continuation character < 0x80 + CHECKINVALID(1, 65, 2); + // Lead followed by non-continuation character > 0xbf + CHECKINVALID(1, 0xc0, 2); + } + + // Test overlong 2-byte + buf[0] = 0xc0; + for (byt = 0x81; byt <= 0xbf; byt++) { + CHECKINVALID(1, byt, 2); + } + buf[0] = 0xc1; + for (byt = 0x80; byt <= 0xbf; byt++) { + CHECKINVALID(1, byt, 2); + } + + // Test overlong 3-byte + buf[0] = 0xe0; + buf[2] = 0x80; + for (byt = 0x80; byt <= 0x9f; byt++) { + CHECKINVALID(1, byt, 3); + } + + // Test overlong 4-byte + buf[0] = 0xf0; + buf[2] = 0x80; + buf[3] = 0x80; + for (byt = 0x80; byt <= 0x8f; byt++) { + CHECKINVALID(1, byt, 4); + } + + // Test 4-byte > 0x10ffff + buf[0] = 0xf4; + buf[2] = 0x80; + buf[3] = 0x80; + for (byt = 0x90; byt <= 0xbf; byt++) { + CHECKINVALID(1, byt, 4); + } + buf[1] = 0x80; + for (byt = 0xf5; byt <= 0xf7; byt++) { + CHECKINVALID(0, byt, 4); + } + + // Test 5-byte + buf[4] = 0x80; + for (byt = 0xf8; byt <= 0xfb; byt++) { + CHECKINVALID(0, byt, 5); + } + + // Test 6-byte + buf[5] = 0x80; + for (byt = 0xfc; byt <= 0xfd; byt++) { + CHECKINVALID(0, byt, 6); + } + + // Test 7-byte + buf[6] = 0x80; + CHECKINVALID(0, 0xfe, 7); + + // Three and above byte sequences + for (byt = 0xe0; byt < 0xf0; byt++) { + // Lead followed by only 1 continuation byte + CHECKINVALID(0, byt, 2); + // Lead ended by non-continuation character < 0x80 + CHECKINVALID(2, 65, 3); + // Lead ended by non-continuation character > 0xbf + CHECKINVALID(2, 0xc0, 3); + } + + // 3-byte encoded surrogate character(s) + buf[0] = 0xed; buf[2] = 0x80; + // Single surrogate + CHECKINVALID(1, 0xa0, 3); + // Trailing surrogate first + CHECKINVALID(1, 0xb0, 3); + + // Four byte sequences + buf[1] = 0x80; + for (byt = 0xf0; byt < 0xf5; byt++) { + // Lead followed by only 1 continuation bytes + CHECKINVALID(0, byt, 2); + // Lead followed by only 2 continuation bytes + CHECKINVALID(0, byt, 3); + // Lead followed by non-continuation character < 0x80 + CHECKINVALID(3, 65, 4); + // Lead followed by non-continuation character > 0xbf + CHECKINVALID(3, 0xc0, 4); + + } + + check(!error, "utf8proc_iterate FAILED %d tests out of %d", error, tests); + printf("utf8proc_iterate tests SUCCEEDED, (%d) tests passed.\n", tests); + + return 0; +} diff --git a/test/tests.h b/test/tests.h index c27185d..6eb5457 100644 --- a/test/tests.h +++ b/test/tests.h @@ -33,10 +33,11 @@ size_t skipspaces(const char *buf, size_t i) separated by whitespace, and terminated by any character not in [0-9a-fA-F] or whitespace, then stores the corresponding utf8 string in dest, returning the number of bytes read from buf */ +utf8proc_ssize_t unsafe_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst); size_t encode(char *dest, const char *buf) { size_t i = 0, j, d = 0; - do { + for (;;) { int c; i = skipspaces(buf, i); for (j=i; buf[j] && strchr("0123456789abcdef", tolower(buf[j])); ++j) @@ -45,9 +46,9 @@ size_t encode(char *dest, const char *buf) dest[d] = 0; /* NUL-terminate destination string */ return i + 1; } - check(sscanf(buf + i, "%x", &c) == 1, "invalid hex input %s", buf+i); + check(sscanf(buf + i, "%x", (unsigned int *)&c) == 1, "invalid hex input %s", buf+i); i = j; /* skip to char after hex input */ - d += utf8proc_encode_char(c, (utf8proc_uint8_t *) (dest + d)); - } while (1); + d += unsafe_encode_char(c, (utf8proc_uint8_t *) (dest + d)); + } } diff --git a/test/valid.c b/test/valid.c new file mode 100644 index 0000000..eadfb85 --- /dev/null +++ b/test/valid.c @@ -0,0 +1,41 @@ +#include "tests.h" +#include <ctype.h> +#include <wchar.h> + +int main(int argc, char **argv) +{ + int c, error = 0; + + (void) argc; /* unused */ + (void) argv; /* unused */ + + /* some simple sanity tests of */ + for (c = 0; c < 0xd800; c++) { + if (!utf8proc_codepoint_valid(c)) { + fprintf(stderr, "Failed: codepoint_valid(%04x) -> false\n", c); + error++; + } + } + for (;c < 0xe000; c++) { + if (utf8proc_codepoint_valid(c)) { + fprintf(stderr, "Failed: codepoint_valid(%04x) -> true\n", c); + error++; + } + } + for (;c < 0x110000; c++) { + if (!utf8proc_codepoint_valid(c)) { + fprintf(stderr, "Failed: codepoint_valid(%06x) -> false\n", c); + error++; + } + } + for (;c < 0x110010; c++) { + if (utf8proc_codepoint_valid(c)) { + fprintf(stderr, "Failed: codepoint_valid(%06x) -> true\n", c); + error++; + } + } + check(!error, "utf8proc_codepoint_valid FAILED %d tests.", error); + printf("Validity tests SUCCEEDED.\n"); + + return 0; +} @@ -108,51 +108,57 @@ UTF8PROC_DLLEXPORT const char *utf8proc_errmsg(utf8proc_ssize_t errcode) { } } +#define utf_cont(ch) (((ch) & 0xc0) == 0x80) UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_iterate( const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *dst ) { - int length; - int i; - utf8proc_int32_t uc = -1; + utf8proc_uint32_t uc; + const utf8proc_uint8_t *end; + *dst = -1; if (!strlen) return 0; - length = utf8proc_utf8class[str[0]]; - if (!length) return UTF8PROC_ERROR_INVALIDUTF8; - if (strlen >= 0 && length > strlen) return UTF8PROC_ERROR_INVALIDUTF8; - for (i=1; i<length; i++) { - if ((str[i] & 0xC0) != 0x80) return UTF8PROC_ERROR_INVALIDUTF8; + end = str + ((strlen < 0) ? 4 : strlen); + uc = *str++; + if (uc < 0x80) { + *dst = uc; + return 1; + } + // Must be between 0xc2 and 0xf4 inclusive to be valid + if ((uc - 0xc2) > (0xf4-0xc2)) return UTF8PROC_ERROR_INVALIDUTF8; + if (uc < 0xe0) { // 2-byte sequence + // Must have valid continuation character + if (!utf_cont(*str)) return UTF8PROC_ERROR_INVALIDUTF8; + *dst = ((uc & 0x1f)<<6) | (*str & 0x3f); + return 2; } - switch (length) { - case 1: - uc = str[0]; - break; - case 2: - uc = ((str[0] & 0x1F) << 6) + (str[1] & 0x3F); - if (uc < 0x80) uc = -1; - break; - case 3: - uc = ((str[0] & 0x0F) << 12) + ((str[1] & 0x3F) << 6) - + (str[2] & 0x3F); - if (uc < 0x800 || (uc >= 0xD800 && uc < 0xE000) || - (uc >= 0xFDD0 && uc < 0xFDF0)) uc = -1; - break; - case 4: - uc = ((str[0] & 0x07) << 18) + ((str[1] & 0x3F) << 12) - + ((str[2] & 0x3F) << 6) + (str[3] & 0x3F); - if (uc < 0x10000 || uc >= 0x110000) uc = -1; - break; + if (uc < 0xf0) { // 3-byte sequence + if ((str + 1 >= end) || !utf_cont(*str) || !utf_cont(str[1])) + return UTF8PROC_ERROR_INVALIDUTF8; + // Check for surrogate chars + if (uc == 0xed && *str > 0x9f) + return UTF8PROC_ERROR_INVALIDUTF8; + uc = ((uc & 0xf)<<12) | ((*str & 0x3f)<<6) | (str[1] & 0x3f); + if (uc < 0x800) + return UTF8PROC_ERROR_INVALIDUTF8; + *dst = uc; + return 3; } - if (uc < 0 || ((uc & 0xFFFF) >= 0xFFFE)) - return UTF8PROC_ERROR_INVALIDUTF8; - *dst = uc; - return length; + // 4-byte sequence + // Must have 3 valid continuation characters + if ((str + 2 >= end) || !utf_cont(*str) || !utf_cont(str[1]) || !utf_cont(str[2])) + return UTF8PROC_ERROR_INVALIDUTF8; + // Make sure in correct range (0x10000 - 0x10ffff) + if (uc == 0xf0) { + if (*str < 0x90) return UTF8PROC_ERROR_INVALIDUTF8; + } else if (uc == 0xf4) { + if (*str > 0x8f) return UTF8PROC_ERROR_INVALIDUTF8; + } + *dst = ((uc & 7)<<18) | ((*str & 0x3f)<<12) | ((str[1] & 0x3f)<<6) | (str[2] & 0x3f); + return 4; } UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_codepoint_valid(utf8proc_int32_t uc) { - if (uc < 0 || uc >= 0x110000 || - ((uc & 0xFFFF) >= 0xFFFE) || (uc >= 0xD800 && uc < 0xE000) || - (uc >= 0xFDD0 && uc < 0xFDF0)) return false; - else return true; + return (((utf8proc_uint32_t)uc)-0xd800 > 0x07ff) && ((utf8proc_uint32_t)uc < 0x110000); } UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst) { @@ -165,12 +171,8 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t uc, ut dst[0] = 0xC0 + (uc >> 6); dst[1] = 0x80 + (uc & 0x3F); return 2; - } else if (uc == 0xFFFF) { - dst[0] = 0xFF; - return 1; - } else if (uc == 0xFFFE) { - dst[0] = 0xFE; - return 1; + // Note: we allow encoding 0xd800-0xdfff here, so as not to change + // the API, however, these are actually invalid in UTF-8 } else if (uc < 0x10000) { dst[0] = 0xE0 + (uc >> 12); dst[1] = 0x80 + ((uc >> 6) & 0x3F); @@ -186,7 +188,38 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t uc, ut } /* internal "unsafe" version that does not check whether uc is in range */ -static const utf8proc_property_t *get_property(utf8proc_int32_t uc) { +utf8proc_ssize_t unsafe_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst) { + if (uc < 0x00) { + return 0; + } else if (uc < 0x80) { + dst[0] = uc; + return 1; + } else if (uc < 0x800) { + dst[0] = 0xC0 + (uc >> 6); + dst[1] = 0x80 + (uc & 0x3F); + return 2; + } else if (uc == 0xFFFF) { + dst[0] = 0xFF; + return 1; + } else if (uc == 0xFFFE) { + dst[0] = 0xFE; + return 1; + } else if (uc < 0x10000) { + dst[0] = 0xE0 + (uc >> 12); + dst[1] = 0x80 + ((uc >> 6) & 0x3F); + dst[2] = 0x80 + (uc & 0x3F); + return 3; + } else if (uc < 0x110000) { + dst[0] = 0xF0 + (uc >> 18); + dst[1] = 0x80 + ((uc >> 12) & 0x3F); + dst[2] = 0x80 + ((uc >> 6) & 0x3F); + dst[3] = 0x80 + (uc & 0x3F); + return 4; + } else return 0; +} + +/* internal "unsafe" version that does not check whether uc is in range */ +static const utf8proc_property_t *unsafe_get_property(utf8proc_int32_t uc) { /* ASSERT: uc >= 0 && uc < 0x110000 */ return utf8proc_properties + ( utf8proc_stage2table[ @@ -196,7 +229,7 @@ static const utf8proc_property_t *get_property(utf8proc_int32_t uc) { } UTF8PROC_DLLEXPORT const utf8proc_property_t *utf8proc_get_property(utf8proc_int32_t uc) { - return uc < 0 || uc >= 0x110000 ? utf8proc_properties : get_property(uc); + return uc < 0 || uc >= 0x110000 ? utf8proc_properties : unsafe_get_property(uc); } /* return whether there is a grapheme break between boundclasses lbc and tbc */ @@ -231,6 +264,18 @@ UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break(utf8proc_int32_t c1, ut utf8proc_get_property(c2)->boundclass); } +UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_tolower(utf8proc_int32_t c) +{ + utf8proc_int32_t cl = utf8proc_get_property(c)->lowercase_mapping; + return cl >= 0 ? cl : c; +} + +UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_toupper(utf8proc_int32_t c) +{ + utf8proc_int32_t cu = utf8proc_get_property(c)->uppercase_mapping; + return cu >= 0 ? cu : c; +} + /* return a character width analogous to wcwidth (except portable and hopefully less buggy than most system wcwidth functions). */ UTF8PROC_DLLEXPORT int utf8proc_charwidth(utf8proc_int32_t c) { @@ -255,7 +300,7 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char(utf8proc_int32_t uc, utf8proc_propval_t category; utf8proc_int32_t hangul_sindex; if (uc < 0 || uc >= 0x110000) return UTF8PROC_ERROR_NOTASSIGNED; - property = get_property(uc); + property = unsafe_get_property(uc); category = property->category; hangul_sindex = uc - UTF8PROC_HANGUL_SBASE; if (options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) { @@ -401,8 +446,8 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose( const utf8proc_property_t *property1, *property2; uc1 = buffer[pos]; uc2 = buffer[pos+1]; - property1 = get_property(uc1); - property2 = get_property(uc2); + property1 = unsafe_get_property(uc1); + property2 = unsafe_get_property(uc2); if (property1->combining_class > property2->combining_class && property2->combining_class > 0) { buffer[pos] = uc2; @@ -460,7 +505,7 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, utf8proc_int32_t composition; for (rpos = 0; rpos < length; rpos++) { current_char = buffer[rpos]; - current_property = get_property(current_char); + current_property = unsafe_get_property(current_char); if (starter && current_property->combining_class > max_combining_class) { /* combination perhaps possible */ utf8proc_int32_t hangul_lindex; @@ -489,7 +534,7 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, } } if (!starter_property) { - starter_property = get_property(*starter); + starter_property = unsafe_get_property(*starter); } if (starter_property->comb1st_index >= 0 && current_property->comb2nd_index >= 0) { @@ -498,7 +543,7 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, current_property->comb2nd_index ]; if (composition >= 0 && (!(options & UTF8PROC_STABLE) || - !(get_property(composition)->comp_exclusion))) { + !(unsafe_get_property(composition)->comp_exclusion))) { *starter = composition; starter_property = NULL; continue; @@ -522,9 +567,16 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, { utf8proc_ssize_t rpos, wpos = 0; utf8proc_int32_t uc; - for (rpos = 0; rpos < length; rpos++) { - uc = buffer[rpos]; - wpos += utf8proc_encode_char(uc, ((utf8proc_uint8_t *)buffer) + wpos); + if (options & UTF8PROC_CHARBOUND) { + for (rpos = 0; rpos < length; rpos++) { + uc = buffer[rpos]; + wpos += unsafe_encode_char(uc, ((utf8proc_uint8_t *)buffer) + wpos); + } + } else { + for (rpos = 0; rpos < length; rpos++) { + uc = buffer[rpos]; + wpos += utf8proc_encode_char(uc, ((utf8proc_uint8_t *)buffer) + wpos); + } } ((utf8proc_uint8_t *)buffer)[wpos] = 0; return wpos; @@ -69,7 +69,7 @@ /** The MAJOR version number (increased when backwards API compatibility is broken). */ #define UTF8PROC_VERSION_MAJOR 1 /** The MINOR version number (increased when new functionality is added in a backwards-compatible manner). */ -#define UTF8PROC_VERSION_MINOR 2 +#define UTF8PROC_VERSION_MINOR 3 /** The PATCH version (increased for fixes that do not change the API). */ #define UTF8PROC_VERSION_PATCH 0 /** @} */ @@ -82,10 +82,13 @@ typedef unsigned char utf8proc_uint8_t; typedef short utf8proc_int16_t; typedef unsigned short utf8proc_uint16_t; typedef int utf8proc_int32_t; +typedef unsigned int utf8proc_uint32_t; # ifdef _WIN64 typedef __int64 utf8proc_ssize_t; +typedef unsigned __int64 utf8proc_size_t; # else typedef int utf8proc_ssize_t; +typedef unsigned int utf8proc_size_t; # endif # ifndef __cplusplus typedef unsigned char utf8proc_bool; @@ -101,6 +104,8 @@ typedef uint8_t utf8proc_uint8_t; typedef int16_t utf8proc_int16_t; typedef uint16_t utf8proc_uint16_t; typedef int32_t utf8proc_int32_t; +typedef uint32_t utf8proc_uint32_t; +typedef size_t utf8proc_size_t; typedef ssize_t utf8proc_ssize_t; typedef bool utf8proc_bool; #endif @@ -506,6 +511,21 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, */ UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break(utf8proc_int32_t codepoint1, utf8proc_int32_t codepoint2); + +/** + * Given a codepoint `c`, return the codepoint of the corresponding + * lower-case character, if any; otherwise (if there is no lower-case + * variant, or if `c` is not a valid codepoint) return `c`. + */ +UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_tolower(utf8proc_int32_t c); + +/** + * Given a codepoint `c`, return the codepoint of the corresponding + * upper-case character, if any; otherwise (if there is no upper-case + * variant, or if `c` is not a valid codepoint) return `c`. + */ +UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_toupper(utf8proc_int32_t c); + /** * Given a codepoint, return a character width analogous to `wcwidth(codepoint)`, * except that a width of 0 is returned for non-printable codepoints |