diff options
author | Steven G. Johnson <stevenj@mit.edu> | 2015-03-06 17:36:08 -0500 |
---|---|---|
committer | Steven G. Johnson <stevenj@mit.edu> | 2015-03-06 17:36:08 -0500 |
commit | 90721f2d39b0cdd5d22409f1bf4f6ce4b7382944 (patch) | |
tree | 2b3ec29840c95bc551d86d40902120db5eb1da12 /test | |
parent | 10f7e2ed5a7f3d05cbbc45f457be12456e6969d3 (diff) | |
download | libutf8proc-90721f2d39b0cdd5d22409f1bf4f6ce4b7382944.tar.gz libutf8proc-90721f2d39b0cdd5d22409f1bf4f6ce4b7382944.tar.bz2 |
directory cleanup: move tests and data into subdirectories
Diffstat (limited to 'test')
-rw-r--r-- | test/graphemetest.c | 72 | ||||
-rw-r--r-- | test/normtest.c | 64 | ||||
-rw-r--r-- | test/printproperty.c | 45 | ||||
-rw-r--r-- | test/tests.h | 53 |
4 files changed, 234 insertions, 0 deletions
diff --git a/test/graphemetest.c b/test/graphemetest.c new file mode 100644 index 0000000..7914dc8 --- /dev/null +++ b/test/graphemetest.c @@ -0,0 +1,72 @@ +#include "tests.h" + +int main(int argc, char **argv) +{ + char *buf = NULL; + size_t bufsize = 0; + FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL; + uint8_t src[1024]; + + check(f != NULL, "error opening GraphemeBreakTest.txt"); + while (getline(&buf, &bufsize, f) > 0) { + size_t bi = 0, si = 0; + lineno += 1; + + if (lineno % 100 == 0) + printf("checking line %zd...\n", lineno); + + if (buf[0] == '#') continue; + + while (buf[bi]) { + bi = skipspaces(buf, bi); + if (buf[bi] == '/') { /* grapheme break */ + src[si++] = '/'; + bi++; + } + else if (buf[bi] == '+') { /* no break */ + bi++; + } + else if (buf[bi] == '#') { /* start of comments */ + break; + } + else { /* hex-encoded codepoint */ + bi += encode((char*) (src + si), buf + bi) - 1; + while (src[si]) ++si; /* advance to NUL termination */ + } + } + if (si && src[si-1] == '/') + --si; /* no break after final grapheme */ + src[si] = 0; /* NUL-terminate */ + + if (si) { + uint8_t utf8[1024]; /* copy src without 0xff grapheme separators */ + size_t i = 0, j = 0; + ssize_t glen; + uint8_t *g; /* utf8proc_map grapheme results */ + while (i < si) { + if (src[i] != '/') + utf8[j++] = src[i++]; + else + i++; + } + glen = utf8proc_map(utf8, j, &g, UTF8PROC_CHARBOUND); + if (glen == UTF8PROC_ERROR_INVALIDUTF8) { + /* the test file contains surrogate codepoints, which are only for UTF-16 */ + printf("line %zd: ignoring invalid UTF-8 codepoints\n", lineno); + } + else { + check(glen >= 0, "utf8proc_map error = %s", + utf8proc_errmsg(glen)); + for (i = 0; i <= glen; ++i) + if (g[i] == 0xff) + g[i] = '/'; /* easier-to-read output (/ is not in test strings) */ + check(!strcmp((char*)g, (char*)src), + "grapheme mismatch: \"%s\" instead of \"%s\"", (char*)g, (char*)src); + } + free(g); + } + } + fclose(f); + printf("Passed tests after %zd lines!\n", lineno); + return 0; +} diff --git a/test/normtest.c b/test/normtest.c new file mode 100644 index 0000000..7add4c6 --- /dev/null +++ b/test/normtest.c @@ -0,0 +1,64 @@ +#include "tests.h" + +#define CHECK_NORM(NRM, norm, src) { \ + char *src_norm = (char*) utf8proc_ ## NRM((uint8_t*) src); \ + check(!strcmp(norm, src_norm), \ + "normalization failed for %s -> %s", src, norm); \ + free(src_norm); \ +} + +int main(int argc, char **argv) +{ + char *buf = NULL; + size_t bufsize = 0; + FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL; + char source[1024], NFC[1024], NFD[1024], NFKC[1024], NFKD[1024]; + + check(f != NULL, "error opening NormalizationTest.txt"); + while (getline(&buf, &bufsize, f) > 0) { + size_t offset; + lineno += 1; + + if (buf[0] == '@') { + printf("line %zd: %s", lineno, buf + 1); + continue; + } + else if (lineno % 1000 == 0) + printf("checking line %zd...\n", lineno); + + if (buf[0] == '#') continue; + + offset = encode(source, buf); + offset += encode(NFC, buf + offset); + offset += encode(NFD, buf + offset); + offset += encode(NFKC, buf + offset); + offset += encode(NFKD, buf + offset); + + CHECK_NORM(NFC, NFC, source); + CHECK_NORM(NFC, NFC, NFC); + CHECK_NORM(NFC, NFC, NFD); + CHECK_NORM(NFC, NFKC, NFKC); + CHECK_NORM(NFC, NFKC, NFKD); + + CHECK_NORM(NFD, NFD, source); + CHECK_NORM(NFD, NFD, NFC); + CHECK_NORM(NFD, NFD, NFD); + CHECK_NORM(NFD, NFKD, NFKC); + CHECK_NORM(NFD, NFKD, NFKD); + + CHECK_NORM(NFKC, NFKC, source); + CHECK_NORM(NFKC, NFKC, NFC); + CHECK_NORM(NFKC, NFKC, NFD); + CHECK_NORM(NFKC, NFKC, NFKC); + CHECK_NORM(NFKC, NFKC, NFKD); + + CHECK_NORM(NFKD, NFKD, source); + CHECK_NORM(NFKD, NFKD, NFC); + CHECK_NORM(NFKD, NFKD, NFD); + CHECK_NORM(NFKD, NFKD, NFKC); + CHECK_NORM(NFKD, NFKD, NFKD); + } + fclose(f); + printf("Passed tests after %zd lines!\n", lineno); + return 0; +} diff --git a/test/printproperty.c b/test/printproperty.c new file mode 100644 index 0000000..37e4fa3 --- /dev/null +++ b/test/printproperty.c @@ -0,0 +1,45 @@ +/* simple test program to print out the utf8proc properties for a codepoint */ + +#include "tests.h" + +int main(int argc, char **argv) +{ + int i; + + for (i = 1; i < argc; ++i) { + int c; + check(sscanf(argv[i],"%x",&c) == 1, "invalid hex input %s", argv[i]); + const utf8proc_property_t *p = utf8proc_get_property(c); + printf("U+%s:\n" + " category = %d\n" + " combining_class = %d\n" + " bidi_class = %d\n" + " decomp_type = %d\n" + " uppercase_mapping = %x\n" + " lowercase_mapping = %x\n" + " titlecase_mapping = %x\n" + " comb1st_index = %d\n" + " comb2nd_index = %d\n" + " bidi_mirrored = %d\n" + " comp_exclusion = %d\n" + " ignorable = %d\n" + " control_boundary = %d\n" + " boundclass = %d\n", + argv[i], + p->category, + p->combining_class, + p->bidi_class, + p->decomp_type, + p->uppercase_mapping, + p->lowercase_mapping, + p->titlecase_mapping, + p->comb1st_index, + p->comb2nd_index, + p->bidi_mirrored, + p->comp_exclusion, + p->ignorable, + p->control_boundary, + p->boundclass); + } + return 0; +} diff --git a/test/tests.h b/test/tests.h new file mode 100644 index 0000000..d4897f8 --- /dev/null +++ b/test/tests.h @@ -0,0 +1,53 @@ +/* Common functions and includes for our test programs. */ + +#include <stdio.h> +#include <stdlib.h> +#include <ctype.h> +#include <string.h> +#include <stdarg.h> + +#include "../utf8proc.h" + +size_t lineno = 0; + +void check(int cond, const char *format, ...) +{ + if (!cond) { + va_list args; + fprintf(stderr, "line %zd: ", lineno); + va_start(args, format); + vfprintf(stderr, format, args); + va_end(args); + fprintf(stderr, "\n"); + exit(1); + } +} + +size_t skipspaces(const char *buf, size_t i) +{ + while (isspace(buf[i])) ++i; + return i; +} + +/* if buf points to a sequence of codepoints encoded as hexadecimal strings, + separated by whitespace, and terminated by any character not in + [0-9a-fA-F] or whitespace, then stores the corresponding utf8 string + in dest, returning the number of bytes read from buf */ +size_t encode(char *dest, const char *buf) +{ + size_t i = 0, j, d = 0; + do { + int c; + i = skipspaces(buf, i); + for (j=i; buf[j] && strchr("0123456789abcdef", tolower(buf[j])); ++j) + ; /* find end of hex input */ + if (j == i) { /* no codepoint found */ + dest[d] = 0; /* NUL-terminate destination string */ + return i + 1; + } + check(sscanf(buf + i, "%x", &c) == 1, "invalid hex input %s", buf+i); + i = j; /* skip to char after hex input */ + d += utf8proc_encode_char(c, (uint8_t *) (dest + d)); + } while (1); +} + |