summaryrefslogtreecommitdiff
path: root/utils/utf8.c
diff options
context:
space:
mode:
authorJohn Mark Bell <jmb@netsurf-browser.org>2005-04-16 05:09:33 +0000
committerJohn Mark Bell <jmb@netsurf-browser.org>2005-04-16 05:09:33 +0000
commitc17dc661eae89cea66959fad4fe48d77f1faf174 (patch)
tree1909b4157137e66b3f74adc75ca1a2bbf3bc3c6f /utils/utf8.c
parent37a4119ab80b1a7c7e37707c4e029218bdcbe549 (diff)
downloadnetsurf-c17dc661eae89cea66959fad4fe48d77f1faf174.tar.gz
netsurf-c17dc661eae89cea66959fad4fe48d77f1faf174.tar.bz2
[project @ 2005-04-16 05:09:32 by jmb]
Split out UTF-8 handling functions. Submit URL-encoded forms in sensible encoding: * First entry in accept-charset list, if present * Document encoding, otherwise We may want to explicitly look for UTF-8, to save converting. Convert cnv_str_local_enc/cnv_local_enc_str to use iconv (they're now veneers for utf8_[to/from]_enc). Provide mechanism for looking up local system charset (derived from system alphabet, under RISC OS) svn path=/import/netsurf/; revision=1647
Diffstat (limited to 'utils/utf8.c')
-rw-r--r--utils/utf8.c280
1 files changed, 280 insertions, 0 deletions
diff --git a/utils/utf8.c b/utils/utf8.c
new file mode 100644
index 000000000..957062a37
--- /dev/null
+++ b/utils/utf8.c
@@ -0,0 +1,280 @@
+/*
+ * This file is part of NetSurf, http://netsurf.sourceforge.net/
+ * Licensed under the GNU General Public License,
+ * http://www.opensource.org/licenses/gpl-license
+ * Copyright 2005 John M Bell <jmb202@ecs.soton.ac.uk>
+ */
+
+/**
+ * \file UTF-8 manipulation functions (implementation)
+ */
+
+#include <assert.h>
+#include <string.h>
+
+#include <iconv.h>
+
+#include "netsurf/utils/utf8.h"
+
+static char *utf8_convert(const char *string, size_t len, const char *from,
+ const char *to);
+
+/**
+ * Convert a UTF-8 multibyte sequence into a single UCS4 character
+ *
+ * Encoding of UCS values outside the UTF-16 plane has been removed from
+ * RFC3629. This function conforms to RFC2279, however, as it is possible
+ * that the platform specific keyboard input handler will generate a UCS4
+ * value outside the UTF-16 plane.
+ *
+ * \param s The sequence to process
+ * \param l Length of sequence
+ * \return UCS4 character
+ */
+size_t utf8_to_ucs4(const char *s, size_t l)
+{
+ size_t c = 0;
+
+ if (!s)
+ assert(0);
+ else if (l > 0 && *s < 0x80)
+ c = *s;
+ else if (l > 1 && (*s & 0xE0) == 0xC0 && (*(s+1) & 0xC0) == 0x80)
+ c = ((*s & 0x1F) << 6) | (*(s+1) & 0x3F);
+ else if (l > 2 && (*s & 0xF0) == 0xE0 && (*(s+1) & 0xC0) == 0x80 &&
+ (*(s+2) & 0xC0) == 0x80)
+ c = ((*s & 0x0F) << 12) | ((*(s+1) & 0x3F) << 6) |
+ (*(s+2) & 0x3F);
+ else if (l > 3 && (*s & 0xF8) == 0xF0 && (*(s+1) & 0xC0) == 0x80 &&
+ (*(s+2) & 0xC0) == 0x80 && (*(s+3) & 0xC0) == 0x80)
+ c = ((*s & 0x0F) << 18) | ((*(s+1) & 0x3F) << 12) |
+ ((*(s+2) & 0x3F) << 6) | (*(s+3) & 0x3F);
+ else if (l > 4 && (*s & 0xFC) == 0xF8 && (*(s+1) & 0xC0) == 0x80 &&
+ (*(s+2) & 0xC0) == 0x80 && (*(s+3) & 0xC0) == 0x80 &&
+ (*(s+4) & 0xC0) == 0x80)
+ c = ((*s & 0x0F) << 24) | ((*(s+1) & 0x3F) << 18) |
+ ((*(s+2) & 0x3F) << 12) | ((*(s+3) & 0x3F) << 6) |
+ (*(s+4) & 0x3F);
+ else if (l > 5 && (*s & 0xFE) == 0xFC && (*(s+1) & 0xC0) == 0x80 &&
+ (*(s+2) & 0xC0) == 0x80 && (*(s+3) & 0xC0) == 0x80 &&
+ (*(s+4) & 0xC0) == 0x80 && (*(s+5) & 0xC0) == 0x80)
+ c = ((*s & 0x0F) << 28) | ((*(s+1) & 0x3F) << 24) |
+ ((*(s+2) & 0x3F) << 18) | ((*(s+3) & 0x3F) << 12) |
+ ((*(s+4) & 0x3F) << 6) | (*(s+5) & 0x3F);
+ else
+ assert(0);
+
+ return c;
+}
+
+/**
+ * Convert a single UCS4 character into a UTF-8 multibyte sequence
+ *
+ * Encoding of UCS values outside the UTF-16 plane has been removed from
+ * RFC3629. This function conforms to RFC2279, however, as it is possible
+ * that the platform specific keyboard input handler will generate a UCS4
+ * value outside the UTF-16 plane.
+ *
+ * \param c The character to process (0 <= c <= 0x7FFFFFFF)
+ * \param s Pointer to 6 byte long output buffer
+ * \return Length of multibyte sequence
+ */
+size_t utf8_from_ucs4(size_t c, char *s)
+{
+ size_t l = 0;
+
+ if (c > 0x7FFFFFFF)
+ assert(0);
+ else if (c < 0x80) {
+ *s = (char)c;
+ l = 1;
+ }
+ else if (c < 0x800) {
+ *s = 0xC0 | ((c >> 6) & 0x1F);
+ *(s+1) = 0x80 | (c & 0x3F);
+ l = 2;
+ }
+ else if (c < 0x10000) {
+ *s = 0xE0 | ((c >> 12) & 0xF);
+ *(s+1) = 0x80 | ((c >> 6) & 0x3F);
+ *(s+2) = 0x80 | (c & 0x3F);
+ l = 3;
+ }
+ else if (c < 0x200000) {
+ *s = 0xF0 | ((c >> 18) & 0x7);
+ *(s+1) = 0x80 | ((c >> 12) & 0x3F);
+ *(s+2) = 0x80 | ((c >> 6) & 0x3F);
+ *(s+3) = 0x80 | (c & 0x3F);
+ l = 4;
+ }
+ else if (c < 0x4000000) {
+ *s = 0xF8 | ((c >> 24) & 0x3);
+ *(s+1) = 0x80 | ((c >> 18) & 0x3F);
+ *(s+2) = 0x80 | ((c >> 12) & 0x3F);
+ *(s+3) = 0x80 | ((c >> 6) & 0x3F);
+ *(s+4) = 0x80 | (c & 0x3F);
+ l = 5;
+ }
+ else if (c <= 0x7FFFFFFF) {
+ *s = 0xFC | ((c >> 30) & 0x1);
+ *(s+1) = 0x80 | ((c >> 24) & 0x3F);
+ *(s+2) = 0x80 | ((c >> 18) & 0x3F);
+ *(s+3) = 0x80 | ((c >> 12) & 0x3F);
+ *(s+4) = 0x80 | ((c >> 6) & 0x3F);
+ *(s+5) = 0x80 | (c & 0x3F);
+ l = 6;
+ }
+
+ return l;
+}
+
+/**
+ * Calculate the length (in characters) of a NULL-terminated UTF-8 string
+ *
+ * \param s The string
+ * \return Length of string
+ */
+size_t utf8_length(const char *s)
+{
+ const char *__s = s;
+ int l = 0;
+ while (*__s != '\0') {
+ if ((*__s & 0x80) == 0x00)
+ __s += 1;
+ else if ((*__s & 0xE0) == 0xC0)
+ __s += 2;
+ else if ((*__s & 0xF0) == 0xE0)
+ __s += 3;
+ else if ((*__s & 0xF8) == 0xF0)
+ __s += 4;
+ else if ((*__s & 0xFC) == 0xF8)
+ __s += 5;
+ else if ((*__s & 0xFE) == 0xFC)
+ __s += 6;
+ else
+ assert(0);
+ l++;
+ }
+
+ return l;
+}
+
+/**
+ * Find previous legal UTF-8 char in string
+ *
+ * \param s The string
+ * \param o Offset in the string to start at
+ * \return Offset of first byte of previous legal character
+ */
+size_t utf8_prev(const char *s, size_t o)
+{
+ while (o != 0 && !(((s[--o] & 0x80) == 0x00) ||
+ ((s[o] & 0xC0) == 0xC0)))
+ /* do nothing */;
+
+ return o;
+}
+
+/**
+ * Find next legal UTF-8 char in string
+ *
+ * \param s The string
+ * \param l Maximum offset in string
+ * \param o Offset in the string to start at
+ * \return Offset of first byte of next legal character
+ */
+size_t utf8_next(const char *s, size_t l, size_t o)
+{
+ while (o != l && !(((s[++o] & 0x80) == 0x00) ||
+ ((s[o] & 0xC0) == 0xC0)))
+ /* do nothing */;
+
+ return o;
+}
+
+
+/**
+ * Convert a UTF8 string into the named encoding
+ *
+ * \param string The NULL-terminated string to convert
+ * \param encname The encoding name (suitable for passing to iconv)
+ * \param len Length of input string to consider (in bytes), or 0
+ * \return Pointer to converted string (on heap) or NULL on error
+ */
+char *utf8_to_enc(const char *string, const char *encname, size_t len)
+{
+ return utf8_convert(string, len, "UTF-8", encname);
+}
+
+/**
+ * Convert a UTF8 string into the named encoding
+ *
+ * \param string The NULL-terminated string to convert
+ * \param encname The encoding name (suitable for passing to iconv)
+ * \param len Length of input string to consider (in bytes), or 0
+ * \return Pointer to converted string (on heap) or NULL on error
+ */
+char *utf8_from_enc(const char *string, const char *encname, size_t len)
+{
+ return utf8_convert(string, len, encname, "UTF-8");
+}
+
+/**
+ * Convert a string from one encoding to another
+ *
+ * \param string The NULL-terminated string to convert
+ * \param len Length of input string to consider (in bytes)
+ * \param from The encoding name to convert from
+ * \param to The encoding name to convert to
+ * \return Pointer to converted string (on heap) or NULL on error
+ */
+char *utf8_convert(const char *string, size_t len, const char *from,
+ const char *to)
+{
+ iconv_t cd;
+ char *ret, *temp, *out, *in;
+ size_t slen, rlen;
+
+ if (!string || !from || !to)
+ return NULL;
+
+ in = (char *)string;
+
+ cd = iconv_open(to, from);
+ if (cd == (iconv_t)-1)
+ return NULL;
+
+ slen = len ? len : strlen(string);
+ /* Worst case = ACSII -> UCS4, so allocate an output buffer
+ * 4 times larger than the input buffer, and add 4 bytes at
+ * the end for the NULL terminator
+ */
+ rlen = slen * 4 + 4;
+
+ temp = out = calloc(rlen, sizeof(char));
+ if (!out) {
+ iconv_close(cd);
+ return NULL;
+ }
+
+ /* perform conversion */
+ if (iconv(cd, &in, &slen, &out, &rlen) == (size_t)-1) {
+ free(temp);
+ iconv_close(cd);
+ return NULL;
+ }
+
+ iconv_close(cd);
+
+ if (rlen > 64 /* allow 64bytes wasted space */) {
+ /* and allocate a more sensibly sized output buffer */
+ ret = calloc(out - temp + 4, sizeof(char));
+ memcpy(ret, temp, out - temp);
+ free(temp);
+ }
+ else
+ ret = temp;
+
+ return ret;
+}
+