[project @ 2005-04-16 05:09:32 by jmb]

Split out UTF-8 handling functions.
Submit URL-encoded forms in sensible encoding:
  * First entry in accept-charset list, if present
  * Document encoding, otherwise
  We may want to explicitly look for UTF-8, to save converting.
Convert cnv_str_local_enc/cnv_local_enc_str to use iconv (they're now veneers for utf8_[to/from]_enc).
Provide mechanism for looking up local system charset (derived from system alphabet, under RISC OS)

svn path=/import/netsurf/; revision=1647

4 files changed, 316 insertions, 102 deletions

diff --git a/utils/utf8.c b/utils/utf8.c
new file mode 100644
index 000000000..957062a37
--- /dev/null
+++ b/utils/utf8.c
@@ -0,0 +1,280 @@
+/*
+ * This file is part of NetSurf, http://netsurf.sourceforge.net/
+ * Licensed under the GNU General Public License,
+ *                http://www.opensource.org/licenses/gpl-license
+ * Copyright 2005 John M Bell <jmb202@ecs.soton.ac.uk>
+ */
+
+/**
+ * \file UTF-8 manipulation functions (implementation)
+ */
+
+#include <assert.h>
+#include <string.h>
+
+#include <iconv.h>
+
+#include "netsurf/utils/utf8.h"
+
+static char *utf8_convert(const char *string, size_t len, const char *from,
+		const char *to);
+
+/**
+ * Convert a UTF-8 multibyte sequence into a single UCS4 character
+ *
+ * Encoding of UCS values outside the UTF-16 plane has been removed from
+ * RFC3629. This function conforms to RFC2279, however, as it is possible
+ * that the platform specific keyboard input handler will generate a UCS4
+ * value outside the UTF-16 plane.
+ *
+ * \param s  The sequence to process
+ * \param l  Length of sequence
+ * \return   UCS4 character
+ */
+size_t utf8_to_ucs4(const char *s, size_t l)
+{
+	size_t c = 0;
+
+	if (!s)
+		assert(0);
+	else if (l > 0 && *s < 0x80)
+		c = *s;
+	else if (l > 1 && (*s & 0xE0) == 0xC0 && (*(s+1) & 0xC0) == 0x80)
+		c = ((*s & 0x1F) << 6) | (*(s+1) & 0x3F);
+	else if (l > 2 && (*s & 0xF0) == 0xE0 && (*(s+1) & 0xC0) == 0x80 &&
+			(*(s+2) & 0xC0) == 0x80)
+		c = ((*s & 0x0F) << 12) | ((*(s+1) & 0x3F) << 6) |
+							(*(s+2) & 0x3F);
+	else if (l > 3 && (*s & 0xF8) == 0xF0 && (*(s+1) & 0xC0) == 0x80 &&
+			(*(s+2) & 0xC0) == 0x80 && (*(s+3) & 0xC0) == 0x80)
+		c = ((*s & 0x0F) << 18) | ((*(s+1) & 0x3F) << 12) |
+				((*(s+2) & 0x3F) << 6) | (*(s+3) & 0x3F);
+	else if (l > 4 && (*s & 0xFC) == 0xF8 && (*(s+1) & 0xC0) == 0x80 &&
+			(*(s+2) & 0xC0) == 0x80 && (*(s+3) & 0xC0) == 0x80 &&
+			(*(s+4) & 0xC0) == 0x80)
+		c = ((*s & 0x0F) << 24) | ((*(s+1) & 0x3F) << 18) |
+			((*(s+2) & 0x3F) << 12) | ((*(s+3) & 0x3F) << 6) |
+			(*(s+4) & 0x3F);
+	else if (l > 5 && (*s & 0xFE) == 0xFC && (*(s+1) & 0xC0) == 0x80 &&
+			(*(s+2) & 0xC0) == 0x80 && (*(s+3) & 0xC0) == 0x80 &&
+			(*(s+4) & 0xC0) == 0x80 && (*(s+5) & 0xC0) == 0x80)
+		c = ((*s & 0x0F) << 28) | ((*(s+1) & 0x3F) << 24) |
+			((*(s+2) & 0x3F) << 18) | ((*(s+3) & 0x3F) << 12) |
+			((*(s+4) & 0x3F) << 6) | (*(s+5) & 0x3F);
+	else
+		assert(0);
+
+	return c;
+}
+
+/**
+ * Convert a single UCS4 character into a UTF-8 multibyte sequence
+ *
+ * Encoding of UCS values outside the UTF-16 plane has been removed from
+ * RFC3629. This function conforms to RFC2279, however, as it is possible
+ * that the platform specific keyboard input handler will generate a UCS4
+ * value outside the UTF-16 plane.
+ *
+ * \param c  The character to process (0 <= c <= 0x7FFFFFFF)
+ * \param s  Pointer to 6 byte long output buffer
+ * \return   Length of multibyte sequence
+ */
+size_t utf8_from_ucs4(size_t c, char *s)
+{
+	size_t l = 0;
+
+	if (c > 0x7FFFFFFF)
+		assert(0);
+	else if (c < 0x80) {
+		*s = (char)c;
+		l = 1;
+	}
+	else if (c < 0x800) {
+		*s = 0xC0 | ((c >> 6) & 0x1F);
+		*(s+1) = 0x80 | (c & 0x3F);
+		l = 2;
+	}
+	else if (c < 0x10000) {
+		*s = 0xE0 | ((c >> 12) & 0xF);
+		*(s+1) = 0x80 | ((c >> 6) & 0x3F);
+		*(s+2) = 0x80 | (c & 0x3F);
+		l = 3;
+	}
+	else if (c < 0x200000) {
+		*s = 0xF0 | ((c >> 18) & 0x7);
+		*(s+1) = 0x80 | ((c >> 12) & 0x3F);
+		*(s+2) = 0x80 | ((c >> 6) & 0x3F);
+		*(s+3) = 0x80 | (c & 0x3F);
+		l = 4;
+	}
+	else if (c < 0x4000000) {
+		*s = 0xF8 | ((c >> 24) & 0x3);
+		*(s+1) = 0x80 | ((c >> 18) & 0x3F);
+		*(s+2) = 0x80 | ((c >> 12) & 0x3F);
+		*(s+3) = 0x80 | ((c >> 6) & 0x3F);
+		*(s+4) = 0x80 | (c & 0x3F);
+		l = 5;
+	}
+	else if (c <= 0x7FFFFFFF) {
+		*s = 0xFC | ((c >> 30) & 0x1);
+		*(s+1) = 0x80 | ((c >> 24) & 0x3F);
+		*(s+2) = 0x80 | ((c >> 18) & 0x3F);
+		*(s+3) = 0x80 | ((c >> 12) & 0x3F);
+		*(s+4) = 0x80 | ((c >> 6) & 0x3F);
+		*(s+5) = 0x80 | (c & 0x3F);
+		l = 6;
+	}
+
+	return l;
+}
+
+/**
+ * Calculate the length (in characters) of a NULL-terminated UTF-8 string
+ *
+ * \param s  The string
+ * \return   Length of string
+ */
+size_t utf8_length(const char *s)
+{
+	const char *__s = s;
+	int l = 0;
+	while (*__s != '\0') {
+		if ((*__s & 0x80) == 0x00)
+			__s += 1;
+		else if ((*__s & 0xE0) == 0xC0)
+			__s += 2;
+		else if ((*__s & 0xF0) == 0xE0)
+			__s += 3;
+		else if ((*__s & 0xF8) == 0xF0)
+			__s += 4;
+		else if ((*__s & 0xFC) == 0xF8)
+			__s += 5;
+		else if ((*__s & 0xFE) == 0xFC)
+			__s += 6;
+		else
+			assert(0);
+		l++;
+	}
+
+	return l;
+}
+
+/**
+ * Find previous legal UTF-8 char in string
+ *
+ * \param s  The string
+ * \param o  Offset in the string to start at
+ * \return Offset of first byte of previous legal character
+ */
+size_t utf8_prev(const char *s, size_t o)
+{
+	while (o != 0 && !(((s[--o] & 0x80) == 0x00) ||
+			((s[o] & 0xC0) == 0xC0)))
+		/* do nothing */;
+
+	return o;
+}
+
+/**
+ * Find next legal UTF-8 char in string
+ *
+ * \param s  The string
+ * \param l  Maximum offset in string
+ * \param o  Offset in the string to start at
+ * \return Offset of first byte of next legal character
+ */
+size_t utf8_next(const char *s, size_t l, size_t o)
+{
+	while (o != l && !(((s[++o] & 0x80) == 0x00) ||
+			((s[o] & 0xC0) == 0xC0)))
+		/* do nothing */;
+
+	return o;
+}
+
+
+/**
+ * Convert a UTF8 string into the named encoding
+ *
+ * \param string  The NULL-terminated string to convert
+ * \param encname The encoding name (suitable for passing to iconv)
+ * \param len     Length of input string to consider (in bytes), or 0
+ * \return Pointer to converted string (on heap) or NULL on error
+ */
+char *utf8_to_enc(const char *string, const char *encname, size_t len)
+{
+	return utf8_convert(string, len, "UTF-8", encname);
+}
+
+/**
+ * Convert a UTF8 string into the named encoding
+ *
+ * \param string  The NULL-terminated string to convert
+ * \param encname The encoding name (suitable for passing to iconv)
+ * \param len     Length of input string to consider (in bytes), or 0
+ * \return Pointer to converted string (on heap) or NULL on error
+ */
+char *utf8_from_enc(const char *string, const char *encname, size_t len)
+{
+	return utf8_convert(string, len, encname, "UTF-8");
+}
+
+/**
+ * Convert a string from one encoding to another
+ *
+ * \param string  The NULL-terminated string to convert
+ * \param len     Length of input string to consider (in bytes)
+ * \param from    The encoding name to convert from
+ * \param to      The encoding name to convert to
+ * \return Pointer to converted string (on heap) or NULL on error
+ */
+char *utf8_convert(const char *string, size_t len, const char *from,
+		const char *to)
+{
+	iconv_t cd;
+	char *ret, *temp, *out, *in;
+	size_t slen, rlen;
+
+	if (!string || !from || !to)
+		return NULL;
+
+	in = (char *)string;
+
+	cd = iconv_open(to, from);
+	if (cd == (iconv_t)-1)
+		return NULL;
+
+	slen = len ? len : strlen(string);
+	/* Worst case = ACSII -> UCS4, so allocate an output buffer
+	 * 4 times larger than the input buffer, and add 4 bytes at
+	 * the end for the NULL terminator
+	 */
+	rlen = slen * 4 + 4;
+
+	temp = out = calloc(rlen, sizeof(char));
+	if (!out) {
+		iconv_close(cd);
+		return NULL;
+	}
+
+	/* perform conversion */
+	if (iconv(cd, &in, &slen, &out, &rlen) == (size_t)-1) {
+		free(temp);
+		iconv_close(cd);
+		return NULL;
+	}
+
+	iconv_close(cd);
+
+	if (rlen > 64 /* allow 64bytes wasted space */) {
+		/* and allocate a more sensibly sized output buffer */
+		ret = calloc(out - temp + 4, sizeof(char));
+		memcpy(ret, temp, out - temp);
+		free(temp);
+	}
+	else
+		ret = temp;
+
+	return ret;
+}
+
diff --git a/utils/utf8.h b/utils/utf8.h
new file mode 100644
index 000000000..02ff0322d
--- /dev/null
+++ b/utils/utf8.h
@@ -0,0 +1,26 @@
+/*
+ * This file is part of NetSurf, http://netsurf.sourceforge.net/
+ * Licensed under the GNU General Public License,
+ *                http://www.opensource.org/licenses/gpl-license
+ * Copyright 2005 John M Bell <jmb202@ecs.soton.ac.uk>
+ */
+
+/**
+ * \file UTF-8 manipulation functions (interface)
+ */
+
+#ifndef _NETSURF_UTILS_UTF8_H_
+#define _NETSURF_UTILS_UTF8_H_
+
+size_t utf8_to_ucs4(const char *s, size_t l);
+size_t utf8_from_ucs4(size_t c, char *s);
+
+size_t utf8_length(const char *s);
+
+size_t utf8_prev(const char *s, size_t o);
+size_t utf8_next(const char *s, size_t l, size_t o);
+
+char *utf8_to_enc(const char *string, const char *encname, size_t len);
+char *utf8_from_enc(const char *string, const char *encname, size_t len);
+
+#endif
diff --git a/utils/utils.c b/utils/utils.c
index 35e3b1c7e..04f24efa5 100644
--- a/utils/utils.c
+++ b/utils/utils.c
@@ -22,6 +22,7 @@
 #define NDEBUG
 #include "netsurf/utils/log.h"
 #include "netsurf/utils/messages.h"
+#include "netsurf/utils/utf8.h"
 #include "netsurf/utils/utils.h"
 
 
@@ -110,56 +111,10 @@ char *cnv_space2nbsp(const char *s)
  * \param s string in local machine encoding. NUL or length terminated (which comes first).
  * \param length maximum number of bytes to consider at s.
  * \return malloc()'ed NUL termined string in UTF-8 encoding.
- *
- * Based on RISCOS-LATIN1 code from libiconv.
- * \todo: we should use libiconv to support more local encodings instead
- * of only RISCOS-LATIN1.
  */
 char *cnv_local_enc_str(const char *s, size_t length)
 {
-	size_t l_out, l_in;
-	const char *s_in;
-	char *d, *d_out;
-	static const unsigned int riscos1_2uni[32] = {
-		/* 0x80 */
-		0x221a, 0x0174, 0x0175, 0x0083, 0x2573, 0x0176, 0x0177, 0x0087,
-		0x21e6, 0x21e8, 0x21e9, 0x21e7, 0x2026, 0x2122, 0x2030, 0x2022,
-		/* 0x90 */
-		0x2018, 0x2019, 0x2039, 0x203a, 0x201c, 0x201d, 0x201e, 0x2013,
-		0x2014, 0x2212, 0x0152, 0x0153, 0x2020, 0x2021, 0xfb01, 0xfb02,
-	};
-
-	/* We're counting on the fact that all riscos1_2uni[] values are
-	 * between 0x80 (incl) and 0x1000 (excl).
-	 */
-	for (s_in = s, l_in = length, l_out = 1;
-			*s_in != '\0' && l_in != 0;
-			++s_in, --l_in)
-		l_out += (*s_in >= 0x80 && *s_in < 0xA0) ? ((riscos1_2uni[*s_in - 0x80] < 0x800) ? 2 : 3) : 1;
-	if ((d_out = (char *)malloc(l_out)) == NULL)
-		return NULL;
-	for (s_in = s, l_in = length, d = d_out;
-			*s_in != '\0' && l_in != 0;
-			++s_in, --l_in) {
-		unsigned int uc = (*s_in >= 0x80 && *s_in < 0xA0) ? riscos1_2uni[*s_in - 0x80] : *s_in;
-		const int cnt = (uc < 0x80) ? 1 : (uc < 0x800) ? 2 : 3;
-		switch (cnt) {
-			case 3:
-				d[2] = 0x80 | (uc & 0x3F);
-				uc = (uc >> 6) | 0x800;
-				/* fall through */
-			case 2:
-				d[1] = 0x80 | (uc & 0x3F);
-				uc = (uc >> 6) | 0xC0;
-				/* fall through */
-			case 1:
-				d[0] = uc;
-		}
-		d += cnt;
-	}
-	*d = '\0';
-
-	return d_out;
+	return utf8_from_enc(s, local_encoding_name(), length);
 }
 
 
@@ -169,7 +124,7 @@ char *cnv_local_enc_str(const char *s, size_t length)
  */
 char *cnv_str_local_enc(const char *s)
 {
-return cnv_strn_local_enc(s, strlen(s), NULL);
+	return cnv_strn_local_enc(s, 0);
 }
 
 
@@ -177,61 +132,12 @@ return cnv_strn_local_enc(s, strlen(s), NULL);
  * Converts UTF-8 string <s> of <length> bytes to the machine local encoding.
  * Caller needs to free return value.
  *
- * When back_map is non-NULL, a ptr to a ptrdiff_t array is filled in which
- * needs to be free'd by the caller.  The array contains per character
- * in the return string, a ptrdiff in the <s> UTF-8 encoded string.
- *
  * \todo: we should use libiconv to support more local encodings instead
  * of only ISOLATIN1.
  */
-char *cnv_strn_local_enc(const char *s, int length, const ptrdiff_t **back_mapPP)
+char *cnv_strn_local_enc(const char *s, int length)
 {
-	/* Buffer at d & back_mapP can be overdimentioned but is certainly
-	 * big enough to carry the end result.
-	 */
-	char *d, *d0;
-	const char * const s0 = s;
-	ptrdiff_t *back_mapP = NULL;
-
-	if (back_mapPP != NULL) {
-		back_mapP = calloc(length + 1, sizeof(ptrdiff_t));
-		if (!back_mapP)
-			return NULL;
-		*back_mapPP = back_mapP;
-	}
-
-	d = calloc(length + 1, sizeof(char));
-	if (!d)
-		return NULL;
-
-	d0 = d;
-
-	while (length != 0) {
-		int u, chars;
-
-		chars = length;
-		u = xmlGetUTF8Char(s, &chars);
-		if (chars <= 0) {
-			s += 1;
-			length -= 1;
-			continue;
-		}
-		if (back_mapP != NULL)
-			*back_mapP++ = s - s0;
-		s += chars;
-		length -= chars;
-		if (u == 0x09 || u == 0x0a || u == 0x0d ||
-				(0x20 <= u && u <= 0x7f) ||
-				(0xa0 <= u && u <= 0xff))
-			*d++ = u;
-		else
-			*d++ = '?';
-	}
-	if (back_mapP != NULL)
-		*back_mapP = s - s0;
-	*d = 0;
-
-	return d0;
+	return utf8_to_enc(s, local_encoding_name(), length);
 }
 
 
diff --git a/utils/utils.h b/utils/utils.h
index 6dcfa1305..278446e0c 100644
--- a/utils/utils.h
+++ b/utils/utils.h
@@ -20,20 +20,22 @@
 #define NOF_ELEMENTS(array) (sizeof(array)/sizeof(*(array)))
 #endif
 
-void die(const char * const error);
 char * strip(char * const s);
 int whitespace(const char * str);
 char * squash_whitespace(const char * s);
 char *cnv_space2nbsp(const char *s);
 char *cnv_local_enc_str(const char *s, size_t length);
 char *cnv_str_local_enc(const char *s);
-char *cnv_strn_local_enc(const char *s, int length,
-		const ptrdiff_t **back_mapPP);
+char *cnv_strn_local_enc(const char *s, int length);
 bool is_dir(const char *path);
 void regcomp_wrapper(regex_t *preg, const char *regex, int cflags);
 void clean_cookiejar(void);
 void unicode_transliterate(unsigned int c, char **r);
 char *human_friendly_bytesize(unsigned long bytesize);
+
+/* Platform specific functions */
+void die(const char * const error);
 void warn_user(const char *warning, const char *detail);
+const char *local_encoding_name(void);
 
 #endif