diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/charset/Makefile | 2 | ||||
| -rw-r--r-- | src/charset/codec.c | 2 | ||||
| -rw-r--r-- | src/charset/codec_utf16.c | 620 | ||||
| -rw-r--r-- | src/input/Makefile | 2 | ||||
| -rw-r--r-- | src/input/inputstream.c | 2 | ||||
| -rw-r--r-- | src/input/utf16_stream.c | 608 | ||||
| -rw-r--r-- | src/utils/Makefile | 2 | ||||
| -rw-r--r-- | src/utils/utf16.c | 239 | ||||
| -rw-r--r-- | src/utils/utf16.h | 38 |
9 files changed, 1512 insertions, 3 deletions
diff --git a/src/charset/Makefile b/src/charset/Makefile index 62817b3..c0a00b3 100644 --- a/src/charset/Makefile +++ b/src/charset/Makefile @@ -22,7 +22,7 @@ CFLAGS += -I$(CURDIR) # Objects -OBJS = aliases codec codec_iconv codec_utf8 detect +OBJS = aliases codec codec_iconv codec_utf8 codec_utf16 detect .PHONY: clean debug distclean export release setup test diff --git a/src/charset/codec.c b/src/charset/codec.c index 12a1bdc..727d600 100644 --- a/src/charset/codec.c +++ b/src/charset/codec.c @@ -13,9 +13,11 @@ extern hubbub_charsethandler hubbub_iconv_codec_handler; extern hubbub_charsethandler hubbub_utf8_codec_handler; +extern hubbub_charsethandler hubbub_utf16_codec_handler; static hubbub_charsethandler *handler_table[] = { &hubbub_utf8_codec_handler, + &hubbub_utf16_codec_handler, &hubbub_iconv_codec_handler, NULL, }; diff --git a/src/charset/codec_utf16.c b/src/charset/codec_utf16.c new file mode 100644 index 0000000..9a94d29 --- /dev/null +++ b/src/charset/codec_utf16.c @@ -0,0 +1,620 @@ +/* + * This file is part of Hubbub. + * Licensed under the MIT License, + * http://www.opensource.org/licenses/mit-license.php + * Copyright 2007 John-Mark Bell <jmb@netsurf-browser.org> + */ + +#include <stdlib.h> +#include <string.h> + +/* These two are for htonl / ntohl */ +#include <arpa/inet.h> +#include <netinet/in.h> + +#include "charset/aliases.h" +#include "utils/utf16.h" +#include "utils/utils.h" + +#include "codec_impl.h" + +/** + * UTF-16 charset codec + */ +typedef struct hubbub_utf16_codec { + hubbub_charsetcodec base; /**< Base class */ + +#define INVAL_BUFSIZE (32) + uint8_t inval_buf[INVAL_BUFSIZE]; /**< Buffer for fixing up + * incomplete input + * sequences */ + size_t inval_len; /*< Byte length of inval_buf **/ + +#define READ_BUFSIZE (8) + uint32_t read_buf[READ_BUFSIZE]; /**< Buffer for partial + * output sequences (decode) + * (host-endian) */ + size_t read_len; /**< Character length of read_buf */ + +#define WRITE_BUFSIZE (8) + uint32_t write_buf[WRITE_BUFSIZE]; /**< Buffer for partial + * output sequences (encode) + * (host-endian) */ + size_t write_len; /**< Character length of write_buf */ + +} hubbub_utf16_codec; + +static bool hubbub_utf16_codec_handles_charset(const char *charset); +static hubbub_charsetcodec *hubbub_utf16_codec_create(const char *charset, + hubbub_alloc alloc, void *pw); +static void hubbub_utf16_codec_destroy (hubbub_charsetcodec *codec); +static hubbub_error hubbub_utf16_codec_encode(hubbub_charsetcodec *codec, + const uint8_t **source, size_t *sourcelen, + uint8_t **dest, size_t *destlen); +static hubbub_error hubbub_utf16_codec_decode(hubbub_charsetcodec *codec, + const uint8_t **source, size_t *sourcelen, + uint8_t **dest, size_t *destlen); +static hubbub_error hubbub_utf16_codec_reset(hubbub_charsetcodec *codec); +static hubbub_error hubbub_utf16_codec_read_char(hubbub_utf16_codec *c, + const uint8_t **source, size_t *sourcelen, + uint8_t **dest, size_t *destlen); +static hubbub_error hubbub_utf16_codec_filter_decoded_char( + hubbub_utf16_codec *c, + uint32_t ucs4, uint8_t **dest, size_t *destlen); + +/** + * Determine whether this codec handles a specific charset + * + * \param charset Charset to test + * \return true if handleable, false otherwise + */ +bool hubbub_utf16_codec_handles_charset(const char *charset) +{ + return hubbub_mibenum_from_name(charset, strlen(charset)) == + hubbub_mibenum_from_name("UTF-16", SLEN("UTF-16")); +} + +/** + * Create a utf16 codec + * + * \param charset The charset to read from / write to + * \param alloc Memory (de)allocation function + * \param pw Pointer to client-specific private data (may be NULL) + * \return Pointer to codec, or NULL on failure + */ +hubbub_charsetcodec *hubbub_utf16_codec_create(const char *charset, + hubbub_alloc alloc, void *pw) +{ + hubbub_utf16_codec *codec; + + UNUSED(charset); + + codec = alloc(NULL, sizeof(hubbub_utf16_codec), pw); + if (codec == NULL) + return NULL; + + codec->inval_buf[0] = '\0'; + codec->inval_len = 0; + + codec->read_buf[0] = 0; + codec->read_len = 0; + + codec->write_buf[0] = 0; + codec->write_len = 0; + + /* Finally, populate vtable */ + codec->base.handler.destroy = hubbub_utf16_codec_destroy; + codec->base.handler.encode = hubbub_utf16_codec_encode; + codec->base.handler.decode = hubbub_utf16_codec_decode; + codec->base.handler.reset = hubbub_utf16_codec_reset; + + return (hubbub_charsetcodec *) codec; +} + +/** + * Destroy a utf16 codec + * + * \param codec The codec to destroy + */ +void hubbub_utf16_codec_destroy (hubbub_charsetcodec *codec) +{ + UNUSED(codec); +} + +/** + * Encode a chunk of UCS4 data into utf16 + * + * \param codec The codec to use + * \param source Pointer to pointer to source data + * \param sourcelen Pointer to length (in bytes) of source data + * \param dest Pointer to pointer to output buffer + * \param destlen Pointer to length (in bytes) of output buffer + * \return HUBBUB_OK on success, + * HUBBUB_NOMEM if output buffer is too small, + * HUBBUB_INVALID if a character cannot be represented and the + * codec's error handling mode is set to STRICT, + * <any_other_error> as a result of the failure of the + * client-provided filter function. + * + * On exit, ::source will point immediately _after_ the last input character + * read. Any remaining output for the character will be buffered by the + * codec for writing on the next call. This buffered data is post-filtering, + * so will not be refiltered on the next call. + * + * In the case of the filter function failing, ::source will point _at_ the + * last input character read; nothing will be written or buffered for the + * failed character. It is up to the client to fix the cause of the failure + * and retry the encoding process. + * + * Note that, if failure occurs whilst attempting to write any output + * buffered by the last call, then ::source and ::sourcelen will remain + * unchanged (as nothing more has been read). + * + * There is no way to determine the output character which caused a + * failure (as it may be one in a filter-injected replacement sequence). + * It is, however, possible to determine which source character caused it + * (this being the character immediately before the location pointed to by + * ::source on exit). + * + * [I.e. the process of filtering results in a potential one-to-many mapping + * between source characters and output characters, and identification of + * individual output characters is impossible.] + * + * ::sourcelen will be reduced appropriately on exit. + * + * ::dest will point immediately _after_ the last character written. + * + * ::destlen will be reduced appropriately on exit. + */ +hubbub_error hubbub_utf16_codec_encode(hubbub_charsetcodec *codec, + const uint8_t **source, size_t *sourcelen, + uint8_t **dest, size_t *destlen) +{ + hubbub_utf16_codec *c = (hubbub_utf16_codec *) codec; + uint32_t ucs4; + uint32_t *towrite; + size_t towritelen; + hubbub_error error; + + /* Process any outstanding characters from the previous call */ + if (c->write_len > 0) { + uint32_t *pwrite = c->write_buf; + uint8_t buf[4]; + size_t len; + + while (c->write_len > 0) { + error = hubbub_utf16_from_ucs4(pwrite[0], buf, &len); + if (error != HUBBUB_OK) + abort(); + + if (*destlen < len) { + /* Insufficient output buffer space */ + for (len = 0; len < c->write_len; len++) + c->write_buf[len] = pwrite[len]; + + return HUBBUB_NOMEM; + } + + memcpy(*dest, buf, len); + + *dest += len; + *destlen -= len; + + pwrite++; + c->write_len--; + } + } + + /* Now process the characters for this call */ + while (*sourcelen > 0) { + ucs4 = ntohl(*((uint32_t *) (void *) *source)); + towrite = &ucs4; + towritelen = 1; + + /* Run character we're about to output through the + * registered filter, so it can replace it. */ + if (c->base.filter != NULL) { + error = c->base.filter(ucs4, + &towrite, &towritelen, + c->base.filter_pw); + if (error != HUBBUB_OK) + return error; + } + + /* Output current characters */ + while (towritelen > 0) { + uint8_t buf[4]; + size_t len; + + error = hubbub_utf16_from_ucs4(towrite[0], buf, &len); + if (error != HUBBUB_OK) + abort(); + + if (*destlen < len) { + /* Insufficient output space */ + if (towritelen >= WRITE_BUFSIZE) + abort(); + + c->write_len = towritelen; + + /* Copy pending chars to save area, for + * processing next call. */ + for (len = 0; len < towritelen; len++) + c->write_buf[len] = towrite[len]; + + /* Claim character we've just buffered, + * so it's not reprocessed */ + *source += 4; + *sourcelen -= 4; + + return HUBBUB_NOMEM; + } + + memcpy(*dest, buf, len); + + *dest += len; + *destlen -= len; + + towrite++; + towritelen--; + } + + *source += 4; + *sourcelen -= 4; + } + + return HUBBUB_OK; +} + +/** + * Decode a chunk of utf16 data into UCS4 + * + * \param codec The codec to use + * \param source Pointer to pointer to source data + * \param sourcelen Pointer to length (in bytes) of source data + * \param dest Pointer to pointer to output buffer + * \param destlen Pointer to length (in bytes) of output buffer + * \return HUBBUB_OK on success, + * HUBBUB_NOMEM if output buffer is too small, + * HUBBUB_INVALID if a character cannot be represented and the + * codec's error handling mode is set to STRICT, + * <any_other_error> as a result of the failure of the + * client-provided filter function. + * + * On exit, ::source will point immediately _after_ the last input character + * read, if the result is _OK or _NOMEM. Any remaining output for the + * character will be buffered by the codec for writing on the next call. + * This buffered data is post-filtering, so will not be refiltered on the + * next call. + * + * In the case of the result being _INVALID or the filter function failing, + * ::source will point _at_ the last input character read; nothing will be + * written or buffered for the failed character. It is up to the client to + * fix the cause of the failure and retry the decoding process. + * + * Note that, if failure occurs whilst attempting to write any output + * buffered by the last call, then ::source and ::sourcelen will remain + * unchanged (as nothing more has been read). + * + * There is no way to determine the output character which caused a + * failure (as it may be one in a filter-injected replacement sequence). + * It is, however, possible to determine which source character caused it + * (this being the character immediately at or before the location pointed + * to by ::source on exit). + * + * [I.e. the process of filtering results in a potential one-to-many mapping + * between source characters and output characters, and identification of + * individual output characters is impossible.] + * + * If STRICT error handling is configured and an illegal sequence is split + * over two calls, then _INVALID will be returned from the second call, + * but ::source will point mid-way through the invalid sequence (i.e. it + * will be unmodified over the second call). In addition, the internal + * incomplete-sequence buffer will be emptied, such that subsequent calls + * will progress, rather than re-evaluating the same invalid sequence. + * + * ::sourcelen will be reduced appropriately on exit. + * + * ::dest will point immediately _after_ the last character written. + * + * ::destlen will be reduced appropriately on exit. + * + * Call this with a source length of 0 to flush the output buffer. + */ +hubbub_error hubbub_utf16_codec_decode(hubbub_charsetcodec *codec, + const uint8_t **source, size_t *sourcelen, + uint8_t **dest, size_t *destlen) +{ + hubbub_utf16_codec *c = (hubbub_utf16_codec *) codec; + hubbub_error error; + + if (c->read_len > 0) { + /* Output left over from last decode */ + uint32_t *pread = c->read_buf; + + while (c->read_len > 0 && *destlen >= c->read_len * 4) { + *((uint32_t *) (void *) *dest) = htonl(pread[0]); + + *dest += 4; + *destlen -= 4; + + pread++; + c->read_len--; + } + + if (*destlen < c->read_len * 4) { + /* Ran out of output buffer */ + size_t i; + + /* Shuffle remaining output down */ + for (i = 0; i < c->read_len; i++) + c->read_buf[i] = pread[i]; + + return HUBBUB_NOMEM; + } + } + + if (c->inval_len > 0) { + /* The last decode ended in an incomplete sequence. + * Fill up inval_buf with data from the start of the + * new chunk and process it. */ + uint8_t *in = c->inval_buf; + size_t ol = c->inval_len; + size_t l = min(INVAL_BUFSIZE - ol - 1, *sourcelen); + size_t orig_l = l; + + memcpy(c->inval_buf + ol, *source, l); + + l += c->inval_len; + + error = hubbub_utf16_codec_read_char(c, + (const uint8_t **) &in, &l, dest, destlen); + if (error != HUBBUB_OK && error != HUBBUB_NOMEM) { + return error; + } + + /* And now, fix up source pointers */ + *source += max((signed) (orig_l - l), 0); + *sourcelen -= max((signed) (orig_l - l), 0); + + /* Failed to resolve an incomplete character and + * ran out of buffer space. No recovery strategy + * possible, so explode everywhere. */ + if ((orig_l + ol) - l == 0) + abort(); + + /* Report memory exhaustion case from above */ + if (error != HUBBUB_OK) + return error; + } + + /* Finally, the "normal" case; process all outstanding characters */ + while (*sourcelen > 0) { + error = hubbub_utf16_codec_read_char(c, + source, sourcelen, dest, destlen); + if (error != HUBBUB_OK) { + return error; + } + } + + return HUBBUB_OK; +} + +/** + * Clear a utf16 codec's encoding state + * + * \param codec The codec to reset + * \return HUBBUB_OK on success, appropriate error otherwise + */ +hubbub_error hubbub_utf16_codec_reset(hubbub_charsetcodec *codec) +{ + hubbub_utf16_codec *c = (hubbub_utf16_codec *) codec; + + c->inval_buf[0] = '\0'; + c->inval_len = 0; + + c->read_buf[0] = 0; + c->read_len = 0; + + c->write_buf[0] = 0; + c->write_len = 0; + + return HUBBUB_OK; +} + + +/** + * Read a character from the UTF-16 to UCS4 (big endian) + * + * \param c The codec + * \param source Pointer to pointer to source buffer (updated on exit) + * \param sourcelen Pointer to length of source buffer (updated on exit) + * \param dest Pointer to pointer to output buffer (updated on exit) + * \param destlen Pointer to length of output buffer (updated on exit) + * \return HUBBUB_OK on success, + * HUBBUB_NOMEM if output buffer is too small, + * HUBBUB_INVALID if a character cannot be represented and the + * codec's error handling mode is set to STRICT, + * <any_other_error> as a result of the failure of the + * client-provided filter function. + * + * On exit, ::source will point immediately _after_ the last input character + * read, if the result is _OK or _NOMEM. Any remaining output for the + * character will be buffered by the codec for writing on the next call. + * This buffered data is post-filtering, so will not be refiltered on the + * next call. + * + * In the case of the result being _INVALID or the filter function failing, + * ::source will point _at_ the last input character read; nothing will be + * written or buffered for the failed character. It is up to the client to + * fix the cause of the failure and retry the decoding process. + * + * ::sourcelen will be reduced appropriately on exit. + * + * ::dest will point immediately _after_ the last character written. + * + * ::destlen will be reduced appropriately on exit. + */ +hubbub_error hubbub_utf16_codec_read_char(hubbub_utf16_codec *c, + const uint8_t **source, size_t *sourcelen, + uint8_t **dest, size_t *destlen) +{ + uint32_t ucs4; + size_t sucs4; + hubbub_error error; + + /* Convert a single character */ + error = hubbub_utf16_to_ucs4(*source, *sourcelen, &ucs4, &sucs4); + if (error == HUBBUB_OK) { + /* Read a character */ + error = hubbub_utf16_codec_filter_decoded_char(c, + ucs4, dest, destlen); + if (error == HUBBUB_OK || error == HUBBUB_NOMEM) { + /* filter function succeeded; update source pointers */ + *source += sucs4; + *sourcelen -= sucs4; + } + + /* Clear inval buffer */ + c->inval_buf[0] = '\0'; + c->inval_len = 0; + + return error; + } else if (error == HUBBUB_NEEDDATA) { + /* Incomplete input sequence */ + if (*sourcelen > INVAL_BUFSIZE) + abort(); + + memmove(c->inval_buf, (char *) *source, *sourcelen); + c->inval_buf[*sourcelen] = '\0'; + c->inval_len = *sourcelen; + + *source += *sourcelen; + *sourcelen = 0; + + return HUBBUB_OK; + } else if (error == HUBBUB_INVALID) { + /* Illegal input sequence */ + uint32_t nextchar; + + /* Clear inval buffer */ + c->inval_buf[0] = '\0'; + c->inval_len = 0; + + /* Strict errormode; simply flag invalid character */ + if (c->base.errormode == HUBBUB_CHARSETCODEC_ERROR_STRICT) { + return HUBBUB_INVALID; + } + + /* Find next valid UTF-16 sequence. + * We're processing client-provided data, so let's + * be paranoid about its validity. */ + error = hubbub_utf16_next_paranoid(*source, *sourcelen, + 0, &nextchar); + if (error != HUBBUB_OK) { + if (error == HUBBUB_NEEDDATA) { + /* Need more data to be sure */ + if (*sourcelen > INVAL_BUFSIZE) + abort(); + + memmove(c->inval_buf, (char *) *source, + *sourcelen); + c->inval_buf[*sourcelen] = '\0'; + c->inval_len = *sourcelen; + + *source += *sourcelen; + *sourcelen = 0; + + nextchar = 0; + } else { + return error; + } + } + + /* output U+FFFD and continue processing. */ + error = hubbub_utf16_codec_filter_decoded_char(c, + 0xFFFD, dest, destlen); + if (error == HUBBUB_OK || error == HUBBUB_NOMEM) { + /* filter function succeeded; update source pointers */ + *source += nextchar; + *sourcelen -= nextchar; + } + + return error; + } + + return HUBBUB_OK; +} + +/** + * Feed a UCS4 character through the registered filter and output the result + * + * \param c Codec to use + * \param ucs4 UCS4 character (host endian) + * \param dest Pointer to pointer to output buffer + * \param destlen Pointer to output buffer length + * \return HUBBUB_OK on success, + * HUBBUB_NOMEM if output buffer is too small, + * <any_other_error> as a result of the failure of the + * client-provided filter function. + */ +hubbub_error hubbub_utf16_codec_filter_decoded_char(hubbub_utf16_codec *c, + uint32_t ucs4, uint8_t **dest, size_t *destlen) +{ + if (c->base.filter != NULL) { + uint32_t *rep; + size_t replen; + hubbub_error error; + + error = c->base.filter(ucs4, &rep, &replen, + c->base.filter_pw); + if (error != HUBBUB_OK) { + return error; + } + + while (replen > 0 && *destlen >= replen * 4) { + *((uint32_t *) (void *) *dest) = htonl(*rep); + + *dest += 4; + *destlen -= 4; + + rep++; + replen--; + } + + if (*destlen < replen * 4) { + /* Run out of output buffer */ + size_t i; + + /* Buffer remaining output */ + c->read_len = replen; + + for (i = 0; i < replen; i++) { + c->read_buf[i] = rep[i]; + } + + return HUBBUB_NOMEM; + } + + } else { + if (*destlen < 4) { + /* Run out of output buffer */ + c->read_len = 1; + c->read_buf[0] = ucs4; + + return HUBBUB_NOMEM; + } + + *((uint32_t *) (void *) *dest) = htonl(ucs4); + *dest += 4; + *destlen -= 4; + } + + return HUBBUB_OK; +} + + +const hubbub_charsethandler hubbub_utf16_codec_handler = { + hubbub_utf16_codec_handles_charset, + hubbub_utf16_codec_create +}; diff --git a/src/input/Makefile b/src/input/Makefile index 8b06c63..06f8ee4 100644 --- a/src/input/Makefile +++ b/src/input/Makefile @@ -22,7 +22,7 @@ CFLAGS += -I$(CURDIR) # Objects -OBJS = filter inputstream utf8_stream +OBJS = filter inputstream utf8_stream utf16_stream .PHONY: clean debug distclean export release setup test diff --git a/src/input/inputstream.c b/src/input/inputstream.c index f82d279..744aa23 100644 --- a/src/input/inputstream.c +++ b/src/input/inputstream.c @@ -22,9 +22,11 @@ struct hubbub_inputstream_bm_handler { }; extern hubbub_streamhandler utf8stream; +extern hubbub_streamhandler utf16stream; static hubbub_streamhandler *handler_table[] = { &utf8stream, + &utf16stream, NULL }; diff --git a/src/input/utf16_stream.c b/src/input/utf16_stream.c new file mode 100644 index 0000000..05ba5d3 --- /dev/null +++ b/src/input/utf16_stream.c @@ -0,0 +1,608 @@ +/* + * This file is part of Hubbub. + * Licensed under the MIT License, + * http://www.opensource.org/licenses/mit-license.php + * Copyright 2007 John-Mark Bell <jmb@netsurf-browser.org> + */ + +#include <stdbool.h> +#include <string.h> + +#include "charset/aliases.h" +#include "charset/detect.h" +#include "input/streamimpl.h" +#include "utils/utf16.h" +#include "utils/utils.h" + +#define BUFFER_CHUNK (4096) + +static bool hubbub_utf16stream_uses_encoding(const char *int_enc); +static hubbub_inputstream *hubbub_utf16stream_create(const char *enc, + const char *int_enc, hubbub_alloc alloc, void *pw); +static void hubbub_utf16stream_destroy(hubbub_inputstream *stream); +static hubbub_error hubbub_utf16stream_append(hubbub_inputstream *stream, + const uint8_t *data, size_t len); +static hubbub_error hubbub_utf16stream_insert(hubbub_inputstream *stream, + const uint8_t *data, size_t len); +static uint32_t hubbub_utf16stream_peek(hubbub_inputstream *stream); +static uint32_t hubbub_utf16stream_cur_pos(hubbub_inputstream *stream, + size_t *len); +static void hubbub_utf16stream_lowercase(hubbub_inputstream *stream); +static void hubbub_utf16stream_uppercase(hubbub_inputstream *stream); +static void hubbub_utf16stream_advance(hubbub_inputstream *stream); +static hubbub_error hubbub_utf16stream_push_back(hubbub_inputstream *stream, + uint32_t character); +static int hubbub_utf16stream_compare_range_ci(hubbub_inputstream *stream, + uint32_t r1, uint32_t r2, size_t len); +static int hubbub_utf16stream_compare_range_cs(hubbub_inputstream *stream, + uint32_t r1, uint32_t r2, size_t len); +static int hubbub_utf16stream_compare_range_ascii(hubbub_inputstream *stream, + uint32_t off, size_t len, const char *data, size_t dlen); +static hubbub_error hubbub_utf16stream_replace_range( + hubbub_inputstream *stream, + uint32_t start, size_t len, uint32_t ucs4); + +/** + * Determine whether a stream implementation uses an internal encoding + * + * \param int_enc The desired encoding + * \return true if handled, false otherwise + */ +bool hubbub_utf16stream_uses_encoding(const char *int_enc) +{ + return (hubbub_mibenum_from_name(int_enc, strlen(int_enc)) == + hubbub_mibenum_from_name("UTF-16", SLEN("UTF-16"))); +} + +/** + * Create an input stream + * + * \param enc Document charset, or NULL if unknown + * \param int_enc Desired encoding of document + * \param alloc Memory (de)allocation function + * \param pw Pointer to client-specific private data (may be NULL) + * \return Pointer to stream instance, or NULL on failure + */ +hubbub_inputstream *hubbub_utf16stream_create(const char *enc, + const char *int_enc, hubbub_alloc alloc, void *pw) +{ + hubbub_inputstream *stream; + + if (hubbub_mibenum_from_name(int_enc, strlen(int_enc)) != + hubbub_mibenum_from_name("UTF-16", SLEN("UTF-16"))) + return NULL; + + stream = alloc(NULL, sizeof(hubbub_inputstream), pw); + if (stream == NULL) + return NULL; + + stream->buffer = alloc(NULL, BUFFER_CHUNK, pw); + if (stream->buffer == NULL) { + alloc(stream, 0, pw); + return NULL; + } + + stream->buffer_len = 0; + stream->buffer_alloc = BUFFER_CHUNK; + + stream->cursor = 0; + + stream->had_eof = false; + + stream->input = hubbub_filter_create(int_enc, alloc, pw); + if (stream->input == NULL) { + alloc(stream->buffer, 0, pw); + alloc(stream, 0, pw); + return NULL; + } + + if (enc != NULL) { + hubbub_error error; + hubbub_filter_optparams params; + + stream->mibenum = hubbub_mibenum_from_name(enc, strlen(enc)); + + if (stream->mibenum != 0) { + params.encoding.name = enc; + + error = hubbub_filter_setopt(stream->input, + HUBBUB_FILTER_SET_ENCODING, ¶ms); + if (error != HUBBUB_OK && error != HUBBUB_INVALID) { + hubbub_filter_destroy(stream->input); + alloc(stream->buffer, 0, pw); + alloc(stream, 0, pw); + return NULL; + } + + stream->encsrc = HUBBUB_CHARSET_DICTATED; + } + } else { + stream->mibenum = 0; + stream->encsrc = HUBBUB_CHARSET_UNKNOWN; + } + + stream->destroy = hubbub_utf16stream_destroy; + stream->append = hubbub_utf16stream_append; + stream->insert = hubbub_utf16stream_insert; + stream->peek = hubbub_utf16stream_peek; + stream->cur_pos = hubbub_utf16stream_cur_pos; + stream->lowercase = hubbub_utf16stream_lowercase; + stream->uppercase = hubbub_utf16stream_uppercase; + stream->advance = hubbub_utf16stream_advance; + stream->push_back = hubbub_utf16stream_push_back; + stream->cmp_range_ci = hubbub_utf16stream_compare_range_ci; + stream->cmp_range_cs = hubbub_utf16stream_compare_range_cs; + stream->cmp_range_ascii = hubbub_utf16stream_compare_range_ascii; + stream->replace_range = hubbub_utf16stream_replace_range; + + return stream; +} + +/** + * Destroy an input stream + * + * \param stream Input stream to destroy + */ +void hubbub_utf16stream_destroy(hubbub_inputstream *stream) +{ + if (stream->input != NULL) { + hubbub_filter_destroy(stream->input); + } + + if (stream->buffer != NULL) { + stream->alloc(stream->buffer, 0, stream->pw); + } + + stream->alloc(stream, 0, stream->pw); +} + +/** + * Append data to an input stream + * + * \param stream Input stream to append data to + * \param data Data to append (in document charset), or NULL to flag EOF + * \param len Length, in bytes, of data + * \return HUBBUB_OK on success, appropriate error otherwise + */ +hubbub_error hubbub_utf16stream_append(hubbub_inputstream *stream, + const uint8_t *data, size_t len) +{ + hubbub_error error; + uint8_t *base; + size_t space; + + if (data == NULL) { + /* EOF indicated */ + size_t dummy_len = 0; + uint8_t *dummy_data = (uint8_t *) &dummy_len; + + base = stream->buffer + stream->buffer_len; + space = stream->buffer_alloc - stream->buffer_len; + + /* Forcibly flush through any remaining buffered data */ + while ((error = hubbub_filter_process_chunk(stream->input, + (const uint8_t **) &dummy_data, &dummy_len, + &base, &space)) == HUBBUB_NOMEM) { + bool moved = false; + uint8_t *temp = stream->alloc(stream->buffer, + stream->buffer_alloc + BUFFER_CHUNK, + stream->pw); + + if (temp == NULL) { + return HUBBUB_NOMEM; + } + + moved = (temp != stream->buffer); + + stream->buffer = temp; + stream->buffer_len += stream->buffer_alloc - + stream->buffer_len - space; + stream->buffer_alloc += BUFFER_CHUNK; + + base = stream->buffer + stream->buffer_len; + space = stream->buffer_alloc - stream->buffer_len; + + if (moved) + hubbub_inputstream_buffer_moved(stream); + } + + /* And fix up buffer length */ + stream->buffer_len += stream->buffer_alloc - + stream->buffer_len - space; + + stream->had_eof = true; + } else { + /* Normal data chunk */ + + if (stream->mibenum == 0) { + /* Haven't found charset yet; detect it */ + error = hubbub_charset_extract(&data, &len, + &stream->mibenum, &stream->encsrc); + if (error) { + return error; + } + + /* We should always have a charset by now */ + if (stream->mibenum == 0) + abort(); + } + + base = stream->buffer + stream->buffer_len; + space = stream->buffer_alloc - stream->buffer_len; + + /* Convert chunk to UTF-16 */ + while ((error = hubbub_filter_process_chunk(stream->input, + &data, &len, + &base, &space)) == HUBBUB_NOMEM) { + bool moved = false; + uint8_t *temp = stream->alloc(stream->buffer, + stream->buffer_alloc + BUFFER_CHUNK, + stream->pw); + + if (temp == NULL) { + return HUBBUB_NOMEM; + } + + moved = (temp != stream->buffer); + + stream->buffer = temp; + stream->buffer_len += stream->buffer_alloc - + stream->buffer_len - space; + stream->buffer_alloc += BUFFER_CHUNK; + + base = stream->buffer + stream->buffer_len; + space = stream->buffer_alloc - stream->buffer_len - + space; + + if (moved) + hubbub_inputstream_buffer_moved(stream); + } + + /* And fix up buffer length */ + stream->buffer_len += stream->buffer_alloc - + stream->buffer_len - space; + } + + return HUBBUB_OK; +} + +/** + * Insert data into stream at current location + * + * \param stream Input stream to insert into + * \param data Data to insert (UTF-16 encoded) + * \param len Length, in bytes, of data + * \return HUBBUB_OK on success, appropriate error otherwise + */ +hubbub_error hubbub_utf16stream_insert(hubbub_inputstream *stream, + const uint8_t *data, size_t len) +{ + size_t space; + uint8_t *curpos; + + space = stream->buffer_alloc - stream->buffer_len; + + /* Need to grow buffer, if there's insufficient space */ + if (space <= len) { + bool moved = false; + uint8_t *temp = stream->alloc(stream->buffer, + stream->buffer_alloc + + ((len + BUFFER_CHUNK - 1) & ~BUFFER_CHUNK) + + BUFFER_CHUNK, + stream->pw); + + if (temp == NULL) + return HUBBUB_NOMEM; + + moved = (temp != stream->buffer); + + stream->buffer = temp; + stream->buffer_alloc += + ((len + BUFFER_CHUNK - 1) & ~BUFFER_CHUNK); + + if (moved) + hubbub_inputstream_buffer_moved(stream); + } + + /* Find the insertion point + * (just before the next character to be read) */ + curpos = stream->buffer + stream->cursor; + + /* Move data above this point up */ + memmove(curpos + len, curpos, stream->buffer_len - stream->cursor); + + /* Copy new data into gap created by memmove */ + memcpy(curpos, data, len); + + /* Fix up buffer length */ + stream->buffer_len += len; + + return HUBBUB_OK; +} + +/** + * Look at the next character in the stream + * + * \param stream Stream to look in + * \return UCS4 (host-endian) character code, or EOF or OOD. + */ +uint32_t hubbub_utf16stream_peek(hubbub_inputstream *stream) +{ + hubbub_error error; + size_t len; + uint32_t ret; + + if (stream->cursor == stream->buffer_len) { + return stream->had_eof ? HUBBUB_INPUTSTREAM_EOF + : HUBBUB_INPUTSTREAM_OOD; + } + + error = hubbub_utf16_to_ucs4(stream->buffer + stream->cursor, + stream->buffer_len - stream->cursor, + &ret, &len); + if (error != HUBBUB_OK && error != HUBBUB_NEEDDATA) + return HUBBUB_INPUTSTREAM_OOD; + + if (error == HUBBUB_NEEDDATA) { + if (stream->had_eof) + return HUBBUB_INPUTSTREAM_EOF; + else + return HUBBUB_INPUTSTREAM_OOD; + } + + return ret; +} + +/** + * Retrieve the byte index and length of the current character in the stream + * + * \param stream Stream to look in + * \param len Pointer to location to receive byte length of character + * \return Byte index of current character from start of stream, + * or (uint32_t) -1 on error + */ +uint32_t hubbub_utf16stream_cur_pos(hubbub_inputstream *stream, + size_t *len) +{ + hubbub_utf16_char_byte_length(stream->buffer + stream->cursor, len); + + return stream->cursor; +} + +/** + * Convert the current character to lower case + * + * \param stream Stream to look in + */ +void hubbub_utf16stream_lowercase(hubbub_inputstream *stream) +{ + uint16_t *buf = (uint16_t *) (stream->buffer + stream->cursor); + + if (0x0041 <= buf[0] && buf[0] <= 0x005B) + buf[0] += 0x0020; +} + +/** + * Convert the current character to upper case + * + * \param stream Stream to look in + */ +void hubbub_utf16stream_uppercase(hubbub_inputstream *stream) +{ + uint16_t *buf = (uint16_t *) (stream->buffer + stream->cursor); + + if (0x0061 <= buf[0] && buf[0] <= 0x007B) + buf[0] -= 0x0020; +} + +/** + * Advance the stream's current position + * + * \param stream The stream whose position to advance + */ +void hubbub_utf16stream_advance(hubbub_inputstream *stream) +{ + hubbub_error error; + uint32_t next; + + error = hubbub_utf16_next(stream->buffer, stream->buffer_len, + stream->cursor, &next); + + if (error == HUBBUB_OK) + stream->cursor = next; +} + +/** + * Push a character back onto the stream + * + * \param stream Stream to push back to + * \param character UCS4 (host-endian) codepoint to push back + * \return HUBBUB_OK on success, appropriate error otherwise + * + * Note that this doesn't actually modify the data in the stream. + * It works by ensuring that the character located just before the + * current stream location is the same as ::character. If it is, + * then the stream pointer is moved back. If it is not, then an + * error is returned and the stream pointer remains unmodified. + */ +hubbub_error hubbub_utf16stream_push_back(hubbub_inputstream *stream, + uint32_t character) +{ + hubbub_error error; + uint32_t prev; + uint8_t buf[4]; + size_t len; + + error = hubbub_utf16_prev(stream->buffer, stream->cursor, &prev); + if (error != HUBBUB_OK) + return error; + + error = hubbub_utf16_from_ucs4(character, buf, &len); + if (error != HUBBUB_OK) + return error; + + if ((stream->cursor - prev) != len || + memcmp(stream->buffer + prev, buf, len) != 0) + return HUBBUB_INVALID; + + stream->cursor = prev; + + return HUBBUB_OK; +} + +/** + * Case insensitively compare a pair of ranges in the input stream + * + * \param stream Input stream to look in + * \param r1 Offset of start of first range + * \param r2 Offset of start of second range + * \param len Byte length of ranges + * \return 0 if ranges match, non-zero otherwise + */ +int hubbub_utf16stream_compare_range_ci(hubbub_inputstream *stream, + uint32_t r1, uint32_t r2, size_t len) +{ + uint8_t *range1 = (stream->buffer + r1); + uint8_t *range2 = (stream->buffer + r2); + int c1, c2; + uint32_t r1next, r2next; + hubbub_error error; + + if (len == 0) + return 0; + + do { + c1 = *((uint16_t *) range1); + c2 = *((uint16_t *) range2); + + if ((0x0041 <= c1 && c1 <= 0x005B)) + c1 |= 0x0020; + + if ((0x0041 <= c2 && c2 <= 0x005B)) + c2 |= 0x0020; + + error = hubbub_utf16_next(range1, len, 0, &r1next); + error = hubbub_utf16_next(range2, len, 0, &r2next); + + range1 += r1next; + range2 += r2next; + + len -= r1next; + } while(c1 != 0 && (c1 == c2) && len > 0); + + return (c1 - c2); +} + +/** + * Case sensitively compare a pair of ranges in the input stream + * + * \param stream Input stream to look in + * \param r1 Offset of start of first range + * \param r2 Offset of start of second range + * \param len Byte length of ranges + * \return 0 if ranges match, non-zero otherwise + */ +int hubbub_utf16stream_compare_range_cs(hubbub_inputstream *stream, + uint32_t r1, uint32_t r2, size_t len) +{ + return memcmp((const char *) (stream->buffer + r1), + (const char *) (stream->buffer + r2), len); +} + +/** + * Case sensitively compare a range of input stream against an ASCII string + * + * \param stream Input stream to look in + * \param off Offset of range start + * \param len Byte length of range + * \param data Comparison string + * \param dlen Byte length of comparison string + * \return 0 if match, non-zero otherwise + */ +int hubbub_utf16stream_compare_range_ascii(hubbub_inputstream *stream, + uint32_t off, size_t len, const char *data, size_t dlen) +{ + uint8_t *range = (stream->buffer + off); + int c1, c2; + + /* Lengths don't match, so strings don't */ + if (len != dlen * 2) + return 1; /* arbitrary */ + + do { + c1 = *((uint16_t *) range); + c2 = *data; + + range += 2; + data++; + + len -= 2; + } while (c1 != 0 && (c1 == c2) && len > 0); + + return (c1 - c2); +} + +/** + * Replace a range of bytes in the input stream with a single character + * + * \param stream Input stream containing data + * \param start Offset of start of range to replace + * \param len Length (in bytes) of range to replace + * \param ucs4 UCS4 (host endian) encoded replacement character + * \return HUBBUB_OK on success, appropriate error otherwise + */ +hubbub_error hubbub_utf16stream_replace_range(hubbub_inputstream *stream, + uint32_t start, size_t len, uint32_t ucs4) +{ + uint8_t buf[4]; + size_t replen; + int32_t diff; + hubbub_error error; + + /* Get utf16 version of replacement character */ + error = hubbub_utf16_from_ucs4(ucs4, buf, &replen); + if (error) + return error; + + diff = replen - len; + + if (stream->buffer_len + diff >= stream->buffer_alloc) { + /* Need more buffer space */ + bool moved = false; + uint8_t *temp = stream->alloc(stream->buffer, + stream->buffer_alloc + + ((diff + BUFFER_CHUNK - 1) & ~BUFFER_CHUNK) + + BUFFER_CHUNK, + stream->pw); + + if (temp == NULL) + return HUBBUB_NOMEM; + + moved = (temp != stream->buffer); + + stream->buffer = temp; + stream->buffer_alloc += + ((diff + BUFFER_CHUNK - 1) & ~BUFFER_CHUNK); + + if (moved) + hubbub_inputstream_buffer_moved(stream); + } + + /* Move subsequent input to correct location */ + memmove(stream->buffer + start + len + diff, + stream->buffer + start + len, + stream->buffer_len - (start + len)); + + /* And fill the gap with the replacement character */ + memcpy(stream->buffer + start, buf, replen); + + /* Finally, update length */ + stream->buffer_len += diff; + + return HUBBUB_OK; +} + +hubbub_streamhandler utf16stream = { + hubbub_utf16stream_uses_encoding, + hubbub_utf16stream_create +}; diff --git a/src/utils/Makefile b/src/utils/Makefile index 59b5512..53341cf 100644 --- a/src/utils/Makefile +++ b/src/utils/Makefile @@ -22,7 +22,7 @@ CFLAGS += -I$(CURDIR) # Objects -OBJS = dict errors utf8 +OBJS = dict errors utf8 utf16 .PHONY: clean debug distclean export release setup test diff --git a/src/utils/utf16.c b/src/utils/utf16.c new file mode 100644 index 0000000..8d337e3 --- /dev/null +++ b/src/utils/utf16.c @@ -0,0 +1,239 @@ +/* + * This file is part of Hubbub. + * Licensed under the MIT License, + * http://www.opensource.org/licenses/mit-license.php + * Copyright 2007 John-Mark Bell <jmb@netsurf-browser.org> + */ + +/** \file + * UTF-16 manipulation functions (implementation). + */ + +#include <stdbool.h> +#include <stdlib.h> +#include <string.h> + +#include "utils/utf16.h" + +/** + * Convert a UTF-16 sequence into a single UCS4 character + * + * \param s The sequence to process + * \param len Length of sequence + * \param ucs4 Pointer to location to receive UCS4 character (host endian) + * \param clen Pointer to location to receive byte length of UTF-16 sequence + * \return HUBBUB_OK on success, appropriate error otherwise + */ +inline hubbub_error hubbub_utf16_to_ucs4(const uint8_t *s, size_t len, + uint32_t *ucs4, size_t *clen) +{ + const uint16_t *ss = (const uint16_t *) s; + + if (s == NULL || ucs4 == NULL || clen == NULL) + return HUBBUB_BADPARM; + + if (len < 2) + return HUBBUB_NEEDDATA; + + if (*ss < 0xD800 || *ss > 0xDFFF) { + *ucs4 = *ss; + *clen = 2; + } else if (0xD800 <= *ss && *ss <= 0xBFFF) { + if (len < 4) + return HUBBUB_NEEDDATA; + + if (0xDC00 <= ss[1] && ss[1] <= 0xE000) { + *ucs4 = (((s[0] >> 6) & 0x1f) + 1) | + ((s[0] & 0x3f) | (s[1] & 0x3ff)); + *clen = 4; + } else { + return HUBBUB_INVALID; + } + } + + return HUBBUB_OK; +} + +/** + * Convert a single UCS4 character into a UTF-16 sequence + * + * \param ucs4 The character to process (0 <= c <= 0x7FFFFFFF) (host endian) + * \param s Pointer to 4 byte long output buffer + * \param len Pointer to location to receive length of multibyte sequence + * \return HUBBUB_OK on success, appropriate error otherwise + */ +inline hubbub_error hubbub_utf16_from_ucs4(uint32_t ucs4, uint8_t *s, + size_t *len) +{ + uint16_t *ss = (uint16_t *) s; + uint32_t l = 0; + + if (s == NULL || len == NULL) + return HUBBUB_BADPARM; + else if (ucs4 < 0x10000) { + *ss = (uint16_t) ucs4; + l = 2; + } else if (ucs4 < 0x110000) { + ss[0] = 0xD800 | (((ucs4 >> 16) & 0x1f) - 1) | (ucs4 >> 10); + ss[1] = 0xDC00 | (ucs4 & 0x3ff); + l = 4; + } else { + return HUBBUB_INVALID; + } + + *len = l; + + return HUBBUB_OK; +} + +/** + * Calculate the length (in characters) of a bounded UTF-16 string + * + * \param s The string + * \param max Maximum length + * \param len Pointer to location to receive length of string + * \return HUBBUB_OK on success, appropriate error otherwise + */ +inline hubbub_error hubbub_utf16_length(const uint8_t *s, size_t max, + size_t *len) +{ + const uint16_t *ss = (const uint16_t *) s; + const uint16_t *end = (const uint16_t *) (s + max); + int l = 0; + + if (s == NULL || len == NULL) + return HUBBUB_BADPARM; + + while (ss < end) { + if (*ss < 0xD800 || 0xDFFF < *ss) + ss++; + else + ss += 2; + + l++; + } + + *len = l; + + return HUBBUB_OK; +} + +/** + * Calculate the length (in bytes) of a UTF-16 character + * + * \param s Pointer to start of character + * \param len Pointer to location to receive length + * \return HUBBUB_OK on success, appropriate error otherwise + */ +inline hubbub_error hubbub_utf16_char_byte_length(const uint8_t *s, + size_t *len) +{ + const uint16_t *ss = (const uint16_t *) s; + + if (s == NULL || len == NULL) + return HUBBUB_BADPARM; + + if (*ss < 0xD800 || 0xDFFF < *ss) + *len = 2; + else + *len = 4; + + return HUBBUB_OK; +} + +/** + * Find previous legal UTF-16 char in string + * + * \param s The string + * \param off Offset in the string to start at + * \param prevoff Pointer to location to receive offset of first byte of + * previous legal character + * \return HUBBUB_OK on success, appropriate error otherwise + */ +inline hubbub_error hubbub_utf16_prev(const uint8_t *s, uint32_t off, + uint32_t *prevoff) +{ + const uint16_t *ss = (const uint16_t *) s; + + if (s == NULL || prevoff == NULL) + return HUBBUB_BADPARM; + + if (off < 2) + *prevoff = 0; + else if (ss[-1] < 0xDC00 || ss[-1] > 0xDFFF) + *prevoff = off - 2; + else + *prevoff = (off < 4) ? 0 : off - 4; + + return HUBBUB_OK; +} + +/** + * Find next legal UTF-16 char in string + * + * \param s The string (assumed valid) + * \param len Maximum offset in string + * \param off Offset in the string to start at + * \param nextoff Pointer to location to receive offset of first byte of + * next legal character + * \return HUBBUB_OK on success, appropriate error otherwise + */ +inline hubbub_error hubbub_utf16_next(const uint8_t *s, uint32_t len, + uint32_t off, uint32_t *nextoff) +{ + const uint16_t *ss = (const uint16_t *) s; + + if (s == NULL || off >= len || nextoff == NULL) + return HUBBUB_BADPARM; + + if (len - off < 4) + *nextoff = len; + else if (ss[1] < 0xD800 || ss[1] > 0xDBFF) + *nextoff = off + 2; + else + *nextoff = (len - off < 6) ? len : off + 4; + + return HUBBUB_OK; +} + +/** + * Find next legal UTF-16 char in string + * + * \param s The string (assumed to be of dubious validity) + * \param len Maximum offset in string + * \param off Offset in the string to start at + * \param nextoff Pointer to location to receive offset of first byte of + * next legal character + * \return HUBBUB_OK on success, appropriate error otherwise + */ +inline hubbub_error hubbub_utf16_next_paranoid(const uint8_t *s, + uint32_t len, uint32_t off, uint32_t *nextoff) +{ + const uint16_t *ss = (const uint16_t *) s; + + if (s == NULL || off >= len || nextoff == NULL) + return HUBBUB_BADPARM; + + while (1) { + if (len - off < 4) { + return HUBBUB_NEEDDATA; + } else if (ss[1] < 0xD800 || ss[1] > 0xDFFF) { + *nextoff = off + 2; + break; + } else if (ss[1] >= 0xD800 && ss[1] <= 0xDBFF) { + if (len - off < 6) + return HUBBUB_NEEDDATA; + + if (ss[2] >= 0xDC00 && ss[2] <= 0xDFFF) { + *nextoff = off + 4; + break; + } else { + ss++; + off += 2; + } + } + } + + return HUBBUB_OK; +} + diff --git a/src/utils/utf16.h b/src/utils/utf16.h new file mode 100644 index 0000000..4605e03 --- /dev/null +++ b/src/utils/utf16.h @@ -0,0 +1,38 @@ +/* + * This file is part of Hubbub. + * Licensed under the MIT License, + * http://www.opensource.org/licenses/mit-license.php + * Copyright 2007 John-Mark Bell <jmb@netsurf-browser.org> + */ + +/** \file + * UTF-16 manipulation functions (interface). + */ + +#ifndef hubbub_utils_utf16_h_ +#define hubbub_utils_utf16_h_ + +#include <inttypes.h> + +#include <hubbub/errors.h> + +inline hubbub_error hubbub_utf16_to_ucs4(const uint8_t *s, size_t len, + uint32_t *ucs4, size_t *clen); +inline hubbub_error hubbub_utf16_from_ucs4(uint32_t ucs4, uint8_t *s, + size_t *len); + +inline hubbub_error hubbub_utf16_length(const uint8_t *s, size_t max, + size_t *len); +inline hubbub_error hubbub_utf16_char_byte_length(const uint8_t *s, + size_t *len); + +inline hubbub_error hubbub_utf16_prev(const uint8_t *s, uint32_t off, + uint32_t *prevoff); +inline hubbub_error hubbub_utf16_next(const uint8_t *s, uint32_t len, + uint32_t off, uint32_t *nextoff); + +inline hubbub_error hubbub_utf16_next_paranoid(const uint8_t *s, + uint32_t len, uint32_t off, uint32_t *nextoff); + +#endif + |