From a4c5929a2fac1cb0c039b2d009d8093ac81a90d7 Mon Sep 17 00:00:00 2001
From: James Bursa <james@netsurf-browser.org>
Date: Sun, 9 Feb 2003 12:58:15 +0000
Subject: [project @ 2003-02-09 12:58:14 by bursa] Reorganization and rewrite
 of fetch, cache, and content handling.

svn path=/import/netsurf/; revision=96
---
 content/cache.c      | 245 +++++++++++++++++++++++++++++++++++
 content/cache.h      |  38 ++++++
 content/content.c    | 145 +++++++++++++++++++++
 content/content.h    |  95 ++++++++++++++
 content/fetch.c      | 295 ++++++++++++++++++++++++++++++++++++++++++
 content/fetch.h      |  20 +++
 content/fetchcache.c | 149 +++++++++++++++++++++
 content/fetchcache.h |  16 +++
 desktop/browser.c    | 358 ++++++---------------------------------------------
 desktop/browser.h    |  69 +---------
 desktop/cache.c      | 128 ------------------
 desktop/cache.h      |  32 -----
 desktop/fetch.c      | 211 ------------------------------
 desktop/fetch.h      |  49 -------
 desktop/netsurf.c    |  25 ++--
 desktop/netsurf.h    |  21 +--
 makefile             |  46 ++++---
 render/box.c         |  26 ++--
 render/box.h         |   8 +-
 render/css.c         |  16 +--
 render/html.c        | 156 ++++++++++++++++++++++
 render/html.h        |  17 +++
 render/layout.c      |  21 +--
 render/textplain.c   |  52 ++++++++
 render/textplain.h   |  17 +++
 riscos/font.c        |   4 +-
 riscos/gui.c         |   5 +-
 riscos/theme.c       |   2 +-
 utils/log.h          |   4 +-
 utils/utils.c        |   4 +-
 utils/utils.h        |   6 +-
 31 files changed, 1374 insertions(+), 906 deletions(-)
 create mode 100644 content/cache.c
 create mode 100644 content/cache.h
 create mode 100644 content/content.c
 create mode 100644 content/content.h
 create mode 100644 content/fetch.c
 create mode 100644 content/fetch.h
 create mode 100644 content/fetchcache.c
 create mode 100644 content/fetchcache.h
 delete mode 100644 desktop/cache.c
 delete mode 100644 desktop/cache.h
 delete mode 100644 desktop/fetch.c
 delete mode 100644 desktop/fetch.h
 create mode 100644 render/html.c
 create mode 100644 render/html.h
 create mode 100644 render/textplain.c
 create mode 100644 render/textplain.h

diff --git a/content/cache.c b/content/cache.c
new file mode 100644
index 000000000..30a39053b
--- /dev/null
+++ b/content/cache.c
@@ -0,0 +1,245 @@
+/**
+ * $Id: cache.c,v 1.1 2003/02/09 12:58:14 bursa Exp $
+ */
+
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include "netsurf/content/cache.h"
+#include "netsurf/utils/utils.h"
+#include "netsurf/utils/log.h"
+
+#ifndef TEST
+#include "netsurf/desktop/browser.h"
+#else
+#include <unistd.h>
+struct content {
+	char *url;
+	struct cache_entry *cache;
+	unsigned long size;
+};
+void content_destroy(struct content *c);
+#endif
+
+
+/**
+ * internal structures and declarations
+ */
+
+struct cache_entry {
+	struct content *content;
+	unsigned int use_count;
+	time_t t;
+	struct cache_entry *next, *prev;
+};
+
+/* doubly-linked lists using a sentinel */
+/* TODO: replace with a structure which can be searched faster */
+static struct cache_entry inuse_list_sentinel  = {0, 0, 0, &inuse_list_sentinel,  &inuse_list_sentinel};
+static struct cache_entry unused_list_sentinel = {0, 0, 0, &unused_list_sentinel, &unused_list_sentinel};
+static struct cache_entry *inuse_list  = &inuse_list_sentinel;
+static struct cache_entry *unused_list = &unused_list_sentinel;
+
+static unsigned long max_size = 1024*1024;	/* TODO: make this configurable */
+static unsigned long current_size = 0;
+
+
+/**
+ * cache_init -- initialise the cache manager
+ */
+
+void cache_init(void)
+{
+}
+
+
+/**
+ * cache_quit -- terminate the cache manager
+ */
+
+void cache_quit(void)
+{
+}
+
+
+/**
+ * cache_get -- retrieve url from memory cache or disc cache
+ */
+
+struct content * cache_get(char * const url)
+{
+	struct cache_entry *e;
+
+	/* search inuse_list first */
+	for (e = inuse_list->next; e != inuse_list && strcmp(e->content->url, url) != 0; e = e->next)
+		;
+	if (e != inuse_list) {
+		LOG(("'%s' in inuse_list, content %p, use_count %u", url, e->content, e->use_count));
+		e->use_count++;
+		return e->content;
+	}
+
+	/* search unused_list if not found */
+	for (e = unused_list->next; e != unused_list && strcmp(e->content->url, url) != 0; e = e->next)
+		;
+	if (e != unused_list) {
+		LOG(("'%s' in unused_list, content %p", url, e->content));
+		/* move to inuse_list */
+		e->use_count = 1;
+		e->prev->next = e->next;
+		e->next->prev = e->prev;
+		e->prev = inuse_list->prev;
+		e->next = inuse_list;
+		inuse_list->prev->next = e;
+		inuse_list->prev = e;
+		return e->content;
+	}
+
+	LOG(("'%s' not in cache", url));
+	return 0;
+}
+
+
+/**
+ * cache_put -- place content in the memory cache
+ */
+
+void cache_put(struct content * content)
+{
+	struct cache_entry * e;
+	LOG(("content %p, url '%s'", content, content->url));
+
+	current_size += content->size;
+	/* clear old data from the usused_list until the size drops below max_size */
+	while (max_size < current_size && unused_list->next != unused_list) {
+		e = unused_list->next;
+		LOG(("size %lu, removing %p '%s'", current_size, e->content, e->content->url));
+		/* TODO: move to disc cache */
+		current_size -= e->content->size;
+		content_destroy(e->content);
+		unused_list->next = e->next;
+		e->next->prev = e->prev;
+		xfree(e);
+	}
+
+	/* add the new content to the inuse_list */
+	e = xcalloc(1, sizeof(struct cache_entry));
+	e->content = content;
+	e->use_count = 1;
+	e->prev = inuse_list->prev;
+	e->next = inuse_list;
+	inuse_list->prev->next = e;
+	inuse_list->prev = e;
+	content->cache = e;
+}
+
+
+/**
+ * cache_free -- free a cache object if it is no longer used
+ */
+
+void cache_free(struct content * content)
+{
+	struct cache_entry * e = content->cache;
+
+	assert(e != 0);
+	LOG(("content %p, url '%s', use_count %u", content, content->url, e->use_count));
+
+	assert(e->use_count != 0);
+	e->use_count--;
+	if (e->use_count == 0) {
+		/* move to unused_list or destroy if insufficient space */
+		e->use_count = 0;
+		e->t = time(0);
+		e->prev->next = e->next;
+		e->next->prev = e->prev;
+		if (max_size < current_size) {
+			LOG(("size %lu, removing", current_size));
+			/* TODO: move to disc cache */
+			current_size -= e->content->size;
+			content_destroy(e->content);
+			xfree(e);
+		} else {
+			LOG(("size %lu, moving to unused_list", current_size));
+			e->prev = unused_list->prev;
+			e->next = unused_list;
+			unused_list->prev->next = e;
+			unused_list->prev = e;
+		}
+	}
+}
+
+
+/**
+ * cache_dump -- dump contents of cache
+ */
+
+void cache_dump(void) {
+	struct cache_entry * e;
+	LOG(("size %lu", current_size));
+	LOG(("inuse_list:"));
+	for (e = inuse_list->next; e != inuse_list; e = e->next)
+		LOG(("  content %p, url '%s', use_count %u", e->content, e->content->url, e->use_count));
+	LOG(("unused_list (time now %lu):", time(0)));
+	for (e = unused_list->next; e != unused_list; e = e->next)
+		LOG(("  content %p, url '%s', t %lu", e->content, e->content->url, e->t));
+	LOG(("end"));
+}
+
+
+/**
+ * testing framework
+ */
+
+#ifdef TEST
+struct content test[] = {
+	{"aaa", 0, 200 * 1024},
+	{"bbb", 0, 100 * 1024},
+	{"ccc", 0, 400 * 1024},
+	{"ddd", 0, 600 * 1024},
+	{"eee", 0, 300 * 1024},
+	{"fff", 0, 500 * 1024},
+};
+
+#define TEST_COUNT (sizeof(test) / sizeof(test[0]))
+
+unsigned int test_state[TEST_COUNT];
+
+void content_destroy(struct content *c)
+{
+}
+
+int main(void)
+{
+	int i;
+	struct content *c;
+	for (i = 0; i != TEST_COUNT; i++)
+		test_state[i] = 0;
+
+	cache_init();
+
+	for (i = 0; i != 100; i++) {
+		int x = rand() % TEST_COUNT;
+		switch (rand() % 2) {
+			case 0:
+				c = cache_get(test[x].url);
+				if (c == 0) {
+					assert(test_state[x] == 0);
+					cache_put(&test[x]);
+				} else
+					assert(c == &test[x]);
+				test_state[x]++;
+				break;
+			case 1:
+				if (test_state[x] != 0) {
+					cache_free(&test[x]);
+					test_state[x]--;
+				}
+				break;
+		}
+	}
+	cache_dump();
+	return 0;
+}
+#endif
diff --git a/content/cache.h b/content/cache.h
new file mode 100644
index 000000000..2ee4e3b81
--- /dev/null
+++ b/content/cache.h
@@ -0,0 +1,38 @@
+/**
+ * $Id: cache.h,v 1.1 2003/02/09 12:58:14 bursa Exp $
+ */
+
+/**
+ * Using the cache:
+ *
+ *     cache_init();
+ *     ...
+ *     c = cache_get(url);
+ *     if (c == 0) {
+ *         ... (create c) ...
+ *         cache_put(c);
+ *     }
+ *     ...
+ *     cache_free(c);
+ *     ...
+ *     cache_quit();
+ *
+ * cache_free informs the cache that the content is no longer being used, so
+ * it can be deleted from the cache if necessary. There must be a call to
+ * cache_free for each cache_get or cache_put.
+ */
+
+#ifndef _NETSURF_DESKTOP_CACHE_H_
+#define _NETSURF_DESKTOP_CACHE_H_
+
+struct content;
+struct cache_entry;
+
+void cache_init(void);
+void cache_quit(void);
+struct content * cache_get(char * const url);
+void cache_put(struct content * content);
+void cache_free(struct content * content);
+void cache_dump(void);
+
+#endif
diff --git a/content/content.c b/content/content.c
new file mode 100644
index 000000000..c8601400d
--- /dev/null
+++ b/content/content.c
@@ -0,0 +1,145 @@
+/**
+ * $Id: content.c,v 1.1 2003/02/09 12:58:14 bursa Exp $
+ */
+
+#include <assert.h>
+#include <string.h>
+#include <stdlib.h>
+#include "netsurf/content/content.h"
+#include "netsurf/render/html.h"
+#include "netsurf/render/textplain.h"
+#include "netsurf/utils/utils.h"
+
+
+/* mime_map must be in sorted order by mime_type */
+struct mime_entry {
+	char mime_type[16];
+	content_type type;
+};
+static const struct mime_entry mime_map[] = {
+/*	{"image/png", CONTENT_PNG},
+	{"text/css", CONTENT_CSS},*/
+	{"text/html", CONTENT_HTML},
+	{"text/plain", CONTENT_TEXTPLAIN},
+};
+#define MIME_MAP_COUNT (sizeof(mime_map) / sizeof(mime_map[0]))
+
+/* handler_map must be ordered as enum content_type */
+struct handler_entry {
+	void (*create)(struct content *c);
+	void (*process_data)(struct content *c, char *data, unsigned long size);
+	int (*convert)(struct content *c, unsigned int width, unsigned int height);
+	void (*revive)(struct content *c, unsigned int width, unsigned int height);
+	void (*reformat)(struct content *c, unsigned int width, unsigned int height);
+	void (*destroy)(struct content *c);
+};
+static const struct handler_entry handler_map[] = {
+	{html_create, html_process_data, html_convert, html_revive, html_reformat, html_destroy},
+	{textplain_create, textplain_process_data, textplain_convert,
+		textplain_revive, textplain_reformat, textplain_destroy},
+/*	{css_create, css_process_data, css_convert, css_revive, css_destroy},
+	{png_create, png_process_data, png_convert, png_revive, png_destroy},*/
+};
+
+
+/**
+ * content_lookup -- look up mime type
+ */
+
+content_type content_lookup(const char *mime_type)
+{
+	struct mime_entry *m;
+	m = bsearch(mime_type, mime_map, MIME_MAP_COUNT, sizeof(mime_map[0]),
+			(int (*)(const void *, const void *)) strcmp);
+	if (m == 0)
+		return CONTENT_OTHER;
+	return m->type;
+}
+
+
+/**
+ * content_create -- create a content structure of the specified mime type
+ */
+
+struct content * content_create(content_type type, char *url)
+{
+	struct content *c;
+	assert(type < CONTENT_OTHER);
+	c = xcalloc(1, sizeof(struct content));
+	c->url = xstrdup(url);
+	c->type = type;
+	c->status = CONTENT_LOADING;
+	c->size = sizeof(struct content);
+	handler_map[type].create(c);
+	return c;
+}
+
+
+/**
+ * content_process_data -- process a block source data
+ */
+
+void content_process_data(struct content *c, char *data, unsigned long size)
+{
+	assert(c != 0);
+	assert(c->type < CONTENT_OTHER);
+	assert(c->status == CONTENT_LOADING);
+	handler_map[c->type].process_data(c, data, size);
+}
+
+
+/**
+ * content_convert -- all data has arrived, complete the conversion
+ */
+
+int content_convert(struct content *c, unsigned long width, unsigned long height)
+{
+	assert(c != 0);
+	assert(c->type < CONTENT_OTHER);
+	assert(c->status == CONTENT_LOADING);
+	if (handler_map[c->type].convert(c, width, height))
+		return 1;
+	c->status = CONTENT_READY;
+	return 0;
+}
+
+
+/**
+ * content_revive -- fix content that has been loaded from the cache
+ *   eg. load dependencies, reformat to current width
+ */
+
+void content_revive(struct content *c, unsigned long width, unsigned long height)
+{
+	assert(c != 0);
+	assert(c->type < CONTENT_OTHER);
+	assert(c->status == CONTENT_READY);
+	handler_map[c->type].revive(c, width, height);
+}
+
+
+/**
+ * content_reformat -- reformat to new size
+ */
+
+void content_reformat(struct content *c, unsigned long width, unsigned long height)
+{
+	assert(c != 0);
+	assert(c->type < CONTENT_OTHER);
+	assert(c->status == CONTENT_READY);
+	handler_map[c->type].reformat(c, width, height);
+}
+
+
+/**
+ * content_destroy -- free content
+ */
+
+void content_destroy(struct content *c)
+{
+	assert(c != 0);
+	assert(c->type < CONTENT_OTHER);
+	handler_map[c->type].destroy(c);
+	xfree(c);
+}
+
diff --git a/content/content.h b/content/content.h
new file mode 100644
index 000000000..ed93e7c24
--- /dev/null
+++ b/content/content.h
@@ -0,0 +1,95 @@
+/**
+ * $Id: content.h,v 1.1 2003/02/09 12:58:14 bursa Exp $
+ */
+
+#ifndef _NETSURF_DESKTOP_CONTENT_H_
+#define _NETSURF_DESKTOP_CONTENT_H_
+
+#include "libxml/HTMLparser.h"
+#include "netsurf/content/cache.h"
+#include "netsurf/render/css.h"
+#include "netsurf/render/box.h"
+#include "netsurf/riscos/font.h"
+
+
+/**
+ * A struct content corresponds to a single url.
+ *
+ * It is in one of the following states:
+ *   CONTENT_FETCHING - the data is being fetched and/or converted
+ *                      for use by the browser
+ *   CONTENT_READY - the content has been processed and is ready
+ *                      to display
+ *
+ * The converted data is stored in the cache, not the source data.
+ * Users of the structure are counted in use_count; when use_count = 0
+ * the content may be removed from the memory cache.
+ */
+
+typedef enum {CONTENT_HTML, CONTENT_TEXTPLAIN, CONTENT_CSS,
+	CONTENT_PNG, CONTENT_OTHER} content_type;
+
+struct box_position
+{
+  struct box* box;
+  int actual_box_x;
+  int actual_box_y;
+  int plot_index;
+  int pixel_offset;
+  int char_offset;
+};
+
+struct content
+{
+  char *url;
+  content_type type;
+  enum {CONTENT_LOADING, CONTENT_READY} status;
+
+  union
+  {
+    struct
+    {
+      htmlParserCtxt* parser;
+      xmlDoc* document;
+      xmlNode* markup;
+      struct box* layout;
+      struct css_stylesheet* stylesheet;
+      struct css_style* style;
+      struct {
+        struct box_position start;
+        struct box_position end;
+        enum {alter_UNKNOWN, alter_START, alter_END} altering;
+        int selected; /* 0 = unselected, 1 = selected */
+      } text_selection;
+      struct font_set* fonts;
+      struct page_elements elements;
+    } html;
+
+    struct
+    {
+      struct css_stylesheet * stylesheet;
+    } css;
+
+    struct
+    {
+      unsigned long width, height;
+      char * sprite;
+    } image;
+  
+  } data;
+
+  struct cache_entry *cache;
+  unsigned long size;
+  char *title;
+};
+
+
+content_type content_lookup(const char *mime_type);
+struct content * content_create(content_type type, char *url);
+void content_process_data(struct content *c, char *data, unsigned long size);
+int content_convert(struct content *c, unsigned long width, unsigned long height);
+void content_revive(struct content *c, unsigned long width, unsigned long height);
+void content_reformat(struct content *c, unsigned long width, unsigned long height);
+void content_destroy(struct content *c);
+
+#endif
diff --git a/content/fetch.c b/content/fetch.c
new file mode 100644
index 000000000..63283238a
--- /dev/null
+++ b/content/fetch.c
@@ -0,0 +1,295 @@
+/**
+ * $Id: fetch.c,v 1.1 2003/02/09 12:58:14 bursa Exp $
+ */
+
+#include <assert.h>
+#include <time.h>
+#include "curl/curl.h"
+#include "netsurf/content/fetch.h"
+#include "netsurf/utils/utils.h"
+#include "netsurf/utils/log.h"
+
+#ifndef TEST
+#else
+#include <unistd.h>
+#endif
+
+struct fetch
+{
+	time_t start_time;
+	CURL * curl_handle;
+	void (*callback)(fetch_msg msg, void *p, char *data, unsigned long size);
+	int had_headers : 1;
+	int in_callback : 1;
+	int aborting : 1;
+	char *url;
+	char error_buffer[CURL_ERROR_SIZE];
+	void *p;
+};
+
+static const char * const user_agent = "NetSurf";
+static CURLM * curl_multi;
+
+static size_t fetch_curl_data(void * data, size_t size, size_t nmemb, struct fetch *f);
+
+
+/**
+ * fetch_init -- initialise the fetcher
+ */
+
+void fetch_init(void)
+{
+	CURLcode code;
+
+	code = curl_global_init(CURL_GLOBAL_ALL);
+	if (code != CURLE_OK)
+		die("curl_global_init failed");
+
+	curl_multi = curl_multi_init();
+	if (curl_multi == 0)
+		die("curl_multi_init failed");
+}
+
+
+/**
+ * fetch_quit -- clean up for quit
+ */
+
+void fetch_quit(void)
+{
+	CURLMcode codem;
+
+	codem = curl_multi_cleanup(curl_multi);
+	if (codem != CURLM_OK)
+		LOG(("curl_multi_cleanup failed: ignoring"));
+
+	curl_global_cleanup();
+}
+
+
+/**
+ * fetch_start -- start fetching data for the given url
+ *
+ * Returns immediately. The callback function will be called when
+ * something interesting happens.
+ */
+
+struct fetch * fetch_start(char *url, char *referer,
+                 void (*callback)(fetch_msg msg, void *p, char *data, unsigned long size), void *p)
+{
+	struct fetch* fetch = (struct fetch*) xcalloc(1, sizeof(struct fetch));
+	CURLcode code;
+	CURLMcode codem;
+
+	LOG(("fetch %p, url '%s'", fetch, url));
+  
+	fetch->start_time = time(&fetch->start_time);
+	fetch->callback = callback;
+	fetch->had_headers = 0;
+	fetch->in_callback = 0;
+	fetch->aborting = 0;
+	fetch->url = xstrdup(url);
+	fetch->p = p;
+
+	/* create the curl easy handle */
+	fetch->curl_handle = curl_easy_init();
+	assert(fetch->curl_handle != 0);  /* TODO: handle curl errors */
+	code = curl_easy_setopt(fetch->curl_handle, CURLOPT_URL, fetch->url);
+	assert(code == CURLE_OK);
+	code = curl_easy_setopt(fetch->curl_handle, CURLOPT_PRIVATE, fetch);
+	assert(code == CURLE_OK);
+	code = curl_easy_setopt(fetch->curl_handle, CURLOPT_ERRORBUFFER, fetch->error_buffer);
+	assert(code == CURLE_OK);
+	code = curl_easy_setopt(fetch->curl_handle, CURLOPT_WRITEFUNCTION, fetch_curl_data);
+	assert(code == CURLE_OK);
+	code = curl_easy_setopt(fetch->curl_handle, CURLOPT_WRITEDATA, fetch);
+	assert(code == CURLE_OK);
+	code = curl_easy_setopt(fetch->curl_handle, CURLOPT_USERAGENT, user_agent);
+	assert(code == CURLE_OK);
+	if (referer != 0) {
+		code = curl_easy_setopt(fetch->curl_handle, CURLOPT_REFERER, referer);
+		assert(code == CURLE_OK);
+	} 
+
+	/* add to the global curl multi handle */
+	codem = curl_multi_add_handle(curl_multi, fetch->curl_handle);
+	assert(codem == CURLM_OK || codem == CURLM_CALL_MULTI_PERFORM);
+
+	/* do any possible work on the fetch */
+	while (codem == CURLM_CALL_MULTI_PERFORM) {
+		int running;
+		codem = curl_multi_perform(curl_multi, &running);
+		assert(codem == CURLM_OK || codem == CURLM_CALL_MULTI_PERFORM);
+	}
+
+	return fetch;
+}
+
+
+/**
+ * fetch_abort -- stop a fetch
+ */
+
+void fetch_abort(struct fetch *f)
+{
+	CURLMcode codem;
+
+	assert(f != 0);
+	LOG(("fetch %p, url '%s'", f, f->url));
+  
+	if (f->in_callback) {
+		LOG(("in callback: will abort later"));
+		f->aborting = 1;
+		return;
+	}
+  
+	/* remove from curl */
+	codem = curl_multi_remove_handle(curl_multi, f->curl_handle);
+	assert(codem == CURLM_OK);
+	curl_easy_cleanup(f->curl_handle);
+
+	xfree(f->url);
+	xfree(f);
+}
+
+
+/**
+ * fetch_poll -- do some work on current fetches
+ *
+ * Must return as soon as possible.
+ */
+
+void fetch_poll(void)
+{
+	CURLcode code;
+	CURLMcode codem;
+	int running, queue;
+	CURLMsg * curl_msg;
+	struct fetch *f;
+
+	/* do any possible work on the current fetches */
+	do {
+		codem = curl_multi_perform(curl_multi, &running);
+		assert(codem == CURLM_OK || codem == CURLM_CALL_MULTI_PERFORM);
+	} while (codem == CURLM_CALL_MULTI_PERFORM);
+
+	/* process curl results */
+	curl_msg = curl_multi_info_read(curl_multi, &queue);
+	while (curl_msg) {
+		switch (curl_msg->msg) {
+			case CURLMSG_DONE:
+				/* find the structure associated with this fetch */
+				code = curl_easy_getinfo(curl_msg->easy_handle, CURLINFO_PRIVATE, &f);
+				assert(code == CURLE_OK);
+
+				LOG(("CURLMSG_DONE, result %i", curl_msg->data.result));
+
+				/* inform the caller that the fetch is done */
+				if (curl_msg->data.result == CURLE_OK)
+					f->callback(FETCH_FINISHED, f->p, 0, 0);
+				else if (curl_msg->data.result != CURLE_WRITE_ERROR)
+					f->callback(FETCH_ERROR, f->p, f->error_buffer, 0);
+
+				/* clean up fetch */
+				fetch_abort(f);
+
+				break;
+
+			default:
+				assert(0);
+		}
+		curl_msg = curl_multi_info_read(curl_multi, &queue);
+	}
+}
+
+
+/**
+ * fetch_curl_data -- callback function for curl (internal)
+ */
+
+size_t fetch_curl_data(void * data, size_t size, size_t nmemb, struct fetch *f)
+{
+	f->in_callback = 1;
+
+	LOG(("fetch %p, size %lu", f, size * nmemb));
+
+	if (!f->had_headers) {
+		/* find the content type and inform the caller */
+		char *type;
+		CURLcode code;
+
+		code = curl_easy_getinfo(f->curl_handle, CURLINFO_CONTENT_TYPE, &type);
+		assert(code == CURLE_OK);
+
+		if (type == 0)
+			type = "text/html";  /* TODO: find type of file: urls */
+
+		LOG(("FETCH_TYPE, '%s'", type));
+		f->callback(FETCH_TYPE, f->p, type, 0);
+		if (f->aborting) {
+			f->in_callback = 0;
+			return 0;
+		}
+
+		f->had_headers = 1;
+	}
+
+	/* send data to the caller */
+	LOG(("FETCH_DATA"));
+	f->callback(FETCH_DATA, f->p, data, size * nmemb);
+
+	f->in_callback = 0;
+	return size * nmemb;
+}
+
+
+/**
+ * testing framework
+ */
+
+#ifdef TEST
+struct test {char *url; struct fetch *f;};
+
+void callback(fetch_msg msg, struct test *t, char *data, unsigned long size)
+{
+	printf("%s: ", t->url);
+	switch (msg) {
+		case FETCH_TYPE:
+			printf("FETCH_TYPE '%s'", data);
+			break;
+		case FETCH_DATA:
+			printf("FETCH_DATA %lu", size);
+			break;
+		case FETCH_FINISHED:
+			printf("FETCH_FINISHED");
+			break;
+		case FETCH_ERROR:
+			printf("FETCH_ERROR '%s'", data);
+			break;
+		default:
+			assert(0);
+	}
+	printf("\n");
+}
+
+struct test test[] = {
+	{"http://www.oxfordstudent.com/", 0},
+	{"http://www.google.co.uk/", 0},
+	{"http://doesnt.exist/", 0},
+	{"blah://blah", 0},
+};
+
+int main(void)
+{
+	int i;
+	fetch_init();
+	for (i = 0; i != sizeof(test) / sizeof(test[0]); i++)
+		test[i].f = fetch_start(test[i].url, 0, callback, &test[i]);
+	while (1) {
+		fetch_poll();
+		sleep(1);
+	}
+	return 0;
+}
+#endif
+
diff --git a/content/fetch.h b/content/fetch.h
new file mode 100644
index 000000000..171bf33cb
--- /dev/null
+++ b/content/fetch.h
@@ -0,0 +1,20 @@
+/**
+ * $Id: fetch.h,v 1.1 2003/02/09 12:58:14 bursa Exp $
+ */
+
+#ifndef _NETSURF_DESKTOP_FETCH_H_
+#define _NETSURF_DESKTOP_FETCH_H_
+
+typedef enum {FETCH_TYPE, FETCH_DATA, FETCH_FINISHED, FETCH_ERROR} fetch_msg;
+
+struct content;
+struct fetch;
+
+void fetch_init(void);
+struct fetch * fetch_start(char *url, char *referer,
+                 void (*callback)(fetch_msg msg, void *p, char *data, unsigned long size), void *p);
+void fetch_abort(struct fetch *f);
+void fetch_poll(void);
+void fetch_quit(void);
+
+#endif
diff --git a/content/fetchcache.c b/content/fetchcache.c
new file mode 100644
index 000000000..cd41f62ff
--- /dev/null
+++ b/content/fetchcache.c
@@ -0,0 +1,149 @@
+/**
+ * $Id: fetchcache.c,v 1.1 2003/02/09 12:58:14 bursa Exp $
+ */
+
+#include <assert.h>
+#include "netsurf/content/cache.h"
+#include "netsurf/content/fetchcache.h"
+#include "netsurf/content/fetch.h"
+#include "netsurf/utils/log.h"
+#include "netsurf/utils/utils.h"
+
+
+struct fetchcache {
+	void *url;
+	void (*callback)(fetchcache_msg msg, struct content *c, void *p, char *error);
+	void *p;
+	struct fetch *f;
+	struct content *c;
+	unsigned long width, height;
+};
+
+
+void fetchcache_free(struct fetchcache *fc);
+void fetchcache_callback(fetchcache_msg msg, struct fetchcache *fc, char *data, unsigned long size);
+
+
+void fetchcache(char *url, char *referer,
+		void (*callback)(fetchcache_msg msg, struct content *c, void *p, char *error),
+		void *p, unsigned long width, unsigned long height)
+{
+	struct content *c;
+	struct fetchcache *fc;
+
+	c = cache_get(url);
+	if (c != 0) {
+		content_revive(c, width, height);
+		callback(FETCHCACHE_OK, c, p, 0);
+		return;
+	}
+
+	fc = xcalloc(1, sizeof(struct fetchcache));
+	fc->url = xstrdup(url);
+	fc->callback = callback;
+	fc->p = p;
+	fc->c = 0;
+	fc->width = width;
+	fc->height = height;
+	fc->f = fetch_start(url, referer, fetchcache_callback, fc);
+}
+
+
+void fetchcache_free(struct fetchcache *fc)
+{
+	free(fc->url);
+	free(fc);
+}
+
+
+void fetchcache_callback(fetch_msg msg, struct fetchcache *fc, char *data, unsigned long size)
+{
+	content_type type;
+	switch (msg) {
+		case FETCH_TYPE:
+			type = content_lookup(data);
+			LOG(("FETCH_TYPE, type %u", type));
+			if (type == CONTENT_OTHER) {
+				fetch_abort(fc->f);
+				fc->callback(FETCHCACHE_BADTYPE, 0, fc->p, 0);
+				free(fc);
+			} else {
+				fc->c = content_create(type, fc->url);
+			}
+			break;
+		case FETCH_DATA:
+			LOG(("FETCH_DATA"));
+			assert(fc->c != 0);
+			content_process_data(fc->c, data, size);
+			break;
+		case FETCH_FINISHED:
+			LOG(("FETCH_FINISHED"));
+			assert(fc->c != 0);
+			if (content_convert(fc->c, fc->width, fc->height) == 0) {
+				cache_put(fc->c);
+				fc->callback(FETCHCACHE_OK, fc->c, fc->p, 0);
+			} else {
+				content_destroy(fc->c);
+				fc->callback(FETCHCACHE_ERROR, 0, fc->p, "Conversion failed");
+			}
+			free(fc);
+			break;
+		case FETCH_ERROR:
+			LOG(("FETCH_ERROR, '%s'", data));
+			if (fc->c != 0)
+				content_destroy(fc->c);
+			fc->callback(FETCHCACHE_ERROR, 0, fc->p, data);
+			free(fc);
+			break;
+		default:
+			assert(0);
+	}
+}
+
+
+#ifdef TEST
+
+#include <unistd.h>
+
+void callback(fetchcache_msg msg, struct content *c, void *p, char *error)
+{
+	switch (msg) {
+		case FETCHCACHE_OK:
+			LOG(("FETCHCACHE_OK, url '%s'", p));
+			break;
+		case FETCHCACHE_BADTYPE:
+			LOG(("FETCHCACHE_BADTYPE, url '%s'", p));
+			break;
+		case FETCHCACHE_ERROR:
+			LOG(("FETCHCACHE_ERROR, url '%s', error '%s'", p, error));
+			break;
+		default:
+			assert(0);
+	}
+}
+
+char *test[] = {"http://www.google.co.uk/", "http://www.ox.ac.uk/", "blah://blah/"};
+
+int main(void)
+{
+	int i;
+
+	cache_init();
+	fetch_init();
+
+	for (i = 0; i != sizeof(test) / sizeof(test[0]); i++)
+		fetchcache(test[i], 0, callback, test[i], 800, 0);
+	for (i = 0; i != 5; i++) {
+		fetch_poll();
+		sleep(1);
+	}
+	for (i = 0; i != sizeof(test) / sizeof(test[0]); i++)
+		fetchcache(test[i], 0, callback, test[i], 800, 0);
+	for (i = 0; i != 20; i++) {
+		fetch_poll();
+		sleep(1);
+	}
+	return 0;
+}
+
+#endif
diff --git a/content/fetchcache.h b/content/fetchcache.h
new file mode 100644
index 000000000..d212b56a5
--- /dev/null
+++ b/content/fetchcache.h
@@ -0,0 +1,16 @@
+/**
+ * $Id: fetchcache.h,v 1.1 2003/02/09 12:58:14 bursa Exp $
+ */
+
+#ifndef _NETSURF_DESKTOP_FETCHCACHE_H_
+#define _NETSURF_DESKTOP_FETCHCACHE_H_
+
+#include "netsurf/content/content.h"
+
+typedef enum {FETCHCACHE_OK, FETCHCACHE_BADTYPE, FETCHCACHE_ERROR} fetchcache_msg;
+
+void fetchcache(char *url, char *referer,
+		void (*callback)(fetchcache_msg msg, struct content *c, void *p, char *error),
+		void *p, unsigned long width, unsigned long height);
+
+#endif
diff --git a/desktop/browser.c b/desktop/browser.c
index 270574901..f18ea38f4 100644
--- a/desktop/browser.c
+++ b/desktop/browser.c
@@ -1,15 +1,14 @@
 /**
- * $Id: browser.c,v 1.22 2003/01/12 17:48:44 bursa Exp $
+ * $Id: browser.c,v 1.23 2003/02/09 12:58:14 bursa Exp $
  */
 
+#include "netsurf/content/cache.h"
+#include "netsurf/content/fetchcache.h"
+#include "netsurf/desktop/browser.h"
 #include "netsurf/riscos/font.h"
 #include "netsurf/render/box.h"
-#include "netsurf/render/layout.h"
-#include "netsurf/render/css.h"
-#include "netsurf/desktop/browser.h"
-#include "netsurf/render/utils.h"
-#include "netsurf/desktop/cache.h"
 #include "netsurf/utils/log.h"
+#include "netsurf/utils/utils.h"
 #include "libxml/uri.h"
 #include "libxml/debugXML.h"
 #include <string.h>
@@ -25,9 +24,11 @@ void browser_window_change_text_selection(struct browser_window* bw, struct box_
 void browser_window_redraw_boxes(struct browser_window* bw, struct box_position* start, struct box_position* end);
 void browser_window_follow_link(struct browser_window* bw,
   int click_x, int click_y, int click_type);
-
+void browser_window_callback(fetchcache_msg msg, struct content *c,
+		struct browser_window* bw, char *error);
 void box_under_area(struct box* box, int x, int y, int ox, int oy, struct box_selection** found, int* count, int* plot_index);
 
+
 void browser_window_start_throbber(struct browser_window* bw)
 {
   bw->throbbing = 1;
@@ -41,180 +42,6 @@ void browser_window_stop_throbber(struct browser_window* bw)
   gui_window_stop_throbber(bw->window);
 }
 
-void content_destroy(struct content* c)
-{
-  if (c == NULL)
-    return;
-
-  switch (c->type)
-  {
-    case CONTENT_HTML:
-      /* free other memory here */
-//      xmlFreeParserCtxt(c->data.html.parser);
-      LOG(("free parser"));
-//      htmlFreeParserCtxt(c->data.html.parser);
-      LOG(("free sheet"));
-//      xfree(c->data.html.stylesheet);
-      LOG(("free style"));
-//      xfree(c->data.html.style);
-      if (c->data.html.layout != NULL)
-      {
-        LOG(("box_free box"));
-//        box_free(c->data.html.layout);
-        LOG(("free box"));
-//        xfree(c->data.html.layout);
-      }
-      LOG(("free font"));
-      font_free_set(c->data.html.fonts);
-      break;
-    default:
-      break;
-  }
-
-  c->main_fetch = fetch_cancel(c->main_fetch);
-  xfree(c);
-
-  return;
-}
-
-size_t content_html_receive_data(struct content* c, void* data, size_t size, size_t nmemb)
-{
-  size_t amount = nmemb;
-  int offset = 0;
-  size_t numInChunk = 2048 / size;  /* process in 2k chunks */
-
-  if (numInChunk > nmemb)
-    numInChunk = nmemb;
-  else if (numInChunk <= (size_t)0)
-    numInChunk = 1;
-
-  while (amount > 0)
-  {
-    htmlParseChunk(c->data.html.parser, (char*)data + (offset * size), numInChunk, 0);
-    offset += numInChunk;
-    amount -= numInChunk;
-    if (amount < numInChunk)
-      numInChunk = amount;
-    gui_multitask();
-  }
-
-  return size * nmemb;
-}
-
-void set_content_html(struct content* c)
-{
-  c->type = CONTENT_HTML;
-  c->data.html.parser = htmlCreatePushParserCtxt(0, 0, "", 0, 0, XML_CHAR_ENCODING_8859_1);
-  c->data.html.document = NULL;
-  c->data.html.markup = NULL;
-  c->data.html.layout = NULL;
-  c->data.html.stylesheet = NULL;
-  c->data.html.style = NULL;
-  return;
-}
-
-
-void content_html_title(struct content *c)
-{
-  xmlNode *node = c->data.html.markup;
-
-  c->title = 0;
-
-  while (node != 0) {
-    if (node->type == XML_ELEMENT_NODE) {
-      if (stricmp(node->name, "html") == 0) {
-        node = node->children;
-        continue;
-      }
-      if (stricmp(node->name, "head") == 0) {
-        node = node->children;
-        continue;
-      }
-      if (stricmp(node->name, "title") == 0) {
-        c->title = xmlNodeGetContent(node);
-	return;
-      }
-    }
-    node = node->next;
-  }
-}
-
-
-void content_html_reformat(struct content* c, int width)
-{
-  char* file;
-  struct css_selector* selector = xcalloc(1, sizeof(struct css_selector));
-
-  LOG(("Starting stuff"));
-
-  if (c->data.html.layout != NULL)
-  {
-    /* TODO: skip if width is unchanged */
-    layout_document(c->data.html.layout->children, (unsigned long)width);
-    return;
-  }
-
-  LOG(("Setting document to myDoc"));
-  c->data.html.document = c->data.html.parser->myDoc;
-  //xmlDebugDumpDocument(stderr, c->data.html.parser->myDoc);
-
-  /* skip to start of html */
-  LOG(("Skipping to html"));
-  if (c->data.html.document == NULL)
-  {
-    LOG(("There is no document!"));
-    return;
-  }
-  for (c->data.html.markup = c->data.html.document->children;
-       c->data.html.markup != 0 &&
-         c->data.html.markup->type != XML_ELEMENT_NODE;
-       c->data.html.markup = c->data.html.markup->next)
-    ;
-
-  if (c->data.html.markup == 0)
-  {
-    LOG(("No markup"));
-    return;
-  }
-  if (strcmp((const char *) c->data.html.markup->name, "html"))
-  {
-    LOG(("Not html"));
-    return;
-  }
-
-//  xfree(c->data.html.stylesheet);
-//  xfree(c->data.html.style);
-
-  content_html_title(c);
-  
-  LOG(("Loading CSS"));
-  file = load("<NetSurf$Dir>.Resources.CSS");  /*!!! not portable! !!!*/
-  c->data.html.stylesheet = css_new_stylesheet();
-  LOG(("Parsing stylesheet"));
-  css_parse_stylesheet(c->data.html.stylesheet, file);
-
-  LOG(("Copying base style"));
-  c->data.html.style = xcalloc(1, sizeof(struct css_style));
-  memcpy(c->data.html.style, &css_base_style, sizeof(struct css_style));
-
-  LOG(("Creating box"));
-  c->data.html.layout = xcalloc(1, sizeof(struct box));
-  c->data.html.layout->type = BOX_BLOCK;
-  c->data.html.layout->node = c->data.html.markup;
-
-  c->data.html.fonts = font_new_set();
-
-  LOG(("XML to box"));
-  xml_to_box(c->data.html.markup, c->data.html.style, c->data.html.stylesheet, &selector, 0, c->data.html.layout, 0, 0, c->data.html.fonts, 0, 0, 0, 0, &c->data.html.elements);
-  box_dump(c->data.html.layout->children, 0);
-  LOG(("Layout document"));
-  layout_document(c->data.html.layout->children, (unsigned long)width);
-  box_dump(c->data.html.layout->children, 0);
-
-  /* can tidy up memory here? */
-
-  return;
-}
 
 void browser_window_reformat(struct browser_window* bw)
 {
@@ -222,8 +49,7 @@ void browser_window_reformat(struct browser_window* bw)
   clock_t time0, time1;
 
   LOG(("Entering..."));
-  if (bw == NULL)
-    return;
+  assert(bw != 0);
   if (bw->current_content == NULL)
     return;
 
@@ -231,34 +57,21 @@ void browser_window_reformat(struct browser_window* bw)
   {
     case CONTENT_HTML:
       LOG(("HTML content."));
-      browser_window_set_status(bw, "Formatting page...");
       time0 = clock();
-      content_html_reformat(bw->current_content, gui_window_get_width(bw->window));
-      if (bw->current_content == 0)
+      if (bw->current_content->title == 0)
         gui_window_set_title(bw->window, bw->url);
       else
         gui_window_set_title(bw->window, bw->current_content->title);
       time1 = clock();
-      LOG(("Content reformatted"));
-      if (bw->current_content->data.html.layout != NULL)
-      {
-        LOG(("Setting extent"));
-        gui_window_set_extent(bw->window, bw->current_content->data.html.layout->children->width, bw->current_content->data.html.layout->children->height);
-        LOG(("Setting scroll"));
-        gui_window_set_scroll(bw->window, 0, 0);
-        LOG(("Redraw window"));
-        gui_window_redraw_window(bw->window);
-        LOG(("Complete"));
-        sprintf(status, "Format complete (%gs).", ((float) time1 - time0) / CLOCKS_PER_SEC);
-        browser_window_set_status(bw, status);
-      }
-      else
-      {
-        LOG(("This isn't html"));
-        browser_window_set_status(bw, "This is not HTML!");
-        cache_free(bw->current_content);
-        bw->current_content = NULL;
-      }
+      LOG(("Setting extent"));
+      gui_window_set_extent(bw->window, bw->current_content->data.html.layout->children->width, bw->current_content->data.html.layout->children->height);
+      LOG(("Setting scroll"));
+      gui_window_set_scroll(bw->window, 0, 0);
+      LOG(("Redraw window"));
+      gui_window_redraw_window(bw->window);
+      LOG(("Complete"));
+      sprintf(status, "Format complete (%gs).", ((float) time1 - time0) / CLOCKS_PER_SEC);
+      browser_window_set_status(bw, status);
       break;
     default:
         LOG(("Unknown content type"));
@@ -346,7 +159,6 @@ struct browser_window* create_browser_window(int flags, int width, int height)
   bw->scale.div = 1;
 
   bw->current_content = NULL;
-  bw->future_content = NULL;
   bw->history = NULL;
 
   bw->url = NULL;
@@ -370,8 +182,6 @@ void browser_window_destroy(struct browser_window* bw)
 
   if (bw->current_content != NULL)
     cache_free(bw->current_content);
-  if (bw->future_content != NULL)
-    cache_free(bw->future_content);
 
   if (bw->history != NULL)
   {
@@ -411,60 +221,9 @@ void browser_window_open_location_historical(struct browser_window* bw, char* ur
 
   assert(bw != 0 && url != 0);
 
-  if (bw->future_content != NULL)
-    cache_free(bw->future_content);
-
-  bw->future_content = cache_get(url);
-  if (bw->future_content == 0)
-  {
-    /* not in cache: start fetch */
-    struct fetch_request* req;
-
-    LOG(("not in cache: starting fetch"));
-
-    req = xcalloc(1, sizeof(struct fetch_request));
-    req->type = REQUEST_FROM_BROWSER;
-    req->requestor.browser = bw;
-
-    bw->future_content = (struct content*) xcalloc(1, sizeof(struct content));
-    bw->future_content->main_fetch = create_fetch(url, bw->url, 0, req);
-
-    cache_put(url, bw->future_content, 1000);
-
-    browser_window_start_throbber(bw);
-  }
-  else
-  {
-    /* in cache: reformat page and display */
-    struct gui_message gmsg;
-    gui_safety previous_safety;
-
-    LOG(("in cache: reformatting"));
-
-    browser_window_start_throbber(bw);
-
-    /* TODO: factor out code shared with browser_window_message(), case msg_FETCH_FINISHED */
-    if (url != bw->url)  /* reload <=> url == bw->url */
-    {
-      if (bw->url != NULL)
-        xfree(bw->url);
-      bw->url = xstrdup(url);
-    }
-
-    gmsg.type = msg_SET_URL;
-    gmsg.data.set_url.url = bw->url;
-    gui_window_message(bw->window, &gmsg);
-
-    previous_safety = gui_window_set_redraw_safety(bw->window, UNSAFE);
-    if (bw->current_content != NULL)
-      cache_free(bw->current_content);
-    bw->current_content = bw->future_content;
-    bw->future_content = NULL;
-    browser_window_reformat(bw);
-    gui_window_set_redraw_safety(bw->window, previous_safety);
-    browser_window_stop_throbber(bw);
-  }
-
+  browser_window_set_status(bw, "Opening page...");
+  fetchcache(url, 0, browser_window_callback, bw, gui_window_get_width(bw->window), 0);
+  
   LOG(("end"));
 }
 
@@ -485,68 +244,25 @@ void browser_window_open_location(struct browser_window* bw, char* url)
   LOG(("end"));
 }
 
-int browser_window_message(struct browser_window* bw, struct browser_message* msg)
+void browser_window_callback(fetchcache_msg msg, struct content *c,
+		struct browser_window* bw, char *error)
 {
   gui_safety previous_safety;
 
-  switch (msg->type)
+  switch (msg)
   {
-    case msg_FETCH_SENDING:
-      browser_window_set_status(bw, "Sending request...");
-      break;
-
-    case msg_FETCH_WAITING:
-      browser_window_set_status(bw, "Waiting for reply...");
-      break;
-
-    case msg_FETCH_FETCH_INFO:
-      browser_window_set_status(bw, "Request received...");
-      if (msg->f == bw->future_content->main_fetch)
-      {
-        switch (msg->data.fetch_info.type)
-        {
-          case type_HTML:
-            set_content_html(bw->future_content);
-            break;
-          default:
-            browser_window_stop_throbber(bw);
-            return 1;
-        }
-      }
-      break;
-
-    case msg_FETCH_DATA:
-      browser_window_set_status(bw, "Data received...");
-      if (msg->f == bw->future_content->main_fetch)
-        content_html_receive_data(bw->future_content, msg->data.fetch_data.block, sizeof(char), msg->data.fetch_data.block_size);
-      break;
-
-    case msg_FETCH_ABORT:
-      browser_window_set_status(bw, "Request failed.");
-      if (msg->f == bw->future_content->main_fetch)
-      {
-        browser_window_stop_throbber(bw);
-        bw->future_content->main_fetch = NULL;
-        cache_free(bw->future_content);
-        bw->future_content = NULL;
-      }
-      break;
-
-    case msg_FETCH_FINISHED:
+    case FETCHCACHE_OK:
       browser_window_set_status(bw, "Request complete.");
-      if (msg->f == bw->future_content->main_fetch)
       {
         struct gui_message gmsg;
-        if (bw->future_content->main_fetch->location != NULL)
+        if (bw->url != 0)
           xfree(bw->url);
-        bw->url = xstrdup(bw->future_content->main_fetch->location);
+        bw->url = xstrdup(c->url);
 
         gmsg.type = msg_SET_URL;
         gmsg.data.set_url.url = bw->url;
         gui_window_message(bw->window, &gmsg);
 
-        htmlParseChunk(bw->future_content->data.html.parser, "", 0, 1);
-        bw->future_content->main_fetch = NULL;
         previous_safety = gui_window_set_redraw_safety(bw->window, UNSAFE);
         if (bw->current_content != NULL)
 	{
@@ -557,20 +273,26 @@ int browser_window_message(struct browser_window* bw, struct browser_message* ms
 	  }
           cache_free(bw->current_content);
 	}
-        bw->current_content = bw->future_content;
-        bw->future_content = NULL;
+        bw->current_content = c;
         browser_window_reformat(bw);
         gui_window_set_redraw_safety(bw->window, previous_safety);
         browser_window_stop_throbber(bw);
       }
       break;
 
-    default:
-      browser_window_set_status(bw, "???");
+    case FETCHCACHE_ERROR:
+      browser_window_set_status(bw, error);
+      browser_window_stop_throbber(bw);
       break;
-  }
 
-  return 0;
+    case FETCHCACHE_BADTYPE:
+      browser_window_set_status(bw, "Unknown type");
+      browser_window_stop_throbber(bw);
+      break;
+
+    default:
+      assert(0);
+  }
 }
 
 void clear_radio_gadgets(struct browser_window* bw, struct box* box, struct gui_gadget* group)
diff --git a/desktop/browser.h b/desktop/browser.h
index 31275bde6..3271155b9 100644
--- a/desktop/browser.h
+++ b/desktop/browser.h
@@ -1,15 +1,13 @@
 /**
- * $Id: browser.h,v 1.6 2003/01/12 17:48:44 bursa Exp $
+ * $Id: browser.h,v 1.7 2003/02/09 12:58:14 bursa Exp $
  */
 
 #ifndef _NETSURF_DESKTOP_BROWSER_H_
 #define _NETSURF_DESKTOP_BROWSER_H_
 
-#include "libxml/HTMLparser.h"
-#include "netsurf/render/css.h"
-#include "netsurf/render/box.h"
+#include "netsurf/content/content.h"
 #include "netsurf/desktop/gui.h"
-#include "netsurf/desktop/fetch.h"
+#include "netsurf/render/box.h"
 #include "netsurf/riscos/font.h"
 
 typedef int browser_window_flags;
@@ -29,45 +27,6 @@ typedef int action_buttons;
 
 
 
-struct box_position
-{
-  struct box* box;
-  int actual_box_x;
-  int actual_box_y;
-  int plot_index;
-  int pixel_offset;
-  int char_offset;
-};
-
-struct content
-{
-  enum {CONTENT_UNKNOWN, CONTENT_HTML, CONTENT_IMAGE} type;
-
-  union
-  {
-    struct
-    {
-      htmlParserCtxt* parser;
-      xmlDoc* document;
-      xmlNode* markup;
-      struct box* layout;
-      struct css_stylesheet* stylesheet;
-      struct css_style* style;
-      struct {
-        struct box_position start;
-        struct box_position end;
-        enum {alter_UNKNOWN, alter_START, alter_END} altering;
-        int selected; /* 0 = unselected, 1 = selected */
-      } text_selection;
-      struct font_set* fonts;
-      struct page_elements elements;
-    } html;
-  } data;
-  struct fetch* main_fetch;
-  unsigned int ref_count;
-  char *title;
-};
-
 
 struct history
 {
@@ -88,7 +47,6 @@ struct browser_window
   struct { int mult; int div; } scale;
 
   struct content* current_content;
-  struct content* future_content;
   struct history* history;
 
   char* url;
@@ -101,26 +59,6 @@ struct browser_window
 };
 
 
-struct browser_message
-{
-  enum { msg_UNKNOWN,
-         msg_FETCH_SENDING, msg_FETCH_WAITING, msg_FETCH_ABORT,
-         msg_FETCH_FETCH_INFO, msg_FETCH_DATA, msg_FETCH_FINISHED
-       } type;
-  struct fetch* f;
-  union {
-    struct {
-      enum { type_UNKNOWN, type_HTML } type; /* should be a MIME type ? */
-      int total_size; /* -1 == unknown size */
-    } fetch_info;
-    struct {
-      char* block;
-      int block_size;
-    } fetch_data;
-  } data;
-};
-
-
 struct browser_action
 {
   enum { act_UNKNOWN,
@@ -156,7 +94,6 @@ struct browser_window* create_browser_window(int flags, int width, int height);
 void browser_window_destroy(struct browser_window* bw);
 void browser_window_open_location(struct browser_window* bw, char* url);
 void browser_window_open_location_historical(struct browser_window* bw, char* url);
-int browser_window_message(struct browser_window* bw, struct browser_message* msg);
 int browser_window_action(struct browser_window* bw, struct browser_action* act);
 void browser_window_set_status(struct browser_window* bw, char* text);
 
diff --git a/desktop/cache.c b/desktop/cache.c
deleted file mode 100644
index 012683ebe..000000000
--- a/desktop/cache.c
+++ /dev/null
@@ -1,128 +0,0 @@
-/**
- * $Id: cache.c,v 1.1 2002/11/02 22:28:05 bursa Exp $
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include "netsurf/desktop/cache.h"
-#include "netsurf/render/utils.h"
-#include "netsurf/utils/log.h"
-#include "curl/curl.h"
-#include "ubiqx/ubi_Cache.h"
-
-/**
- * internal structures and declarations
- */
-
-ubi_cacheRoot memcache;
-
-struct memcache_entry {
-	ubi_cacheEntry Node;
-	char * url;
-	struct content * content;
-};
-
-static int memcache_compare(ubi_trItemPtr item, ubi_trNode * node);
-void memcache_free(ubi_trNode * node);
-
-
-/**
- * cache_init -- initialise the cache manager
- */
-
-void cache_init(void)
-{
-	/* memory cache */
-	ubi_cacheInit(&memcache, memcache_compare, memcache_free, 40, 100*1024);
-}
-
-
-/**
- * cache_quit -- terminate the cache manager
- */
-
-void cache_quit(void)
-{
-	ubi_cacheClear(&memcache);
-}
-
-
-/**
- * cache_get -- retrieve url from memory cache or disc cache
- */
-
-struct content * cache_get(char * const url)
-{
-	struct memcache_entry * entry;
-
-	entry = (struct memcache_entry *) ubi_cacheGet(&memcache, url);
-	if (entry != 0) {
-		LOG(("url %s in cache, node %p", url, entry));
-		entry->content->ref_count++;
-		return entry->content;
-	}
-
-	LOG(("url %s not cached", url));
-
-	/* TODO: check disc cache */
-
-	return 0;
-}
-
-
-/**
- * cache_put -- place content in the memory cache
- */
-
-void cache_put(char * const url, struct content * content, unsigned long size)
-{
-	struct memcache_entry * entry;
-	
-	entry = xcalloc(1, sizeof(struct memcache_entry));
-	entry->url = xstrdup(url);
-	entry->content = content;
-	content->ref_count = 2;  /* cache, caller */
-	ubi_cachePut(&memcache, size,
-			(ubi_cacheEntry *) entry, entry->url);
-}
-
-
-/**
- * cache_free -- free a cache object if it is no longer used
- */
-
-void cache_free(struct content * content)
-{
-	LOG(("content %p, ref_count %u", content, content->ref_count));
-	if (--content->ref_count == 0) {
-		LOG(("ref count 0, freeing"));
-		content_destroy(content);
-	}
-}
-
-
-/**
- * memory cache
- */
-
-static int memcache_compare(ubi_trItemPtr item, ubi_trNode * node)
-{
-	return strcmp((char *) item, ((struct memcache_entry *) node)->url);
-}
-
-
-void memcache_free(ubi_trNode * node)
-{
-	struct memcache_entry * entry = (struct memcache_entry *) node;
-
-	LOG(("node %p, node->url %s", node, entry->url));
-
-	cache_free(entry->content);
-	free(entry->url);
-	free(entry);
-
-	/* TODO: place the object in a disc cache */
-}
-
-
diff --git a/desktop/cache.h b/desktop/cache.h
deleted file mode 100644
index 90131d8c2..000000000
--- a/desktop/cache.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * $Id: cache.h,v 1.1 2002/11/02 22:28:05 bursa Exp $
- */
-
-/**
- * Using the cache:
- *
- *     cache_init();
- *     ...
- *     c = cache_get(url);
- *     if (c == 0) {
- *         ... (create c) ...
- *         cache_put(url, c, size);
- *     }
- *     ...
- *     cache_free(c);
- *     ...
- *     cache_quit();
- *
- * cache_free informs the cache that the content is no longer being used, so
- * it can be deleted from the cache if necessary. There must be a call to
- * cache_free for each cache_get or cache_put.
- */
-
-#include "netsurf/desktop/browser.h"
-
-void cache_init(void);
-void cache_quit(void);
-struct content * cache_get(char * const url);
-void cache_put(char * const url, struct content * content, unsigned long size);
-void cache_free(struct content * content);
-
diff --git a/desktop/fetch.c b/desktop/fetch.c
deleted file mode 100644
index 3ff8dd09c..000000000
--- a/desktop/fetch.c
+++ /dev/null
@@ -1,211 +0,0 @@
-/**
- * $Id: fetch.c,v 1.7 2003/01/06 00:04:43 bursa Exp $
- */
-
-#include "libxml/HTMLparser.h"
-#include "netsurf/render/box.h"
-#include "netsurf/render/css.h"
-#include "netsurf/desktop/browser.h"
-#include "netsurf/desktop/netsurf.h"
-#include "netsurf/desktop/fetch.h"
-#include "netsurf/render/utils.h"
-#include "netsurf/utils/log.h"
-#include "curl/curl.h"
-#include <time.h>
-#include <string.h>
-#include <stdio.h>
-
-struct fetch* create_fetch(char* location, char* previous, fetch_flags f, struct fetch_request* r)
-{
-  struct fetch* fetch = (struct fetch*) xcalloc(1, sizeof(struct fetch));
-
-  if (fetch->location != NULL)
-    free(fetch->location);
-
-  fetch->location = xstrdup(location);
-  fetch->type = fetch_CURL;
-
-  fetch->flags = f;
-
-  fetch->status = fetch_STATUS_WAIT;
-  fetch->bytes_fetched = 0;
-  fetch->bytes_total = -1;
-
-  fetch->request = r;
-
-  fetch->start_time = time(&fetch->start_time);
-
-  fetch->next = netsurf_fetches;
-  netsurf_fetches = fetch;
-
-  return fetch;
-}
-
-void fetch_destroy(struct fetch* f)
-{
-  if (f == NULL)
-    return;
-
-  if (netsurf_fetches == f)
-    netsurf_fetches = f->next;
-  else
-  {
-    struct fetch* ff = netsurf_fetches;
-    while (ff->next != f && ff->next != NULL)
-      ff = ff->next;
-    if (ff->next == f)
-      ff->next = f->next;
-  }
-
-  xfree(f->location);
-  xfree(f->request);
-  xfree(f);
-}
-
-struct fetch* fetch_cancel(struct fetch* f)
-{
-  if (f == NULL)
-    return NULL;
-
-  /* may need to contact server here */
-
-  f->status = fetch_DELETED;
-  /* fetch may not necessarily be destroyed if the cancelling can't be done
-     instantly */
-  return f;
-}
-
-void fetch_receive(struct fetch* f, int amount, char* bytes)
-{
-  struct browser_message msg;
-
-  f->bytes_fetched = f->bytes_fetched + amount;
-
-  switch (f->request->type)
-  {
-    case REQUEST_FROM_BROWSER:
-      msg.type = msg_FETCH_DATA;
-      msg.f = f;
-      msg.data.fetch_data.block = bytes;
-      msg.data.fetch_data.block_size = amount;
-      if (browser_window_message(f->request->requestor.browser, &msg) != 0)
-      {
-        fetch_cancel(f);
-        return;
-      }
-      break;
-    default:
-      break;
-  }
-
-  if (f->bytes_fetched >= f->bytes_total && f->bytes_total != -1)
-  {
-    msg.type = msg_FETCH_FINISHED;
-    msg.f = f;
-    browser_window_message(f->request->requestor.browser, &msg);
-    fetch_destroy(f);
-  }
-
-  return;
-}
-
-size_t fetch_curl_data(void * data, size_t size, size_t nmemb, struct fetch* f)
-{
-  struct browser_message msg;
-  msg.type = msg_FETCH_DATA;
-  msg.f = f;
-  msg.data.fetch_data.block = data;
-  msg.data.fetch_data.block_size = size * nmemb;
-  LOG(("sending curl's FETCH_DATA to browser"));
-  browser_window_message(f->request->requestor.browser, &msg);
-  return size * nmemb;
-}
-
-struct fetch* fetch_poll(struct fetch* f)
-{
-  struct fetch* ret = f;
-
-/*   LOG(("polling...")); */
-
-  if (f == NULL)
-  {
-/*     LOG(("null fetch; returning")); */
-    return f;
-  }
-
-  if (f->status == fetch_DELETED)
-  {
-    ret = f->next;
-    LOG(("deleting marked fetch"));
-    fetch_destroy(f);
-    LOG(("moving on..."));
-    return fetch_poll(ret);
-  }
-  else if (f->type == fetch_CURL && f->status == fetch_STATUS_WAIT)
-  {
-    struct browser_message msg;
-    CURL* curl;
-
-    LOG(("init curl"));
-    curl = curl_easy_init();
-    LOG(("init curl returned"));
-    if (curl != 0)
-    {
-      LOG(("init curl OK"));
-      /* shouldn't assume this!  somehow work it out instead. */
-      msg.type = msg_FETCH_FETCH_INFO;
-      msg.f = f;
-      msg.data.fetch_info.type = type_HTML;
-      msg.data.fetch_info.total_size = -1;
-
-      if (browser_window_message(f->request->requestor.browser, &msg) == 0)
-      {
-        LOG(("about to set options"));
-        curl_easy_setopt(curl, CURLOPT_URL, f->location);
-        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fetch_curl_data);
-        curl_easy_setopt(curl, CURLOPT_WRITEDATA, f);
-        curl_easy_setopt(curl, CURLOPT_USERAGENT, "NetSurf/0.00 (alpha)");
-        LOG(("about to perform"));
-        curl_easy_perform(curl);
-        LOG(("about to cleanup"));
-        curl_easy_cleanup(curl);
-
-        LOG(("cleanup finished"));
-        msg.type = msg_FETCH_FINISHED;
-        msg.f = f;
-        LOG(("sending FETCH_FINISHED to browser"));
-        browser_window_message(f->request->requestor.browser, &msg);
-        LOG(("FETCH_FINISHED accepted"));
-
-        ret = f->next;
-        LOG(("Destroying f"));
-        fetch_destroy(f);
-        LOG(("Moving on..."));
-        return fetch_poll(ret);
-      }
-      LOG(("about to cleanup since requestor went funny"));
-      curl_easy_cleanup(curl);
-
-      LOG(("Requesting browser didn't like something"));
-      ret = f->next;
-      LOG(("Cancelling fetch"));
-      f = fetch_cancel(f);
-      return fetch_poll(ret);
-    }
-
-    LOG(("we are aborting the mission"));
-    msg.type = msg_FETCH_ABORT;
-    msg.f = f;
-    browser_window_message(f->request->requestor.browser, &msg);
-    LOG(("ABORT message sent to browser"));
-
-    ret = f->next;
-    fetch_destroy(f);
-    return fetch_poll(ret); /* carry on polling */
-  }
-
-  LOG(("Moving on (at end of function with f->next)"));
-  f->next = fetch_poll(f->next);
-  return f;
-}
-
diff --git a/desktop/fetch.h b/desktop/fetch.h
deleted file mode 100644
index f9366c8fa..000000000
--- a/desktop/fetch.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * $Id: fetch.h,v 1.2 2003/01/06 00:04:43 bursa Exp $
- */
-
-#ifndef _NETSURF_DESKTOP_FETCH_H_
-#define _NETSURF_DESKTOP_FETCH_H_
-
-#include "libxml/HTMLparser.h"
-#include "netsurf/render/box.h"
-#include "netsurf/render/css.h"
-#include "netsurf/desktop/browser.h"
-#include <time.h>
-
-typedef enum {fetch_CURL} fetch_type;
-typedef enum {fetch_STATUS_SEND, fetch_STATUS_WAIT, fetch_STATUS_FETCH, fetch_STATUS_FINISH, fetch_DELETED} fetch_status;
-
-typedef int fetch_flags;
-#define fetch_DO_NOT_CHECK_CACHE       ((fetch_flags) 1);
-#define fetch_DO_NOT_STORE_IN_CACHE    ((fetch_flags) 2);
-
-struct fetch_request {
-  enum {REQUEST_FROM_BROWSER} type;
-  union {struct browser_window* browser;} requestor;
-};
-
-struct fetch
-{
-  char* location;
-  fetch_type type;
-  fetch_flags flags;
-
-  fetch_status status;
-  int bytes_fetched;
-  int bytes_total;
-
-  struct fetch_request* request;
-
-  time_t start_time;
-
-  struct fetch* next;
-};
-
-struct fetch* create_fetch(char* location, char* previous, fetch_flags f, struct fetch_request* r);
-void fetch_destroy(struct fetch* f);
-struct fetch* fetch_cancel(struct fetch* f);
-void fetch_receive(struct fetch* f, int amount, char* bytes);
-struct fetch* fetch_poll(struct fetch* f);
-
-#endif
diff --git a/desktop/netsurf.c b/desktop/netsurf.c
index 144d9fa7d..f8161c975 100644
--- a/desktop/netsurf.c
+++ b/desktop/netsurf.c
@@ -1,23 +1,23 @@
 /**
- * $Id: netsurf.c,v 1.5 2002/12/29 22:27:35 monkeyson Exp $
+ * $Id: netsurf.c,v 1.6 2003/02/09 12:58:15 bursa Exp $
  */
 
 #include "netsurf/desktop/netsurf.h"
-#include "netsurf/desktop/fetch.h"
 #include "netsurf/desktop/browser.h"
 #include "netsurf/desktop/gui.h"
-#include "netsurf/desktop/cache.h"
+#include "netsurf/content/cache.h"
+#include "netsurf/content/fetch.h"
+#include "netsurf/utils/log.h"
 #include <stdlib.h>
 
 int netsurf_quit = 0;
 gui_window* netsurf_gui_windows = NULL;
-struct fetch* netsurf_fetches = NULL;
 
 
 void netsurf_poll(void)
 {
   gui_poll();
-  netsurf_fetches = fetch_poll(netsurf_fetches);
+  fetch_poll();
 }
 
 
@@ -25,6 +25,7 @@ void netsurf_init(int argc, char** argv)
 {
   stdout = stderr;
   gui_init(argc, argv);
+  fetch_init();
   cache_init();
 }
 
@@ -32,6 +33,7 @@ void netsurf_init(int argc, char** argv)
 void netsurf_exit(void)
 {
   cache_quit();
+  fetch_quit();
 }
 
 
@@ -42,21 +44,10 @@ int main(int argc, char** argv)
   while (netsurf_quit == 0)
     netsurf_poll();
 
-  fprintf(stderr, "Netsurf quit!\n");
+  LOG(("Netsurf quit!"));
   netsurf_exit();
 
   return 0;
 }
 
 
-void Log(char* func, char* msg)
-{
-#ifdef NETSURF_DUMP_MONKEYS
-  FILE* logfile = NULL;
-  logfile = fopen("logfile","a");
-  if (logfile == NULL)
-    die("can't open logfile");
-  fprintf(logfile, "%s: %s\n", func, msg);
-  fclose(logfile);
-#endif
-}
diff --git a/desktop/netsurf.h b/desktop/netsurf.h
index 1ebdf9d88..7acab2537 100644
--- a/desktop/netsurf.h
+++ b/desktop/netsurf.h
@@ -1,35 +1,16 @@
 /**
- * $Id: netsurf.h,v 1.1 2002/09/11 14:24:02 monkeyson Exp $
+ * $Id: netsurf.h,v 1.2 2003/02/09 12:58:15 bursa Exp $
  */
 
 #ifndef _NETSURF_DESKTOP_NETSURF_H_
 #define _NETSURF_DESKTOP_NETSURF_H_
 
-#include "netsurf/desktop/fetch.h"
 #include "netsurf/desktop/browser.h"
 
-extern struct fetch* netsurf_fetches;
 extern gui_window* netsurf_gui_windows;
-
 extern int netsurf_quit;
 
 void netsurf_poll(void);
-void Log(char* func, char* msg);
 
 #endif
 
-#ifndef _NETSURF_DESKTOP_NETSURF_H_
-#define _NETSURF_DESKTOP_NETSURF_H_
-
-#include "netsurf/desktop/fetch.h"
-#include "netsurf/desktop/browser.h"
-
-extern struct fetch* netsurf_fetches;
-extern gui_window* netsurf_gui_windows;
-
-extern int netsurf_quit;
-
-void netsurf_poll(void);
-void Log(char* func, char* msg);
-
-#endif
diff --git a/makefile b/makefile
index 80dadc638..1a9dbaaec 100644
--- a/makefile
+++ b/makefile
@@ -1,37 +1,41 @@
-# $Id: makefile,v 1.12 2003/01/02 13:13:27 bursa Exp $
+# $Id: makefile,v 1.13 2003/02/09 12:58:14 bursa Exp $
 
 all: !NetSurf/!RunImage,ff8
 clean:
 	rm */arm-riscos-aof/*
 
-setup: render/arm-riscos-aof riscos/arm-riscos-aof desktop/arm-riscos-aof
+setup: render/arm-riscos-aof riscos/arm-riscos-aof desktop/arm-riscos-aof \
+ content/arm-riscos-aof utils/arm-riscos-aof
 %/arm-riscos-aof:
 	mkdir $@
 
-FLAGS = -g -Wall -W -Wundef -Wpointer-arith -Wbad-function-cast -Wcast-qual \
+FLAGS = -Wall -W -Wundef -Wpointer-arith -Wbad-function-cast -Wcast-qual \
  -Wcast-align -Wwrite-strings -Wconversion -Wstrict-prototypes -Wmissing-prototypes \
- -Wmissing-declarations -Wredundant-decls -Wnested-externs -Winline -std=c9x \
+ -Wmissing-declarations -Wredundant-decls -Wnested-externs -Winline \
  -I.. -I/usr/local/riscoslibs/include \
  -Dfd_set=long -mpoke-function-name -DNETSURF_DUMP
 CC = riscos-gcc
-OBJECTS = render/arm-riscos-aof/utils.o render/arm-riscos-aof/css.o \
- render/arm-riscos-aof/css_enum.o render/arm-riscos-aof/box.o \
- render/arm-riscos-aof/layout.o \
- riscos/arm-riscos-aof/gui.o riscos/arm-riscos-aof/font.o \
- riscos/arm-riscos-aof/theme.o \
- desktop/arm-riscos-aof/browser.o desktop/arm-riscos-aof/fetch.o \
- desktop/arm-riscos-aof/netsurf.o desktop/arm-riscos-aof/cache.o
-HEADERS = render/box.h render/css.h render/css_enum.h \
- render/layout.h render/utils.h riscos/font.h riscos/gui.h \
- riscos/theme.h \
- desktop/browser.h desktop/fetch.h desktop/gui.h desktop/netsurf.h \
- desktop/cache.h
+OBJECTS = \
+ content/arm-riscos-aof/cache.o  content/arm-riscos-aof/content.o \
+ content/arm-riscos-aof/fetch.o  content/arm-riscos-aof/fetchcache.o \
+ desktop/arm-riscos-aof/browser.o  desktop/arm-riscos-aof/netsurf.o \
+ render/arm-riscos-aof/box.o  render/arm-riscos-aof/css.o \
+ render/arm-riscos-aof/css_enum.o  render/arm-riscos-aof/html.o \
+ render/arm-riscos-aof/layout.o render/arm-riscos-aof/textplain.o \
+ riscos/arm-riscos-aof/font.o  riscos/arm-riscos-aof/gui.o  riscos/arm-riscos-aof/theme.o \
+ utils/arm-riscos-aof/utils.o
+HEADERS = \
+ content/cache.h    content/content.h  content/fetch.h    content/fetchcache.h \
+ desktop/browser.h  desktop/gui.h      desktop/netsurf.h  render/box.h \
+ render/css.h       render/css_enum.h  render/html.h      render/layout.h \
+ riscos/font.h      riscos/gui.h       riscos/theme.h     utils/log.h \
+ utils/utils.h      render/textplain.h
 LIBS = \
  /usr/local/riscoslibs/libxml2/libxml2.ro \
  /usr/local/riscoslibs/OSLib/OSLib.ro \
  /usr/local/riscoslibs/curl/libcurl.ro \
- /usr/local/riscoslibs/libutf-8/libutf-8.ro \
- /usr/local/riscoslibs/ubiqx/ubiqx.ro
+ /usr/local/riscoslibs/libutf-8/libutf-8.ro
+# /usr/local/riscoslibs/ubiqx/ubiqx.ro
 
 !NetSurf/!RunImage,ff8: $(OBJECTS)
 	$(CC) $(FLAGS) -o !NetSurf/!RunImage,ff8 $(OBJECTS) $(LIBS)
@@ -48,6 +52,12 @@ riscos/arm-riscos-aof/%.o: riscos/%.c $(HEADERS)
 desktop/arm-riscos-aof/%.o: desktop/%.c $(HEADERS) 
 	$(CC) $(FLAGS) -o $@ -c $<
 
+content/arm-riscos-aof/%.o: content/%.c $(HEADERS) 
+	$(CC) $(FLAGS) -o $@ -c $<
+
+utils/arm-riscos-aof/%.o: utils/%.c $(HEADERS) 
+	$(CC) $(FLAGS) -o $@ -c $<
+
 netsurf.zip: !NetSurf/!RunImage,ff8
 	rm netsurf.zip; riscos-zip -9vr, netsurf.zip !NetSurf
 
diff --git a/render/box.c b/render/box.c
index a5a4b525d..1a07be7ba 100644
--- a/render/box.c
+++ b/render/box.c
@@ -1,5 +1,5 @@
 /**
- * $Id: box.c,v 1.32 2003/01/11 17:36:40 monkeyson Exp $
+ * $Id: box.c,v 1.33 2003/02/09 12:58:15 bursa Exp $
  */
 
 #include <assert.h>
@@ -12,7 +12,7 @@
 #include "netsurf/render/css.h"
 #include "netsurf/riscos/font.h"
 #include "netsurf/render/box.h"
-#include "netsurf/render/utils.h"
+#include "netsurf/utils/utils.h"
 #include "netsurf/utils/log.h"
 #include "netsurf/desktop/gui.h"
 
@@ -24,13 +24,13 @@ struct box* input(xmlNode * n, struct css_style* style, struct form* current_for
 
 void box_add_child(struct box * parent, struct box * child);
 struct box * box_create(xmlNode * node, box_type type, struct css_style * style,
-		const char *href);
-char * tolat1(const xmlChar * s);
+		char *href);
+char * tolat1(xmlChar * s);
 struct box * convert_xml_to_box(xmlNode * n, struct css_style * parent_style,
 		struct css_stylesheet * stylesheet,
 		struct css_selector ** selector, unsigned int depth,
 		struct box * parent, struct box * inline_container,
-		const char *href, struct font_set *fonts,
+		char *href, struct font_set *fonts,
 		struct gui_gadget* current_select, struct formoption* current_option,
 		struct gui_gadget* current_textarea, struct form* current_form,
 		struct page_elements* elements);
@@ -43,7 +43,7 @@ void box_normalise_table_row(struct box *row);
 void box_normalise_inline_container(struct box *cont);
 void gadget_free(struct gui_gadget* g);
 void box_free_box(struct box *box);
-struct box* box_image(xmlNode * n, struct css_style* style, const char* href);
+struct box* box_image(xmlNode * n, struct css_style* style, char* href);
 struct box* box_textarea(xmlNode* n, struct css_style* style, struct form* current_form);
 struct box* box_select(xmlNode * n, struct css_style* style, struct form* current_form);
 struct formoption* box_option(xmlNode* n, struct css_style* style, struct gui_gadget* current_select);
@@ -79,7 +79,7 @@ void box_add_child(struct box * parent, struct box * child)
  */
 
 struct box * box_create(xmlNode * node, box_type type, struct css_style * style,
-		const char *href)
+		char *href)
 {
 	struct box * box = xcalloc(1, sizeof(struct box));
 	box->type = type;
@@ -106,14 +106,14 @@ struct box * box_create(xmlNode * node, box_type type, struct css_style * style,
 }
 
 
-char * tolat1(const xmlChar * s)
+char * tolat1(xmlChar * s)
 {
-	char *d = xcalloc(strlen((const char*)s) + 1, sizeof(char));
+	char *d = xcalloc(strlen((char*) s) + 1, sizeof(char));
 	char *d0 = d;
 	unsigned int u, chars;
 
 	while (*s != 0) {
-		u = sgetu8((const unsigned char*)s, (int*) &chars);
+		u = sgetu8((unsigned char*) s, (int*) &chars);
 		s += chars;
 		if (u == 0x09 || u == 0x0a || u == 0x0d)
 			*d = ' ';
@@ -148,7 +148,7 @@ void xml_to_box(xmlNode * n, struct css_style * parent_style,
 		struct css_stylesheet * stylesheet,
 		struct css_selector ** selector, unsigned int depth,
 		struct box * parent, struct box * inline_container,
-		const char *href, struct font_set *fonts,
+		char *href, struct font_set *fonts,
 		struct gui_gadget* current_select, struct formoption* current_option,
 		struct gui_gadget* current_textarea, struct form* current_form,
 		struct page_elements* elements)
@@ -165,7 +165,7 @@ struct box * convert_xml_to_box(xmlNode * n, struct css_style * parent_style,
 		struct css_stylesheet * stylesheet,
 		struct css_selector ** selector, unsigned int depth,
 		struct box * parent, struct box * inline_container,
-		const char *href, struct font_set *fonts,
+		char *href, struct font_set *fonts,
 		struct gui_gadget* current_select, struct formoption* current_option,
 		struct gui_gadget* current_textarea, struct form* current_form,
 		struct page_elements* elements)
@@ -972,7 +972,7 @@ void box_free_box(struct box *box)
 	}
 }
 
-struct box* box_image(xmlNode * n, struct css_style* style, const char* href)
+struct box* box_image(xmlNode * n, struct css_style* style, char* href)
 {
 	struct box* box = 0;
 	char* s;
diff --git a/render/box.h b/render/box.h
index aad278f44..78d78d456 100644
--- a/render/box.h
+++ b/render/box.h
@@ -1,5 +1,5 @@
 /**
- * $Id: box.h,v 1.18 2003/01/11 17:36:40 monkeyson Exp $
+ * $Id: box.h,v 1.19 2003/02/09 12:58:15 bursa Exp $
  */
 
 #ifndef _NETSURF_RENDER_BOX_H_
@@ -85,9 +85,9 @@ struct box {
 	struct css_style * style;
 	unsigned long x, y, width, height;
 	unsigned long min_width, max_width;
-	const char * text;
+	char * text;
 	unsigned int space : 1;	/* 1 <=> followed by a space */
-	const char * href;
+	char * href;
 	unsigned int length;
 	unsigned int columns;
 	struct box * next;
@@ -136,7 +136,7 @@ struct page_elements
 void xml_to_box(xmlNode * n, struct css_style * parent_style, struct css_stylesheet * stylesheet,
 		struct css_selector ** selector, unsigned int depth,
 		struct box * parent, struct box * inline_container,
-		const char *href, struct font_set *fonts,
+		char *href, struct font_set *fonts,
 		struct gui_gadget* current_select, struct formoption* current_option,
 		struct gui_gadget* current_textarea, struct form* current_form,
 		struct page_elements* elements);
diff --git a/render/css.c b/render/css.c
index 8cb983b78..e4a9002b5 100644
--- a/render/css.c
+++ b/render/css.c
@@ -1,12 +1,12 @@
 /**
- * $Id: css.c,v 1.12 2002/12/27 20:42:31 bursa Exp $
+ * $Id: css.c,v 1.13 2003/02/09 12:58:15 bursa Exp $
  */
 
 #include <string.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include "css.h"
-#include "utils.h"
+#include "netsurf/render/css.h"
+#include "netsurf/utils/utils.h"
 
 /**
  * internal structures
@@ -704,9 +704,9 @@ int main(int argv, char *argc[])
 	css_parse_stylesheet(s, argc[1]);
 	css_dump_stylesheet(s);
 
-/* 	r->selectors = 1; */
-/* 	css_stylesheet_add_rule(s, r); */
-/*	puts("********** finding h1:");
+ 	r->selectors = 1;
+ 	css_stylesheet_add_rule(s, r);
+	puts("********** finding h1:");
 	r = find_rule(s, &sel, 1);
 	if (r)
 		dump_rule(r);
@@ -719,9 +719,9 @@ int main(int argv, char *argc[])
 		css_dump_style(style);
 	}
 
-/* 	fprintf(stderr, "%x %x\n", r, r2); */
+ 	fprintf(stderr, "%x %x\n", r, r2); 
 
-/*	struct css_style *s;
+	struct css_style *s;
 	struct css_selector *sel;
 
 	s = parse_property_list(argc[1]);
diff --git a/render/html.c b/render/html.c
new file mode 100644
index 000000000..f500d3405
--- /dev/null
+++ b/render/html.c
@@ -0,0 +1,156 @@
+/**
+ * $Id: html.c,v 1.1 2003/02/09 12:58:15 bursa Exp $
+ */
+
+#include <assert.h>
+#include <string.h>
+#include <stdlib.h>
+#include "netsurf/desktop/gui.h"
+#include "netsurf/render/html.h"
+#include "netsurf/render/layout.h"
+#include "netsurf/utils/utils.h"
+#include "netsurf/utils/log.h"
+
+
+static void html_title(struct content *c);
+
+
+void html_create(struct content *c)
+{
+	c->data.html.parser = htmlCreatePushParserCtxt(0, 0, "", 0, 0, XML_CHAR_ENCODING_8859_1);
+	c->data.html.document = NULL;
+	c->data.html.markup = NULL;
+	c->data.html.layout = NULL;
+	c->data.html.stylesheet = NULL;
+	c->data.html.style = NULL;
+	c->data.html.fonts = NULL;
+}
+
+
+#define CHUNK 4096
+
+void html_process_data(struct content *c, char *data, unsigned long size)
+{
+	unsigned long x;
+	for (x = 0; x < size; x += CHUNK) {
+		htmlParseChunk(c->data.html.parser, data + x, CHUNK, 0);
+		gui_multitask();
+	}
+}
+
+
+int html_convert(struct content *c, unsigned int width, unsigned int height)
+{
+  char* file;
+  struct css_selector* selector = xcalloc(1, sizeof(struct css_selector));
+
+	htmlParseChunk(c->data.html.parser, "", 0, 1);
+	c->data.html.document = c->data.html.parser->myDoc;
+	/*xmlDebugDumpDocument(stderr, c->data.html.parser->myDoc);*/
+
+	LOG(("Skipping to html"));
+	if (c->data.html.document == NULL) {
+		LOG(("There is no document!"));
+		return 1;
+	}
+	for (c->data.html.markup = c->data.html.document->children;
+			c->data.html.markup != 0 && c->data.html.markup->type != XML_ELEMENT_NODE;
+			c->data.html.markup = c->data.html.markup->next)
+		;
+
+	if (c->data.html.markup == 0) {
+		LOG(("No markup"));
+		return 1;
+	}
+	if (stricmp((const char *) c->data.html.markup->name, "html")) {
+		LOG(("Not html"));
+		return 1;
+	}
+	
+	html_title(c);
+
+	/* TODO: rewrite stylesheet handling */
+  LOG(("Loading CSS"));
+  file = load("<NetSurf$Dir>.Resources.CSS");  /*!!! not portable! !!!*/
+  c->data.html.stylesheet = css_new_stylesheet();
+  LOG(("Parsing stylesheet"));
+  css_parse_stylesheet(c->data.html.stylesheet, file);
+
+  LOG(("Copying base style"));
+  c->data.html.style = xcalloc(1, sizeof(struct css_style));
+  memcpy(c->data.html.style, &css_base_style, sizeof(struct css_style));
+
+	LOG(("Creating box"));
+	c->data.html.layout = xcalloc(1, sizeof(struct box));
+	c->data.html.layout->type = BOX_BLOCK;
+	c->data.html.layout->node = c->data.html.markup;
+
+	c->data.html.fonts = font_new_set();
+
+	LOG(("XML to box"));
+	xml_to_box(c->data.html.markup, c->data.html.style, c->data.html.stylesheet,
+			&selector, 0, c->data.html.layout, 0, 0, c->data.html.fonts,
+			0, 0, 0, 0, &c->data.html.elements);
+	box_dump(c->data.html.layout->children, 0);
+
+	LOG(("Layout document"));
+	layout_document(c->data.html.layout->children, width);
+	box_dump(c->data.html.layout->children, 0);
+
+	return 0;
+}
+
+
+void html_title(struct content *c)
+{
+	xmlNode *node = c->data.html.markup;
+
+	c->title = 0;
+
+	while (node != 0) {
+		if (node->type == XML_ELEMENT_NODE) {
+			if (stricmp(node->name, "html") == 0) {
+				node = node->children;
+				continue;
+			}
+			if (stricmp(node->name, "head") == 0) {
+				node = node->children;
+				continue;
+			}
+			if (stricmp(node->name, "title") == 0) {
+				c->title = xmlNodeGetContent(node);
+				return;
+			}
+		}
+		node = node->next;
+	}
+}
+
+
+void html_revive(struct content *c, unsigned int width, unsigned int height)
+{
+	/* TODO: reload stylesheets and images and fix any pointers to them */
+	layout_document(c->data.html.layout->children, width);
+}
+
+
+void html_reformat(struct content *c, unsigned int width, unsigned int height)
+{
+	layout_document(c->data.html.layout->children, width);
+}
+
+
+void html_destroy(struct content *c)
+{
+	LOG(("content %p", c));
+
+	htmlFreeParserCtxt(c->data.html.parser);
+
+	if (c->data.html.document != 0)
+		xmlFreeDoc(c->data.html.document);
+	if (c->data.html.layout != 0)
+		box_free(c->data.html.layout);
+	if (c->data.html.fonts != 0)
+		font_free_set(c->data.html.fonts);
+	/* TODO: stylesheets */
+}
diff --git a/render/html.h b/render/html.h
new file mode 100644
index 000000000..1dd9e3a51
--- /dev/null
+++ b/render/html.h
@@ -0,0 +1,17 @@
+/**
+ * $Id: html.h,v 1.1 2003/02/09 12:58:15 bursa Exp $
+ */
+
+#ifndef _NETSURF_RENDER_HTML_H_
+#define _NETSURF_RENDER_HTML_H_
+
+#include "netsurf/content/content.h"
+
+void html_create(struct content *c);
+void html_process_data(struct content *c, char *data, unsigned long size);
+int html_convert(struct content *c, unsigned int width, unsigned int height);
+void html_revive(struct content *c, unsigned int width, unsigned int height);
+void html_reformat(struct content *c, unsigned int width, unsigned int height);
+void html_destroy(struct content *c);
+
+#endif
diff --git a/render/layout.c b/render/layout.c
index a27fdbb30..afc8c25df 100644
--- a/render/layout.c
+++ b/render/layout.c
@@ -1,5 +1,5 @@
 /**
- * $Id: layout.c,v 1.32 2003/01/06 23:53:40 bursa Exp $
+ * $Id: layout.c,v 1.33 2003/02/09 12:58:15 bursa Exp $
  */
 
 #include <assert.h>
@@ -8,11 +8,12 @@
 #include <stdlib.h>
 #include <string.h>
 #include "libxml/HTMLparser.h"
-#include "netsurf/render/css.h"
-#include "netsurf/riscos/font.h"
+#include "netsurf/desktop/gui.h"
 #include "netsurf/render/box.h"
-#include "netsurf/render/utils.h"
+#include "netsurf/render/css.h"
 #include "netsurf/render/layout.h"
+#include "netsurf/riscos/font.h"
+#include "netsurf/utils/utils.h"
 #include "netsurf/utils/log.h"
 
 #define DEBUG_LAYOUT
@@ -441,14 +442,14 @@ struct box * layout_line(struct box * first, unsigned long width, unsigned long
 	if (x1 - x0 < x) {
 		/* the last box went over the end */
 		char * space = 0;
-		unsigned long w;
+		unsigned int w;
 		struct box * c2;
 
 		x = x_previous;
 
 		if (c->text != 0)
 			space = strchr(c->text, ' ');
-		if (space != 0 && c->length <= space - c->text)
+		if (space != 0 && c->length <= (unsigned int) (space - c->text))
 			/* space after end of string */
 			space = 0;
 
@@ -457,7 +458,7 @@ struct box * layout_line(struct box * first, unsigned long width, unsigned long
 		if (space == 0)
 			w = c->width;
 		else
-			w = font_width(c->font, c->text, space - c->text);
+			w = font_width(c->font, c->text, (unsigned int) (space - c->text));
 
 		if (x1 - x0 < x + space_before + w && left == 0 && right == 0 && c == first) {
 			/* first word doesn't fit, but no floats and first on line so force in */
@@ -488,8 +489,8 @@ struct box * layout_line(struct box * first, unsigned long width, unsigned long
 			assert(space != 0);
 			space = font_split(c->font, c->text, c->length,
 					x1 - x0 - x - space_before, &w);
-			LOG(("'%.*s' %lu %lu (%c) %lu", c->length, c->text,
-					x1 - x0, space - c->text, *space, w));
+			LOG(("'%.*s' %lu %u (%c) %u", (int) c->length, c->text,
+					(unsigned long) (x1 - x0), space - c->text, *space, w));
 			c2 = memcpy(xcalloc(1, sizeof(struct box)), c, sizeof(struct box));
 			c2->text = xstrdup(space + 1);
 			c2->length = c->length - ((space + 1) - c->text);
@@ -759,7 +760,7 @@ void calculate_inline_container_widths(struct box *box)
 							word = space + 1,
 							space = strchr(word, ' ')) {
 						width = font_width(child->font, word,
-								space - word);
+								(unsigned int) (space - word));
 						if (min < width) min = width;
 					}
 					width = font_width(child->font, word, strlen(word));
diff --git a/render/textplain.c b/render/textplain.c
new file mode 100644
index 000000000..c5407c6f5
--- /dev/null
+++ b/render/textplain.c
@@ -0,0 +1,52 @@
+/**
+ * $Id: textplain.c,v 1.1 2003/02/09 12:58:15 bursa Exp $
+ */
+
+#include <assert.h>
+#include <string.h>
+#include <stdlib.h>
+#include "libxml/HTMLparser.h"
+#include "netsurf/render/html.h"
+#include "netsurf/render/textplain.h"
+
+
+static const char header[] = "<html><body><pre>";
+static const char footer[] = "</pre></body></html>";
+
+
+void textplain_create(struct content *c)
+{
+	html_create(c);
+	htmlParseChunk(c->data.html.parser, header, sizeof(header), 0);
+}
+
+
+void textplain_process_data(struct content *c, char *data, unsigned long size)
+{
+	html_process_data(c, data, size);
+}
+
+
+int textplain_convert(struct content *c, unsigned int width, unsigned int height)
+{
+	htmlParseChunk(c->data.html.parser, footer, sizeof(footer), 0);
+	return html_convert(c, width, height);
+}
+
+
+void textplain_revive(struct content *c, unsigned int width, unsigned int height)
+{
+	html_revive(c, width, height);
+}
+
+
+void textplain_reformat(struct content *c, unsigned int width, unsigned int height)
+{
+	html_reformat(c, width, height);
+}
+
+
+void textplain_destroy(struct content *c)
+{
+	html_destroy(c);
+}
diff --git a/render/textplain.h b/render/textplain.h
new file mode 100644
index 000000000..7221ab0b8
--- /dev/null
+++ b/render/textplain.h
@@ -0,0 +1,17 @@
+/**
+ * $Id: textplain.h,v 1.1 2003/02/09 12:58:15 bursa Exp $
+ */
+
+#ifndef _NETSURF_RENDER_TEXTPLAIN_H_
+#define _NETSURF_RENDER_TEXTPLAIN_H_
+
+#include "netsurf/content/content.h"
+
+void textplain_create(struct content *c);
+void textplain_process_data(struct content *c, char *data, unsigned long size);
+int textplain_convert(struct content *c, unsigned int width, unsigned int height);
+void textplain_revive(struct content *c, unsigned int width, unsigned int height);
+void textplain_reformat(struct content *c, unsigned int width, unsigned int height);
+void textplain_destroy(struct content *c);
+
+#endif
diff --git a/riscos/font.c b/riscos/font.c
index 966e24688..8c0d6de44 100644
--- a/riscos/font.c
+++ b/riscos/font.c
@@ -1,5 +1,5 @@
 /**
- * $Id: font.c,v 1.9 2002/12/29 22:27:35 monkeyson Exp $
+ * $Id: font.c,v 1.10 2003/02/09 12:58:15 bursa Exp $
  */
 
 #include <assert.h>
@@ -7,7 +7,7 @@
 #include "libutf-8/utf-8.h"
 #include "netsurf/render/css.h"
 #include "netsurf/riscos/font.h"
-#include "netsurf/render/utils.h"
+#include "netsurf/utils/utils.h"
 #include "netsurf/desktop/gui.h"
 #include "netsurf/utils/log.h"
 #include "oslib/font.h"
diff --git a/riscos/gui.c b/riscos/gui.c
index 8c046192d..6ebcf5197 100644
--- a/riscos/gui.c
+++ b/riscos/gui.c
@@ -1,10 +1,10 @@
 /**
- * $Id: gui.c,v 1.16 2003/01/12 17:48:44 bursa Exp $
+ * $Id: gui.c,v 1.17 2003/02/09 12:58:15 bursa Exp $
  */
 
 #include "netsurf/riscos/font.h"
 #include "netsurf/desktop/gui.h"
-#include "netsurf/render/utils.h"
+#include "netsurf/utils/utils.h"
 #include "netsurf/desktop/netsurf.h"
 #include "oslib/osfile.h"
 #include "oslib/os.h"
@@ -14,6 +14,7 @@
 #include "netsurf/utils/log.h"
 #include <string.h>
 #include <stdlib.h>
+#include <time.h>
 #include <math.h>
 
 int gadget_subtract_x;
diff --git a/riscos/theme.c b/riscos/theme.c
index b6b70a711..806255ff8 100644
--- a/riscos/theme.c
+++ b/riscos/theme.c
@@ -4,7 +4,7 @@
 #include "oslib/messagetrans.h"
 #include "oslib/osspriteop.h"
 #include "string.h"
-#include "netsurf/render/utils.h"
+#include "netsurf/utils/utils.h"
 #include "stdio.h"
 
 void ro_theme_preload_template(ro_theme* theme, char* template_name,
diff --git a/utils/log.h b/utils/log.h
index 4ca658419..b1bebde34 100644
--- a/utils/log.h
+++ b/utils/log.h
@@ -1,5 +1,5 @@
 /**
- * $Id: log.h,v 1.1 2002/10/08 09:38:29 bursa Exp $
+ * $Id: log.h,v 1.2 2003/02/09 12:58:15 bursa Exp $
  */
 
 #include <stdio.h>
@@ -14,7 +14,7 @@
 #else
 
 #ifdef __GNUC__
-#define LOG(x) (printf(__FILE__ " " __PRETTY_FUNCTION__ " %i: ", __LINE__), printf x, printf("\n"))
+#define LOG(x) (printf(__FILE__ " %s %i: ", __PRETTY_FUNCTION__, __LINE__), printf x, printf("\n"))
 #else
 #define LOG(x) (printf(__FILE__ " %i: ", __LINE__), printf x, printf("\n"))
 #endif
diff --git a/utils/utils.c b/utils/utils.c
index 923c16e26..8604d9ed4 100644
--- a/utils/utils.c
+++ b/utils/utils.c
@@ -1,12 +1,12 @@
 /**
- * $Id: utils.c,v 1.5 2002/09/11 14:24:02 monkeyson Exp $
+ * $Id: utils.c,v 1.6 2003/02/09 12:58:15 bursa Exp $
  */
 
 #include <ctype.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
-#include "utils.h"
+#include "netsurf/utils/utils.h"
 
 void die(const char * const error)
 {
diff --git a/utils/utils.h b/utils/utils.h
index c7c608cf2..5ae167eb9 100644
--- a/utils/utils.h
+++ b/utils/utils.h
@@ -1,9 +1,9 @@
 /**
- * $Id: utils.h,v 1.4 2003/01/07 23:15:23 bursa Exp $
+ * $Id: utils.h,v 1.5 2003/02/09 12:58:15 bursa Exp $
  */
 
-#ifndef _NETSURF_RENDER_UTILS_H_
-#define _NETSURF_RENDER_UTILS_H_
+#ifndef _NETSURF_UTILS_UTILS_H_
+#define _NETSURF_UTILS_UTILS_H_
 
 #include <stdlib.h>
 
-- 
cgit v1.2.3