hoist the Referer header generation logic up to the low level cache

This removes the need for the fetchers to have any interaction with
 the Referer header. It has not been completely removed from the
 fetch interface as fetch.c:fetch_set_cookie() still uses it for
 unverifiable cookie decision logic. (There is an anchient todo here)

5 files changed, 152 insertions, 104 deletions

diff --git a/content/fetch.c b/content/fetch.c
index 533f75e87..1fcc3f996 100644
--- a/content/fetch.c
+++ b/content/fetch.c
@@ -90,7 +90,6 @@ struct fetch {
 	fetch_callback callback;/**< Callback function. */
 	nsurl *url;		/**< URL. */
 	nsurl *referer;		/**< Referer URL. */
-	bool send_referer;	/**< Valid to send the referer */
 	bool verifiable;	/**< Transaction is verifiable */
 	void *p;		/**< Private data for callback. */
 	lwc_string *host;	/**< Host part of URL, interned */
@@ -461,7 +460,6 @@ fetch_start(nsurl *url,
 {
 	struct fetch *fetch;
 	lwc_string *scheme;
-	bool match;
 
 	fetch = calloc(1, sizeof (*fetch));
 	if (fetch == NULL) {
@@ -474,8 +472,8 @@ fetch_start(nsurl *url,
 
 	/* try and obtain a fetcher for this scheme */
 	fetch->fetcherd = get_fetcher_for_scheme(scheme);
+	lwc_string_unref(scheme);
 	if (fetch->fetcherd == -1) {
-		lwc_string_unref(scheme);
 		free(fetch);
 		return NSERROR_NO_FETCH_HANDLER;
 	}
@@ -490,48 +488,9 @@ fetch_start(nsurl *url,
 	fetch->host = nsurl_get_component(url, NSURL_HOST);
 
 	if (referer != NULL) {
-		lwc_string *ref_scheme;
 		fetch->referer = nsurl_ref(referer);
-
-		ref_scheme = nsurl_get_component(referer, NSURL_SCHEME);
-		/* Not a problem if referer has no scheme */
-
-		/* Determine whether to send the Referer header */
-		if (nsoption_bool(send_referer) && ref_scheme != NULL) {
-			/* User permits us to send the header
-			 * Only send it if:
-			 *    1) The fetch and referer schemes match
-			 * or 2) The fetch is https and the referer is http
-			 *
-			 * This ensures that referer information is only sent
-			 * across schemes in the special case of an https
-			 * request from a page served over http. The inverse
-			 * (https -> http) should not send the referer (15.1.3)
-			 */
-			bool match1;
-			bool match2;
-			if (lwc_string_isequal(scheme, ref_scheme,
-					       &match) != lwc_error_ok) {
-				match = false;
-			}
-			if (lwc_string_isequal(scheme, corestring_lwc_https,
-					       &match1) != lwc_error_ok) {
-				match1 = false;
-			}
-			if (lwc_string_isequal(ref_scheme, corestring_lwc_http,
-					       &match2) != lwc_error_ok) {
-				match2= false;
-			}
-			if (match == true || (match1 == true && match2 == true))
-				fetch->send_referer = true;
-		}
-		if (ref_scheme != NULL)
-			lwc_string_unref(ref_scheme);
 	}
 
-	/* these aren't needed past here */
-	lwc_string_unref(scheme);
-
 	/* try and set up the fetch */
 	fetch->fetcher_handle = fetchers[fetch->fetcherd].ops.setup(fetch, url,
 						only_2xx, downgrade_tls,
@@ -839,13 +798,6 @@ void fetch_set_http_code(struct fetch *fetch, long http_code)
 	fetch->http_code = http_code;
 }
 
-/* exported interface documented in content/fetch.h */
-const char *fetch_get_referer_to_send(struct fetch *fetch)
-{
-	if (fetch->send_referer)
-		return nsurl_access(fetch->referer);
-	return NULL;
-}
 
 /* exported interface documented in content/fetch.h */
 void fetch_set_cookie(struct fetch *fetch, const char *data)
diff --git a/content/fetch.h b/content/fetch.h
index fdb3bfbd8..843fec96e 100644
--- a/content/fetch.h
+++ b/content/fetch.h
@@ -230,11 +230,6 @@ void fetch_free(struct fetch *f);
 void fetch_set_http_code(struct fetch *fetch, long http_code);
 
 /**
- * get the referer from the fetch
- */
-const char *fetch_get_referer_to_send(struct fetch *fetch);
-
-/**
  * set cookie data on a fetch
  */
 void fetch_set_cookie(struct fetch *fetch, const char *data);
diff --git a/content/fetchers/curl.c b/content/fetchers/curl.c
index f9cafae69..d36f44c09 100644
--- a/content/fetchers/curl.c
+++ b/content/fetchers/curl.c
@@ -889,7 +889,6 @@ static CURLcode fetch_curl_set_options(struct curl_fetch_info *f)
 	SETOPT(CURLOPT_WRITEDATA, f);
 	SETOPT(CURLOPT_WRITEHEADER, f);
 	SETOPT(CURLOPT_PROGRESSDATA, f);
-	SETOPT(CURLOPT_REFERER, fetch_get_referer_to_send(f->fetch_handle));
 	SETOPT(CURLOPT_HTTPHEADER, f->headers);
 	if (f->post_urlenc) {
 		SETOPT(CURLOPT_HTTPPOST, NULL);
diff --git a/content/hlcache.c b/content/hlcache.c
index 2e15edd56..d860015a5 100644
--- a/content/hlcache.c
+++ b/content/hlcache.c
@@ -673,11 +673,15 @@ void hlcache_finalise(void)
 }
 
 /* See hlcache.h for documentation */
-nserror hlcache_handle_retrieve(nsurl *url, uint32_t flags,
-		nsurl *referer, llcache_post_data *post,
-		hlcache_handle_callback cb, void *pw,
-		hlcache_child_context *child,
-		content_type accepted_types, hlcache_handle **result)
+nserror
+hlcache_handle_retrieve(nsurl *url,
+			uint32_t flags,
+			nsurl *referer,
+			llcache_post_data *post,
+			hlcache_handle_callback cb, void *pw,
+			hlcache_child_context *child,
+			content_type accepted_types,
+			hlcache_handle **result)
 {
 	hlcache_retrieval_ctx *ctx;
 	nserror error;
diff --git a/content/llcache.c b/content/llcache.c
index c1ddea54c..81e08383c 100644
--- a/content/llcache.c
+++ b/content/llcache.c
@@ -47,6 +47,7 @@
 #include "utils/utils.h"
 #include "utils/time.h"
 #include "utils/http.h"
+#include "utils/nsoption.h"
 #include "netsurf/misc.h"
 #include "desktop/gui_internal.h"
 
@@ -807,6 +808,87 @@ static nserror llcache_fetch_process_header(llcache_object *object,
 }
 
 /**
+ * construct a Referer header appropriate for the request
+ *
+ * \param url The url being navigated to
+ * \param referer The referring url
+ * \param header_out A pointer to receive the header. The buffer must
+ *                    be freed by the caller.
+ * \return NSERROR_OK and \a header_out updated on success else error code
+ */
+static nserror get_referer_header(nsurl *url, nsurl *referer, char **header_out)
+{
+	nserror res = NSERROR_INVALID;
+	lwc_string *ref_scheme;
+	lwc_string *scheme;
+	bool match;
+	bool match1;
+	bool match2;
+	char *header;
+
+	/* Determine whether to send the Referer header */
+	if (!nsoption_bool(send_referer)) {
+		return NSERROR_INVALID;
+	}
+
+	scheme = nsurl_get_component(url, NSURL_SCHEME);
+	if (scheme == NULL) {
+		return NSERROR_BAD_URL;
+	}
+
+	ref_scheme = nsurl_get_component(referer, NSURL_SCHEME);
+	if (ref_scheme == NULL) {
+		/* referer has no scheme so no header */
+		lwc_string_unref(scheme);
+		return NSERROR_INVALID;
+	}
+
+	/* User permits us to send the header
+	 * Only send it if:
+	 *    1) The fetch and referer schemes match
+	 * or 2) The fetch is https and the referer is http
+	 *
+	 * This ensures that referer information is only sent
+	 * across schemes in the special case of an https
+	 * request from a page served over http. The inverse
+	 * (https -> http) should not send the referer (15.1.3)
+	 */
+	if (lwc_string_isequal(scheme, ref_scheme,
+			       &match) != lwc_error_ok) {
+		match = false;
+	}
+	if (lwc_string_isequal(scheme, corestring_lwc_https,
+			       &match1) != lwc_error_ok) {
+		match1 = false;
+	}
+	if (lwc_string_isequal(ref_scheme, corestring_lwc_http,
+			       &match2) != lwc_error_ok) {
+		match2 = false;
+	}
+	if (match == true || (match1 == true && match2 == true)) {
+		const size_t len = SLEN("Referer: ") +
+			nsurl_length(referer) + 1;
+
+		header = malloc(len);
+		if (header == NULL) {
+			res = NSERROR_NOMEM;
+		} else {
+			snprintf(header, len, "Referer: %s",
+				 nsurl_access(referer));
+
+			*header_out = header;
+			res = NSERROR_OK;
+		}
+	}
+
+
+	lwc_string_unref(scheme);
+	lwc_string_unref(ref_scheme);
+
+	return res;
+}
+
+/**
  * (Re)fetch an object
  *
  * Sets up headers and attempts to start an actual fetch from the
@@ -834,12 +916,13 @@ static nserror llcache_object_refetch(llcache_object *object)
 		}
 	}
 
-	/* Generate cache-control headers */
-	headers = malloc(3 * sizeof(char *));
+	/* Generate headers */
+	headers = malloc(4 * sizeof(char *));
 	if (headers == NULL) {
 		return NSERROR_NOMEM;
 	}
 
+	/* cache-control header for etag */
 	if (object->cache.etag != NULL) {
 		const size_t len = SLEN("If-None-Match: ") +
 				strlen(object->cache.etag) + 1;
@@ -856,6 +939,7 @@ static nserror llcache_object_refetch(llcache_object *object)
 		header_idx++;
 	}
 
+	/* cache-control header for modification time */
 	if (object->cache.last_modified != 0) {
 		/* Maximum length of an RFC 1123 date is 29 bytes */
 		const size_t len = SLEN("If-Modified-Since: ") + 29 + 1;
@@ -873,6 +957,15 @@ static nserror llcache_object_refetch(llcache_object *object)
 
 		header_idx++;
 	}
+
+	/* Referer header */
+	if (object->fetch.referer != NULL) {
+		if (get_referer_header(object->url,
+				       object->fetch.referer,
+				       &headers[header_idx]) == NSERROR_OK) {
+			header_idx++;
+		}
+	}
 	headers[header_idx] = NULL;
 
 	/* Reset cache control data */
@@ -3571,6 +3664,46 @@ total_object_size(llcache_object *object)
 	return tot;
 }
 
+/**
+ * Catch up the cache users with state changes from fetchers.
+ *
+ * \param ignored We ignore this because all our state comes from llcache.
+ */
+static void llcache_catch_up_all_users(void *ignored)
+{
+	llcache_object *object;
+
+	/* Assume after this we'll be all caught up.  If any user of a handle
+	 * defers then we'll invalidate all_caught_up and reschedule via
+	 * llcache_users_not_caught_up()
+	 */
+	llcache->all_caught_up = true;
+
+	/* Catch new users up with state of objects */
+	for (object = llcache->cached_objects; object != NULL;
+			object = object->next) {
+		llcache_object_notify_users(object);
+	}
+
+	for (object = llcache->uncached_objects; object != NULL;
+			object = object->next) {
+		llcache_object_notify_users(object);
+	}
+}
+
+/**
+ * Ask for ::llcache_catch_up_all_users to be scheduled ASAP to pump the
+ * user state machines.
+ */
+static void llcache_users_not_caught_up(void)
+{
+	if (llcache->all_caught_up) {
+		llcache->all_caught_up = false;
+		guit->misc->schedule(0, llcache_catch_up_all_users, NULL);
+	}
+}
+
+
 /******************************************************************************
  * Public API								      *
  ******************************************************************************/
@@ -3843,51 +3976,16 @@ void llcache_finalise(void)
 	llcache = NULL;
 }
 
-/**
- * Catch up the cache users with state changes from fetchers.
- *
- * \param ignored We ignore this because all our state comes from llcache.
- */
-static void llcache_catch_up_all_users(void *ignored)
-{
-	llcache_object *object;
-
-	/* Assume after this we'll be all caught up.  If any user of a handle
-	 * defers then we'll invalidate all_caught_up and reschedule via
-	 * llcache_users_not_caught_up()
-	 */
-	llcache->all_caught_up = true;
-
-	/* Catch new users up with state of objects */
-	for (object = llcache->cached_objects; object != NULL;
-			object = object->next) {
-		llcache_object_notify_users(object);
-	}
-
-	for (object = llcache->uncached_objects; object != NULL;
-			object = object->next) {
-		llcache_object_notify_users(object);
-	}
-}
-
-/**
- * Ask for ::llcache_catch_up_all_users to be scheduled ASAP to pump the
- * user state machines.
- */
-static void llcache_users_not_caught_up(void)
-{
-	if (llcache->all_caught_up) {
-		llcache->all_caught_up = false;
-		guit->misc->schedule(0, llcache_catch_up_all_users, NULL);
-	}
-}
 
 
 /* Exported interface documented in content/llcache.h */
-nserror llcache_handle_retrieve(nsurl *url, uint32_t flags,
-		nsurl *referer, const llcache_post_data *post,
-		llcache_handle_callback cb, void *pw,
-		llcache_handle **result)
+nserror
+llcache_handle_retrieve(nsurl *url,
+			uint32_t flags,
+			nsurl *referer,
+			const llcache_post_data *post,
+			llcache_handle_callback cb, void *pw,
+			llcache_handle **result)
 {
 	nserror error;
 	llcache_object_user *user;