diff options
Diffstat (limited to 'content/llcache.c')
| -rw-r--r-- | content/llcache.c | 1815 |
1 files changed, 1815 insertions, 0 deletions
diff --git a/content/llcache.c b/content/llcache.c new file mode 100644 index 000000000..4d2a7f0b5 --- /dev/null +++ b/content/llcache.c @@ -0,0 +1,1815 @@ +/* + * Copyright 2009 John-Mark Bell <jmb@netsurf-browser.org> + * + * This file is part of NetSurf, http://www.netsurf-browser.org/ + * + * NetSurf is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * NetSurf is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +/** \file + * Low-level resource cache (implementation) + */ + +#define _GNU_SOURCE /* For strndup. Ugh. */ +#include <stdlib.h> +#include <string.h> +#include <time.h> + +#include <curl/curl.h> + +#include "content/fetch.h" +#include "content/llcache.h" +#include "utils/messages.h" +#include "utils/url.h" +#include "utils/utils.h" + +/** State of a low-level cache object fetch */ +typedef enum { + LLCACHE_FETCH_INIT, /**< Initial state, before fetch */ + LLCACHE_FETCH_HEADERS, /**< Fetching headers */ + LLCACHE_FETCH_DATA, /**< Fetching object data */ + LLCACHE_FETCH_COMPLETE /**< Fetch completed */ +} llcache_fetch_state; + +/** Type of low-level cache object */ +typedef struct llcache_object llcache_object; + +/** Handle to low-level cache object */ +struct llcache_handle { + llcache_object *object; /**< Pointer to associated object */ + + llcache_handle_callback cb; /**< Client callback */ + void *pw; /**< Client data */ + + llcache_fetch_state state; /**< Last known state of object fetch */ + size_t bytes; /**< Last reported byte count */ +}; + +/** Low-level cache object user record */ +typedef struct llcache_object_user { + /* Must be first in struct */ + llcache_handle handle; /**< Handle data for client */ + + bool iterator_target; /**< This is the an iterator target */ + bool queued_for_delete; /**< This user is queued for deletion */ + + struct llcache_object_user *prev; /**< Previous in list */ + struct llcache_object_user *next; /**< Next in list */ +} llcache_object_user; + +/** Low-level cache object fetch context */ +typedef struct { + uint32_t flags; /**< Fetch flags */ + char *referer; /**< Referring URL, or NULL if none */ + llcache_post_data *post; /**< POST data, or NULL for GET */ + + struct fetch *fetch; /**< Fetch handle for this object */ + + llcache_fetch_state state; /**< Current state of object fetch */ +} llcache_fetch_ctx; + +/** Cache control data */ +typedef struct { + time_t req_time; /**< Time of request */ + time_t res_time; /**< Time of response */ + time_t date; /**< Date: response header */ + time_t expires; /**< Expires: response header */ +#define INVALID_AGE -1 + int age; /**< Age: response header */ + int max_age; /**< Max-Age Cache-control parameter */ + bool no_cache; /**< No-Cache Cache-control parameter */ + char *etag; /**< Etag: response header */ + time_t last_modified; /**< Last-Modified: response header */ +} llcache_cache_control; + +/** Representation of a fetch header */ +typedef struct { + char *name; /**< Header name */ + char *value; /**< Header value */ +} llcache_header; + +/** Low-level cache object */ +/** \todo Consider whether a list is a sane container */ +struct llcache_object { + llcache_object *prev; /**< Previous in list */ + llcache_object *next; /**< Next in list */ + + char *url; /**< Post-redirect URL for object */ + + /** \todo We need a generic dynamic buffer object */ + uint8_t *source_data; /**< Source data for object */ + size_t source_len; /**< Byte length of source data */ + size_t source_alloc; /**< Allocated size of source buffer */ + + llcache_object_user *users; /**< List of users */ + + llcache_fetch_ctx fetch; /**< Fetch context for object */ + + llcache_cache_control cache; /**< Cache control data for object */ + llcache_object *candidate; /**< Object to use, if fetch determines + * that it is still fresh */ + uint32_t candidate_count; /**< Count of objects this is a + * candidate for */ + + llcache_header *headers; /**< Fetch headers */ + size_t num_headers; /**< Number of fetch headers */ +}; + +/** Handler for fetch-related queries */ +static llcache_query_callback query_cb; +/** Data for fetch-related query handler */ +static void *query_cb_pw; + +/** Head of the low-level cached object list */ +static llcache_object *llcache_cached_objects; +/** Head of the low-level uncached object list */ +static llcache_object *llcache_uncached_objects; + +static nserror llcache_object_user_new(llcache_handle_callback cb, void *pw, + llcache_object_user **user); +static nserror llcache_object_user_destroy(llcache_object_user *user); + +static nserror llcache_object_retrieve(const char *url, uint32_t flags, + const char *referer, const llcache_post_data *post, + llcache_object **result); +static nserror llcache_object_retrieve_from_cache(const char *url, + uint32_t flags, const char *referer, + const llcache_post_data *post, llcache_object **result); +static bool llcache_object_is_fresh(const llcache_object *object); +static nserror llcache_object_cache_update(llcache_object *object); +static nserror llcache_object_clone_cache_data(const llcache_object *source, + llcache_object *destination, bool deep); +static nserror llcache_object_fetch(llcache_object *object, uint32_t flags, + const char *referer, const llcache_post_data *post); +static nserror llcache_object_refetch(llcache_object *object); + +static nserror llcache_object_new(const char *url, llcache_object **result); +static nserror llcache_object_destroy(llcache_object *object); +static nserror llcache_object_add_user(llcache_object *object, + llcache_object_user *user); +static nserror llcache_object_remove_user(llcache_object *object, + llcache_object_user *user); + +static nserror llcache_object_add_to_list(llcache_object *object, + llcache_object **list); +static nserror llcache_object_remove_from_list(llcache_object *object, + llcache_object **list); + +static nserror llcache_object_notify_users(llcache_object *object); + +static nserror llcache_clean(void); + +static nserror llcache_post_data_clone(const llcache_post_data *orig, + llcache_post_data **clone); + +static nserror llcache_query_handle_response(bool proceed, void *cbpw); + +static void llcache_fetch_callback(fetch_msg msg, void *p, const void *data, + unsigned long size, fetch_error_code errorcode); +static nserror llcache_fetch_redirect(llcache_object *object, + const char *target, llcache_object **replacement); +static nserror llcache_fetch_notmodified(llcache_object *object, + llcache_object **replacement); +static nserror llcache_fetch_split_header(const char *data, size_t len, + char **name, char **value); +static nserror llcache_fetch_parse_header(llcache_object *object, + const char *data, size_t len, char **name, char **value); +static nserror llcache_fetch_process_header(llcache_object *object, + const char *data, size_t len); +static nserror llcache_fetch_process_data(llcache_object *object, + const uint8_t *data, size_t len); +static nserror llcache_fetch_auth(llcache_object *object, + const char *realm); +static nserror llcache_fetch_cert_error(llcache_object *object, + const struct ssl_cert_info *certs, size_t num); + + +/****************************************************************************** + * Public API * + ******************************************************************************/ + +/** + * Initialise the low-level cache + * + * \param cb Query handler + * \param pw Pointer to query handler data + * \return NSERROR_OK on success, appropriate error otherwise. + */ +nserror llcache_initialise(llcache_query_callback cb, void *pw) +{ + query_cb = cb; + query_cb_pw = pw; + + return NSERROR_OK; +} + +/** + * Poll the low-level cache + * + * \return NSERROR_OK on success, appropriate error otherwise. + */ +nserror llcache_poll(void) +{ + llcache_object *object; + + /* Catch new users up with state of objects */ + for (object = llcache_cached_objects; object != NULL; + object = object->next) { + llcache_object_notify_users(object); + } + + for (object = llcache_uncached_objects; object != NULL; + object = object->next) { + llcache_object_notify_users(object); + } + + /* Attempt to clean the cache */ + llcache_clean(); + + return NSERROR_OK; +} + +/** + * Retrieve a handle for a low-level cache object + * + * \param url URL of the object to fetch + * \param flags Object retrieval flags + * \param referer Referring URL, or NULL if none + * \param post POST data, or NULL for a GET request + * \param cb Client callback for events + * \param pw Pointer to client-specific data + * \param result Pointer to location to recieve cache handle + * \return NSERROR_OK on success, appropriate error otherwise + */ +nserror llcache_handle_retrieve(const char *url, uint32_t flags, + const char *referer, const llcache_post_data *post, + llcache_handle_callback cb, void *pw, + llcache_handle **result) +{ + nserror error; + llcache_object_user *user; + llcache_object *object; + + /* Can we fetch this URL at all? */ + if (fetch_can_fetch(url) == false) + return NSERROR_NO_FETCH_HANDLER; + + /* Create a new object user */ + error = llcache_object_user_new(cb, pw, &user); + if (error != NSERROR_OK) + return error; + + /* Retrieve a suitable object from the cache, + * creating a new one if needed. */ + error = llcache_object_retrieve(url, flags, referer, post, &object); + if (error != NSERROR_OK) { + llcache_object_user_destroy(user); + return error; + } + + /* Add user to object */ + llcache_object_add_user(object, user); + + *result = &user->handle; + + return NSERROR_OK; +} + +/** + * Change the callback associated with a low-level cache handle + * + * \param handle Handle to change callback of + * \param cb New callback + * \param pw Client data for new callback + * \return NSERROR_OK on success, appropriate error otherwise + */ +nserror llcache_handle_change_callback(llcache_handle *handle, + llcache_handle_callback cb, void *pw) +{ + handle->cb = cb; + handle->pw = pw; + + return NSERROR_OK; +} + +/** + * Release a low-level cache handle + * + * \param handle Handle to release + * \return NSERROR_OK on success, appropriate error otherwise + */ +nserror llcache_handle_release(llcache_handle *handle) +{ + nserror error = NSERROR_OK; + llcache_object *object = handle->object; + llcache_object_user *user = (llcache_object_user *) handle; + + /* Remove the user from the object and destroy it */ + error = llcache_object_remove_user(object, user); + if (error == NSERROR_OK) { + /* Can't delete user object if it's the target of an iterator */ + if (user->iterator_target) + user->queued_for_delete = true; + else + error = llcache_object_user_destroy(user); + } + + return error; +} + +/** + * Retrieve the post-redirect URL of a low-level cache object + * + * \param handle Handle to retrieve URL from + * \return Post-redirect URL of cache object + */ +const char *llcache_handle_get_url(const llcache_handle *handle) +{ + return handle->object != NULL ? handle->object->url : NULL; +} + +/** + * Retrieve source data of a low-level cache object + * + * \param handle Handle to retrieve source data from + * \param size Pointer to location to receive byte length of data + * \return Pointer to source data + */ +const uint8_t *llcache_handle_get_source_data(const llcache_handle *handle, + size_t *size) +{ + *size = handle->object != NULL ? handle->object->source_len : 0; + + return handle->object != NULL ? handle->object->source_data : NULL; +} + +/** + * Retrieve a header value associated with a low-level cache object + * + * \param handle Handle to retrieve header from + * \param key Header name + * \return Header value, or NULL if header does not exist + * + * \todo Make the key an enumeration, to avoid needless string comparisons + * \todo Forcing the client to parse the header value seems wrong. + * Better would be to return the actual value part and an array of + * key-value pairs for any additional parameters. + */ +const char *llcache_handle_get_header(const llcache_handle *handle, + const char *key) +{ + const llcache_object *object = handle->object; + size_t i; + + if (object == NULL) + return NULL; + + /* About as trivial as possible */ + for (i = 0; i < object->num_headers; i++) { + if (strcasecmp(key, object->headers[i].name) == 0) + return object->headers[i].value; + } + + return NULL; +} + +/** + * Determine if the same underlying object is referenced by the given handles + * + * \param a First handle + * \param b Second handle + * \return True if handles reference the same object, false otherwise + */ +bool llcache_handle_references_same_object(const llcache_handle *a, + const llcache_handle *b) +{ + return a->object == b->object; +} + +/****************************************************************************** + * Low-level cache internals * + ******************************************************************************/ + +/** + * Create a new object user + * + * \param cb Callback routine + * \param pw Private data for callback + * \param user Pointer to location to receive result + * \return NSERROR_OK on success, appropriate error otherwise + */ +nserror llcache_object_user_new(llcache_handle_callback cb, void *pw, + llcache_object_user **user) +{ + llcache_object_user *u = calloc(1, sizeof(llcache_object_user)); + if (u == NULL) + return NSERROR_NOMEM; + + u->handle.cb = cb; + u->handle.pw = pw; + + *user = u; + + return NSERROR_OK; +} + +/** + * Destroy an object user + * + * \param user User to destroy + * \return NSERROR_OK on success, appropriate error otherwise + * + * \pre User is not attached to an object + */ +nserror llcache_object_user_destroy(llcache_object_user *user) +{ + free(user); + + return NSERROR_OK; +} + +/** + * Retrieve an object from the cache, fetching it if necessary. + * + * \param url URL of object to retrieve + * \param flags Fetch flags + * \param referer Referring URL, or NULL if none + * \param post POST data, or NULL for a GET request + * \param result Pointer to location to recieve retrieved object + * \return NSERROR_OK on success, appropriate error otherwise + */ +nserror llcache_object_retrieve(const char *url, uint32_t flags, + const char *referer, const llcache_post_data *post, + llcache_object **result) +{ + nserror error; + llcache_object *obj; + bool has_query; + url_func_result res; + struct url_components components; + + /** + * Caching Rules: + * + * 1) Forced fetches are never cached + * 2) GET requests with query segments are never cached + * 3) POST requests are never cached + * + * \todo Find out if restriction (2) can be removed + */ + + /* Look for a query segment */ + res = url_get_components(url, &components); + if (res == URL_FUNC_NOMEM) + return NSERROR_NOMEM; + + has_query = (components.query != NULL); + + url_destroy_components(&components); + + if (flags & LLCACHE_RETRIEVE_FORCE_FETCH || has_query || post != NULL) { + /* Create new object */ + error = llcache_object_new(url, &obj); + if (error != NSERROR_OK) + return error; + + /* Attempt to kick-off fetch */ + error = llcache_object_fetch(obj, flags, referer, post); + if (error != NSERROR_OK) { + llcache_object_destroy(obj); + return error; + } + + /* Add new object to uncached list */ + llcache_object_add_to_list(obj, &llcache_uncached_objects); + } else { + error = llcache_object_retrieve_from_cache(url, flags, referer, + post, &obj); + if (error != NSERROR_OK) + return error; + + /* Returned object is already in the cached list */ + } + + *result = obj; + + return NSERROR_OK; +} + +/** + * Retrieve a potentially cached object + * + * \param url URL of object to retrieve + * \param flags Fetch flags + * \param referer Referring URL, or NULL if none + * \param post POST data, or NULL for a GET request + * \param result Pointer to location to recieve retrieved object + * \return NSERROR_OK on success, appropriate error otherwise + */ +nserror llcache_object_retrieve_from_cache(const char *url, uint32_t flags, + const char *referer, const llcache_post_data *post, + llcache_object **result) +{ + nserror error; + llcache_object *obj, *newest = NULL; + + /* Search for the most recently fetched matching object */ + for (obj = llcache_cached_objects; obj != NULL; obj = obj->next) { + if (strcasecmp(obj->url, url) == 0 && (newest == NULL || + obj->cache.req_time > newest->cache.req_time)) + newest = obj; + } + + if (newest != NULL && llcache_object_is_fresh(newest)) { + /* Found a suitable object, and it's still fresh, so use it */ + obj = newest; + + /* The client needs to catch up with the object's state. + * This will occur the next time that llcache_poll is called. + */ + } else if (newest != NULL) { + /* Found a candidate object but it needs freshness validation */ + /* Create a new object */ + error = llcache_object_new(url, &obj); + if (error != NSERROR_OK) + return error; + + /* Clone candidate's cache data */ + error = llcache_object_clone_cache_data(newest, obj, true); + if (error != NSERROR_OK) { + llcache_object_destroy(obj); + return error; + } + + /* Record candidate, so we can fall back if it is still fresh */ + newest->candidate_count++; + obj->candidate = newest; + + /* Attempt to kick-off fetch */ + error = llcache_object_fetch(obj, flags, referer, post); + if (error != NSERROR_OK) { + newest->candidate_count--; + llcache_object_destroy(obj); + return error; + } + + /* Add new object to cache */ + llcache_object_add_to_list(obj, &llcache_cached_objects); + } else { + /* No object found; create a new one */ + /* Create new object */ + error = llcache_object_new(url, &obj); + if (error != NSERROR_OK) + return error; + + /* Attempt to kick-off fetch */ + error = llcache_object_fetch(obj, flags, referer, post); + if (error != NSERROR_OK) { + llcache_object_destroy(obj); + return error; + } + + /* Add new object to cache */ + llcache_object_add_to_list(obj, &llcache_cached_objects); + } + + *result = obj; + + return NSERROR_OK; +} + +/** + * Determine if an object is still fresh + * + * \param object Object to consider + * \return True if object is still fresh, false otherwise + */ +bool llcache_object_is_fresh(const llcache_object *object) +{ + const llcache_cache_control *cd = &object->cache; + int current_age, freshness_lifetime; + time_t now = time(NULL); + + /* Calculate staleness of cached object as per RFC 2616 13.2.3/13.2.4 */ + current_age = max(0, (cd->res_time - cd->date)); + current_age = max(current_age, (cd->age == INVALID_AGE) ? 0 : cd->age); + current_age += cd->res_time - cd->req_time + now - cd->res_time; + + /* Determine freshness lifetime of this object */ + if (cd->max_age != INVALID_AGE) + freshness_lifetime = cd->max_age; + else if (cd->expires != 0) + freshness_lifetime = cd->expires - cd->date; + else if (cd->last_modified != 0) + freshness_lifetime = (now - cd->last_modified) / 10; + else + freshness_lifetime = 0; + + /* The object is fresh if its current age is within the freshness + * lifetime or if we're still fetching the object */ + return (freshness_lifetime > current_age || + object->fetch.state != LLCACHE_FETCH_COMPLETE); +} + +/** + * Update an object's cache state + * + * \param object Object to update cache for + * \return NSERROR_OK. + */ +nserror llcache_object_cache_update(llcache_object *object) +{ + if (object->cache.date == 0) + object->cache.date = time(NULL); + + /** \todo Any magic we need to do for no_cache? */ + + return NSERROR_OK; +} + +/** + * Clone an object's cache data + * + * \param source Source object containing cache data to clone + * \param destination Destination object to clone cache data into + * \param deep Whether to deep-copy the data or not + * \return NSERROR_OK on success, appropriate error otherwise + */ +nserror llcache_object_clone_cache_data(const llcache_object *source, + llcache_object *destination, bool deep) +{ + /* ETag must be first, as it can fail when deep cloning */ + if (source->cache.etag != NULL) { + char *etag = source->cache.etag; + + if (deep) { + /* Copy the etag */ + etag = strdup(source->cache.etag); + if (etag == NULL) + return NSERROR_NOMEM; + } + + if (destination->cache.etag != NULL) + free(destination->cache.etag); + + destination->cache.etag = etag; + } + + destination->cache.req_time = source->cache.req_time; + destination->cache.res_time = source->cache.res_time; + + if (source->cache.date != 0) + destination->cache.date = source->cache.date; + + if (source->cache.expires != 0) + destination->cache.expires = source->cache.expires; + + if (source->cache.age != INVALID_AGE) + destination->cache.age = source->cache.age; + + if (source->cache.max_age != INVALID_AGE) + destination->cache.max_age = source->cache.max_age; + + if (source->cache.no_cache) + destination->cache.no_cache = source->cache.no_cache; + + if (source->cache.last_modified != 0) + destination->cache.last_modified = source->cache.last_modified; + + return NSERROR_OK; +} + +/** + * Kick-off a fetch for an object + * + * \param object Object to fetch + * \param flags Fetch flags + * \param referer Referring URL, or NULL for none + * \param post POST data, or NULL for GET + * \return NSERROR_OK on success, appropriate error otherwise + * + * \pre object::url must contain the URL to fetch + * \pre If there is a freshness validation candidate, + * object::candidate and object::cache must be filled in + * \pre There must not be a fetch in progress for \a object + */ +nserror llcache_object_fetch(llcache_object *object, uint32_t flags, + const char *referer, const llcache_post_data *post) +{ + nserror error; + char *referer_clone = NULL; + llcache_post_data *post_clone = NULL; + + if (referer != NULL) { + referer_clone = strdup(referer); + if (referer_clone == NULL) + return NSERROR_NOMEM; + } + + if (post != NULL) { + error = llcache_post_data_clone(post, &post_clone); + if (error != NSERROR_OK) { + free(referer_clone); + return error; + } + } + + object->fetch.flags = flags; + object->fetch.referer = referer_clone; + object->fetch.post = post_clone; + + return llcache_object_refetch(object); +} + +/** + * (Re)fetch an object + * + * \param object Object to refetch + * \return NSERROR_OK on success, appropriate error otherwise + * + * \pre The fetch parameters in object->fetch must be populated + */ +nserror llcache_object_refetch(llcache_object *object) +{ + const char *urlenc = NULL; + /** \todo Why is fetch_start's post_multipart parameter not const? */ + struct fetch_multipart_data *multipart = NULL; + /** \todo Why is the headers parameter of fetch_start not const? */ + char **headers = NULL; + int header_idx = 0; + + if (object->fetch.post != NULL) { + if (object->fetch.post->type == LLCACHE_POST_URL_ENCODED) + urlenc = object->fetch.post->data.urlenc; + else + multipart = object->fetch.post->data.multipart; + } + + /* Generate cache-control headers */ + headers = malloc(3 * sizeof(char *)); + if (headers == NULL) + return NSERROR_NOMEM; + + if (object->cache.etag != NULL) { + const size_t len = SLEN("If-None-Match: ") + + strlen(object->cache.etag) + 1; + + headers[header_idx] = malloc(len); + if (headers[header_idx] == NULL) { + free(headers); + return NSERROR_NOMEM; + } + + snprintf(headers[header_idx], len, "If-None-Match: %s", + object->cache.etag); + + header_idx++; + } + if (object->cache.date != 0) { + /* Maximum length of an RFC 1123 date is 29 bytes */ + const size_t len = SLEN("If-Modified-Since: ") + 29 + 1; + + headers[header_idx] = malloc(len); + if (headers[header_idx] == NULL) { + while (--header_idx >= 0) + free(headers[header_idx]); + free(headers); + return NSERROR_NOMEM; + } + + snprintf(headers[header_idx], len, "If-Modified-Since: %s", + rfc1123_date(object->cache.date)); + + header_idx++; + } + headers[header_idx] = NULL; + + /* Reset cache control data */ + object->cache.req_time = time(NULL); + object->cache.res_time = 0; + object->cache.date = 0; + object->cache.expires = 0; + object->cache.age = INVALID_AGE; + object->cache.max_age = INVALID_AGE; + object->cache.no_cache = false; + free(object->cache.etag); + object->cache.etag = NULL; + object->cache.last_modified = 0; + + /* Kick off fetch */ + object->fetch.fetch = fetch_start(object->url, object->fetch.referer, + llcache_fetch_callback, object, + object->fetch.flags & LLCACHE_RETRIEVE_NO_ERROR_PAGES, + urlenc, multipart, + object->fetch.flags & LLCACHE_RETRIEVE_VERIFIABLE, + NULL, /** \todo Remove parent from this API */ + headers); + + /* Clean up cache-control headers */ + while (--header_idx >= 0) + free(headers[header_idx]); + free(headers); + + /* Did we succeed in creating a fetch? */ + if (object->fetch.fetch == NULL) + return NSERROR_NOMEM; + + return NSERROR_OK; +} + +/** + * Create a new low-level cache object + * + * \param url URL of object to create + * \param result Pointer to location to receive result + * \return NSERROR_OK on success, appropriate error otherwise + */ +nserror llcache_object_new(const char *url, llcache_object **result) +{ + llcache_object *obj = calloc(1, sizeof(llcache_object)); + if (obj == NULL) + return NSERROR_NOMEM; + + obj->url = strdup(url); + if (obj->url == NULL) { + free(obj); + return NSERROR_NOMEM; + } + + *result = obj; + + return NSERROR_OK; +} + +/** + * Destroy a low-level cache object + * + * \param object Object to destroy + * \return NSERROR_OK on success, appropriate error otherwise + * + * \pre Object is detached from cache list + * \pre Object has no users + * \pre Object is not a candidate (i.e. object::candidate_count == 0) + */ +nserror llcache_object_destroy(llcache_object *object) +{ + size_t i; + + free(object->url); + free(object->source_data); + + if (object->fetch.fetch != NULL) { + fetch_abort(object->fetch.fetch); + object->fetch.fetch = NULL; + } + + free(object->fetch.referer); + + if (object->fetch.post != NULL) { + if (object->fetch.post->type == LLCACHE_POST_URL_ENCODED) { + free(object->fetch.post->data.urlenc); + } else { + fetch_multipart_data_destroy( + object->fetch.post->data.multipart); + } + + free(object->fetch.post); + } + + free(object->cache.etag); + + for (i = 0; i < object->num_headers; i++) { + free(object->headers[i].name); + free(object->headers[i].value); + } + free(object->headers); + + free(object); + + return NSERROR_OK; +} + +/** + * Add a user to a low-level cache object + * + * \param object Object to add user to + * \param user User to add + * \return NSERROR_OK. + */ +nserror llcache_object_add_user(llcache_object *object, + llcache_object_user *user) +{ + user->handle.object = object; + + user->prev = NULL; + user->next = object->users; + + if (object->users != NULL) + object->users->prev = user; + object->users = user; + + return NSERROR_OK; +} + +/** + * Remove a user from a low-level cache object + * + * \param object Object to remove user from + * \param user User to remove + * \return NSERROR_OK. + */ +nserror llcache_object_remove_user(llcache_object *object, + llcache_object_user *user) +{ + if (user == object->users) + object->users = user->next; + else + user->prev->next = user->next; + + if (user->next != NULL) + user->next->prev = user->prev; + + return NSERROR_OK; +} + +/** + * Add a low-level cache object to a cache list + * + * \param object Object to add + * \param list List to add to + * \return NSERROR_OK + */ +nserror llcache_object_add_to_list(llcache_object *object, + llcache_object **list) +{ + object->prev = NULL; + object->next = *list; + + if (*list != NULL) + (*list)->prev = object; + *list = object; + + return NSERROR_OK; +} + +/** + * Remove a low-level cache object from a cache list + * + * \param object Object to remove + * \param list List to remove from + * \return NSERROR_OK + */ +nserror llcache_object_remove_from_list(llcache_object *object, + llcache_object **list) +{ + if (object == *list) + *list = object->next; + else + object->prev->next = object->next; + + if (object->next != NULL) + object->next->prev = object->next; + + return NSERROR_OK; +} + +/** + * Notify users of an object's current state + * + * \param object Object to notify users about + * \return NSERROR_OK on success, appropriate error otherwise + */ +nserror llcache_object_notify_users(llcache_object *object) +{ + nserror error; + llcache_object_user *user, *next_user; + llcache_event event; + + /** + * State transitions and event emission for users. + * Rows: user state. Cols: object state. + * + * User\Obj INIT HEADERS DATA COMPLETE + * INIT - T T* T* + * HEADERS - - T T* + * DATA - - M T + * COMPLETE - - - - + * + * T => transition user to object state + * M => no transition required, but may need to emit event + * + * The transitions marked with an asterisk can be removed by moving + * the user context into the subsequent state and then reevaluating. + * + * Events are issued as follows: + * + * HAD_HEADERS: on transition from HEADERS -> DATA state + * HAD_DATA : in DATA state, whenever there's new source data + * DONE : on transition from DATA -> COMPLETE state + */ + + for (user = object->users; user != NULL; user = next_user) { + /* Emit necessary events to bring the user up-to-date */ + llcache_handle *handle = &user->handle; + llcache_fetch_state hstate = handle->state; + llcache_fetch_state objstate = object->fetch.state; + + /* Save identity of next user in case client destroys + * the user underneath us */ + user->iterator_target = true; + next_user = user->next; + + /* User: INIT, Obj: HEADERS, DATA, COMPLETE => User->HEADERS */ + if (hstate == LLCACHE_FETCH_INIT && + objstate > LLCACHE_FETCH_INIT) { + hstate = LLCACHE_FETCH_HEADERS; + } + + /* User: HEADERS, Obj: DATA, COMPLETE => User->DATA */ + if (hstate == LLCACHE_FETCH_HEADERS && + objstate > LLCACHE_FETCH_HEADERS) { + /* Emit HAD_HEADERS event */ + event.type = LLCACHE_EVENT_HAD_HEADERS; + + error = handle->cb(handle, &event, handle->pw); + if (error != NSERROR_OK) { + user->iterator_target = false; + return error; + } + + if (user->queued_for_delete) { + llcache_object_user_destroy(user); + continue; + } + + hstate = LLCACHE_FETCH_DATA; + } + + /* User: DATA, Obj: DATA, COMPLETE, more source available */ + if (hstate == LLCACHE_FETCH_DATA && + objstate >= LLCACHE_FETCH_DATA && + object->source_len > handle->bytes) { + /* Emit HAD_DATA event */ + event.type = LLCACHE_EVENT_HAD_DATA; + event.data.data.buf = + object->source_data + handle->bytes; + event.data.data.len = + object->source_len - handle->bytes; + + error = handle->cb(handle, &event, handle->pw); + if (error != NSERROR_OK) { + user->iterator_target = false; + return error; + } + + if (user->queued_for_delete) { + llcache_object_user_destroy(user); + continue; + } + + /* Update record of last byte emitted */ + handle->bytes = object->source_len; + } + + /* User: DATA, Obj: COMPLETE => User->COMPLETE */ + if (hstate == LLCACHE_FETCH_DATA && + objstate > LLCACHE_FETCH_DATA) { + /* Emit DONE event */ + event.type = LLCACHE_EVENT_DONE; + + error = handle->cb(handle, &event, handle->pw); + if (error != NSERROR_OK) { + user->iterator_target = false; + return error; + } + + if (user->queued_for_delete) { + llcache_object_user_destroy(user); + continue; + } + + hstate = LLCACHE_FETCH_COMPLETE; + } + + /* No longer the target of an iterator */ + user->iterator_target = false; + + /* Sync handle's state with reality */ + handle->state = hstate; + } + + return NSERROR_OK; +} + +/** + * Attempt to clean the cache + * + * \return NSERROR_OK. + */ +nserror llcache_clean(void) +{ + llcache_object *object, *next; + + /* Candidates for cleaning are (in order of priority): + * + * 1) Uncacheable objects with no users + * 2) Stale cacheable objects with no users or pending fetches + * 3) Fresh cacheable objects with no users or pending fetches + */ + + /* 1) Uncacheable objects with no users */ + for (object = llcache_uncached_objects; object != NULL; object = next) { + next = object->next; + + /* The candidate count of uncacheable objects is always 0 */ + if (object->users == NULL && object->candidate_count == 0) { + llcache_object_remove_from_list(object, + &llcache_uncached_objects); + llcache_object_destroy(object); + } + } + + /* 2) Stale cacheable objects with no users or pending fetches */ + for (object = llcache_cached_objects; object != NULL; object = next) { + next = object->next; + + if (object->users == NULL && object->candidate_count == 0 && + llcache_object_is_fresh(object) == false) { + llcache_object_remove_from_list(object, + &llcache_cached_objects); + llcache_object_destroy(object); + } + } + + /* 3) Fresh cacheable objects with no users or pending fetches */ + /** \todo This one only happens if the cache is too large */ + + return NSERROR_OK; +} + +/** + * Clone a POST data object + * + * \param orig Object to clone + * \param clone Pointer to location to receive clone + * \return NSERROR_OK on success, appropriate error otherwise + */ +nserror llcache_post_data_clone(const llcache_post_data *orig, + llcache_post_data **clone) +{ + llcache_post_data *post_clone; + + post_clone = calloc(1, sizeof(llcache_post_data)); + if (post_clone == NULL) + return NSERROR_NOMEM; + + post_clone->type = orig->type; + + /* Deep-copy the type-specific data */ + if (orig->type == LLCACHE_POST_URL_ENCODED) { + post_clone->data.urlenc = strdup(orig->data.urlenc); + if (post_clone->data.urlenc == NULL) { + free(post_clone); + + return NSERROR_NOMEM; + } + } else { + post_clone->data.multipart = fetch_multipart_data_clone( + orig->data.multipart); + if (post_clone->data.multipart == NULL) { + free(post_clone); + + return NSERROR_NOMEM; + } + } + + *clone = post_clone; + + return NSERROR_OK; +} + +/** + * Handle a query response + * + * \param proceed Whether to proceed with fetch + * \param cbpw Our context for query + * \return NSERROR_OK on success, appropriate error otherwise + */ +nserror llcache_query_handle_response(bool proceed, void *cbpw) +{ + nserror error; + llcache_event event; + llcache_object_user *user; + llcache_object *object = cbpw; + + /* Refetch, using existing fetch parameters, if client allows us to */ + if (proceed) + return llcache_object_refetch(object); + + /* Inform client(s) that object fetch failed */ + event.type = LLCACHE_EVENT_ERROR; + /** \todo More appropriate error message */ + event.data.msg = messages_get("FetchFailed"); + + for (user = object->users; user != NULL; user = user->next) { + error = user->handle.cb(&user->handle, &event, user->handle.pw); + if (error != NSERROR_OK) + return error; + } + + return NSERROR_OK; +} + +/** + * Handler for fetch events + * + * \param msg Type of fetch event + * \param p Our private data + * \param data Event data + * \param size Length of data in bytes + * \param errorcode Reason for fetch error + */ +void llcache_fetch_callback(fetch_msg msg, void *p, const void *data, + unsigned long size, fetch_error_code errorcode) +{ + nserror error = NSERROR_OK; + llcache_object *object = p; + llcache_object_user *user; + llcache_event event; + + switch (msg) { + /* 3xx responses */ + case FETCH_REDIRECT: + /* Request resulted in a redirect */ + error = llcache_fetch_redirect(object, data, &object); + break; + case FETCH_NOTMODIFIED: + /* Conditional request determined that cached object is fresh */ + error = llcache_fetch_notmodified(object, &object); + break; + + /* Normal 2xx state machine */ + case FETCH_HEADER: + /* Received a fetch header */ + object->fetch.state = LLCACHE_FETCH_HEADERS; + + error = llcache_fetch_process_header(object, data, size); + case FETCH_TYPE: + /** \todo Purge FETCH_TYPE completely */ + break; + case FETCH_DATA: + /* Received some data */ + object->fetch.state = LLCACHE_FETCH_DATA; + + error = llcache_fetch_process_data(object, data, size); + break; + case FETCH_FINISHED: + /* Finished fetching */ + object->fetch.state = LLCACHE_FETCH_COMPLETE; + object->fetch.fetch = NULL; + + llcache_object_cache_update(object); + break; + + /* Out-of-band information */ + case FETCH_ERROR: + /* An error occurred while fetching */ + fetch_abort(object->fetch.fetch); + object->fetch.fetch = NULL; + /** \todo Ensure this object becomes stale */ + + /** \todo Consider using errorcode for something */ + + event.type = LLCACHE_EVENT_ERROR; + event.data.msg = data; + + for (user = object->users; user != NULL; user = user->next) { + error = user->handle.cb(&user->handle, &event, + user->handle.pw); + if (error != NSERROR_OK) + break; + } + break; + case FETCH_PROGRESS: + /* Progress update */ + event.type = LLCACHE_EVENT_PROGRESS; + event.data.msg = data; + + for (user = object->users; user != NULL; user = user->next) { + error = user->handle.cb(&user->handle, &event, + user->handle.pw); + if (error != NSERROR_OK) + break; + } + break; + + /* Events requiring action */ + case FETCH_AUTH: + /* Need Authentication */ + error = llcache_fetch_auth(object, data); + break; + case FETCH_CERT_ERR: + /* Something went wrong when validating TLS certificates */ + error = llcache_fetch_cert_error(object, data, size); + break; + } + + /* Deal with any errors reported by event handlers */ + if (error != NSERROR_OK) { + /** \todo Error handling */ + if (object->fetch.fetch != NULL) { + fetch_abort(object->fetch.fetch); + object->fetch.fetch = NULL; + } + return; + } + + /* Keep users in sync with reality */ + error = llcache_object_notify_users(object); + if (error != NSERROR_OK) { + /** \todo Error handling */ + if (object->fetch.fetch != NULL) { + fetch_abort(object->fetch.fetch); + object->fetch.fetch = NULL; + } + } +} + +/** + * Handle FETCH_REDIRECT event + * + * \param object Object being redirected + * \param target Target of redirect (may be relative) + * \param replacement Pointer to location to receive replacement object + * \return NSERROR_OK on success, appropriate error otherwise + */ +nserror llcache_fetch_redirect(llcache_object *object, const char *target, + llcache_object **replacement) +{ + nserror error; + llcache_object *dest; + llcache_object_user *user, *next; + const llcache_post_data *post = object->fetch.post; + char *url, *absurl; + url_func_result result; + /* Extract HTTP response code from the fetch object */ + long http_code = fetch_http_code(object->fetch.fetch); + + /* Abort fetch for this object */ + fetch_abort(object->fetch.fetch); + object->fetch.fetch = NULL; + + /** \todo Limit redirect depth, or detect cycles */ + + /* Make target absolute */ + result = url_join(target, object->url, &absurl); + if (result != URL_FUNC_OK) { + return NSERROR_NOMEM; + } + + /* Ensure target is normalised */ + result = url_normalize(absurl, &url); + + /* No longer require absolute url */ + free(absurl); + + if (result != URL_FUNC_OK) { + return NSERROR_NOMEM; + } + + /** \todo Ensure that redirects to file:/// don't happen? */ + + /** \todo What happens if we've no way of handling this URL? */ + + /** \todo All the magical processing for the various redirect types */ + if (http_code == 301 || http_code == 302 || http_code == 303) { + /* 301, 302, 303 redirects are all unconditional GET requests */ + post = NULL; + } else { + /** \todo 300, 305, 307 */ + free(url); + return NSERROR_OK; + } + + /* Attempt to fetch target URL */ + error = llcache_object_retrieve(url, object->fetch.flags, + object->fetch.referer, object->fetch.post, + &dest); + + /* No longer require url */ + free(url); + + if (error != NSERROR_OK) + return error; + + /* Move user(s) to replacement object */ + for (user = object->users; user != NULL; user = next) { + next = user->next; + + llcache_object_remove_user(object, user); + llcache_object_add_user(dest, user); + } + + /* Dest is now our object */ + *replacement = dest; + + return NSERROR_OK; +} + +/** + * Handle FETCH_NOTMODIFIED event + * + * \param object Object to process + * \param replacement Pointer to location to receive replacement object + * \return NSERROR_OK. + */ +nserror llcache_fetch_notmodified(llcache_object *object, + llcache_object **replacement) +{ + llcache_object_user *user, *next; + + /* Move user(s) to candidate content */ + for (user = object->users; user != NULL; user = next) { + next = user->next; + + llcache_object_remove_user(object, user); + llcache_object_add_user(object->candidate, user); + } + + /* Candidate is no longer a candidate for us */ + object->candidate->candidate_count--; + + /* Clone our cache control data into the candidate */ + llcache_object_clone_cache_data(object, object->candidate, false); + /* Bring candidate's cache data up to date */ + llcache_object_cache_update(object->candidate); + + /* Invalidate our cache-control data */ + memset(&object->cache, 0, sizeof(llcache_cache_control)); + + /* Ensure fetch has stopped */ + /** \todo Are there any other fields that need invalidating? */ + fetch_abort(object->fetch.fetch); + object->fetch.fetch = NULL; + + /* Candidate is now our object */ + *replacement = object->candidate; + + /** \todo Ensure that old object gets flushed from the cache */ + + return NSERROR_OK; +} + +/** + * Split a fetch header into name and value + * + * \param data Header string + * \param len Byte length of header + * \param name Pointer to location to receive header name + * \param value Pointer to location to receive header value + * \return NSERROR_OK on success, appropriate error otherwise + */ +nserror llcache_fetch_split_header(const char *data, size_t len, char **name, + char **value) +{ + char *n, *v; + const char *colon; + + /* Find colon */ + colon = strchr(data, ':'); + if (colon == NULL) { + /* Failed, assume a key with no value */ + n = strdup(data); + if (n == NULL) + return NSERROR_NOMEM; + + v = strdup(""); + if (v == NULL) { + free(n); + return NSERROR_NOMEM; + } + } else { + /* Split header into name & value */ + + /* Strip leading whitespace from name */ + while (data[0] == ' ' || data[0] == '\t' || + data[0] == '\r' || data[0] == '\n') { + data++; + } + + /* Strip trailing whitespace from name */ + while (colon > data && (colon[-1] == ' ' || + colon[-1] == '\t' || colon[-1] == '\r' || + colon[-1] == '\n')) + colon--; + + n = strndup(data, colon - data); + if (n == NULL) + return NSERROR_NOMEM; + + /* Find colon again */ + while (*colon != ':') { + colon++; + } + + /* Skip over colon and any subsequent whitespace */ + do { + colon++; + } while (*colon == ' ' || *colon == '\t' || + *colon == '\r' || *colon == '\n'); + + /* Strip trailing whitespace from value */ + while (len > 0 && (data[len - 1] == ' ' || + data[len - 1] == '\t' || + data[len - 1] == '\r' || + data[len - 1] == '\n')) { + len--; + } + + v = strndup(colon, len - (colon - data)); + if (v == NULL) { + free(n); + return NSERROR_NOMEM; + } + } + + *name = n; + *value = v; + + return NSERROR_OK; +} + +/** + * Parse a fetch header + * + * \param object Object to parse header for + * \param data Header string + * \param len Byte length of header + * \param name Pointer to location to receive header name + * \param value Pointer to location to receive header value + * \return NSERROR_OK on success, appropriate error otherwise + */ +nserror llcache_fetch_parse_header(llcache_object *object, const char *data, + size_t len, char **name, char **value) +{ + nserror error; + + /* Set fetch response time if not already set */ + if (object->cache.res_time == 0) + object->cache.res_time = time(NULL); + + /* Decompose header into name-value pair */ + error = llcache_fetch_split_header(data, len, name, value); + if (error != NSERROR_OK) + return error; + + /* Parse cache headers to populate cache control data */ +#define SKIP_ST(p) while (*p != '\0' && (*p == ' ' || *p == '\t')) p++ + + if (5 < len && strcasecmp(*name, "Date") == 0) { + /* extract Date header */ + object->cache.date = curl_getdate(*value, NULL); + } else if (4 < len && strcasecmp(*name, "Age") == 0) { + /* extract Age header */ + if ('0' <= **value && **value <= '9') + object->cache.age = atoi(*value); + } else if (8 < len && strcasecmp(*name, "Expires") == 0) { + /* extract Expires header */ + object->cache.expires = curl_getdate(*value, NULL); + } else if (14 < len && strcasecmp(*name, "Cache-Control") == 0) { + /* extract and parse Cache-Control header */ + const char *start = *value; + const char *comma = *value; + + while (*comma != '\0') { + while (*comma != '\0' && *comma != ',') + comma++; + + if (8 < comma - start && (strncasecmp(start, + "no-cache", 8) == 0 || + strncasecmp(start, "no-store", 8) == 0)) + /* When we get a disk cache we should + * distinguish between these two */ + object->cache.no_cache = true; + else if (7 < comma - start && + strncasecmp(start, "max-age", 7) == 0) { + /* Find '=' */ + while (start < comma && *start != '=') + start++; + + /* Skip over it */ + start++; + + /* Skip whitespace */ + SKIP_ST(start); + + if (start < comma) + object->cache.max_age = atoi(start); + } + + if (*comma != '\0') { + /* Skip past comma */ + comma++; + /* Skip whitespace */ + SKIP_ST(comma); + } + + /* Set start for next token */ + start = comma; + } + } else if (5 < len && strcasecmp(*name, "ETag") == 0) { + /* extract ETag header */ + free(object->cache.etag); + object->cache.etag = strdup(*value); + if (object->cache.etag == NULL) + return NSERROR_NOMEM; + } else if (14 < len && strcasecmp(*name, "Last-Modified") == 0) { + /* extract Last-Modified header */ + object->cache.last_modified = curl_getdate(*value, NULL); + } + +#undef SKIP_ST + + return NSERROR_OK; +} + +/** + * Process a fetch header + * + * \param object Object being fetched + * \param data Header string + * \param len Byte length of header + * \return NSERROR_OK on success, appropriate error otherwise + */ +nserror llcache_fetch_process_header(llcache_object *object, const char *data, + size_t len) +{ + nserror error; + char *name, *value; + llcache_header *temp; + + error = llcache_fetch_parse_header(object, data, len, &name, &value); + if (error != NSERROR_OK) + return error; + + /* Append header data to the object's headers array */ + temp = realloc(object->headers, (object->num_headers + 1) * + sizeof(llcache_header)); + if (temp == NULL) { + free(name); + free(value); + return NSERROR_NOMEM; + } + + object->headers = temp; + + object->headers[object->num_headers].name = name; + object->headers[object->num_headers].value = value; + + object->num_headers++; + + return NSERROR_OK; +} + +/** + * Process a chunk of fetched data + * + * \param object Object being fetched + * \param data Data to process + * \param len Byte length of data + * \return NSERROR_OK on success, appropriate error otherwise. + */ +nserror llcache_fetch_process_data(llcache_object *object, const uint8_t *data, + size_t len) +{ + /* Resize source buffer if it's too small */ + if (object->source_len + len >= object->source_alloc) { + const size_t new_len = object->source_len + len + 64 * 1024; + uint8_t *temp = realloc(object->source_data, new_len); + if (temp == NULL) + return NSERROR_NOMEM; + + object->source_data = temp; + object->source_alloc = new_len; + } + + /* Append this data chunk to source buffer */ + memcpy(object->source_data + object->source_len, data, len); + object->source_len += len; + + return NSERROR_OK; +} + +/** + * Handle an authentication request + * + * \param object Object being fetched + * \param realm Authentication realm + * \return NSERROR_OK on success, appropriate error otherwise. + */ +nserror llcache_fetch_auth(llcache_object *object, const char *realm) +{ + nserror error = NSERROR_OK; + + /* Abort fetch for this object */ + fetch_abort(object->fetch.fetch); + object->fetch.fetch = NULL; + + if (query_cb != NULL) { + llcache_query query; + + /* Destroy headers */ + while (object->num_headers > 0) { + object->num_headers--; + + free(object->headers[object->num_headers].name); + free(object->headers[object->num_headers].value); + } + free(object->headers); + object->headers = NULL; + + /* Emit query for authentication details */ + query.type = LLCACHE_QUERY_AUTH; + query.url = object->url; + query.data.auth.realm = realm; + + error = query_cb(&query, query_cb_pw, + llcache_query_handle_response, object); + } else { + llcache_object_user *user; + llcache_event event; + + /* Inform client(s) that object fetch failed */ + event.type = LLCACHE_EVENT_ERROR; + /** \todo More appropriate error message */ + event.data.msg = messages_get("FetchFailed"); + + for (user = object->users; user != NULL; user = user->next) { + error = user->handle.cb(&user->handle, &event, + user->handle.pw); + if (error != NSERROR_OK) + break; + } + } + + return error; +} + +/** + * Handle a TLS certificate verification failure + * + * \param object Object being fetched + * \param certs Certificate chain + * \param num Number of certificates in chain + * \return NSERROR_OK on success, appropriate error otherwise + */ +nserror llcache_fetch_cert_error(llcache_object *object, + const struct ssl_cert_info *certs, size_t num) +{ + nserror error = NSERROR_OK; + + /* Abort fetch for this object */ + fetch_abort(object->fetch.fetch); + object->fetch.fetch = NULL; + + if (query_cb != NULL) { + llcache_query query; + + /* Emit query for TLS */ + query.type = LLCACHE_QUERY_SSL; + query.url = object->url; + query.data.ssl.certs = certs; + query.data.ssl.num = num; + + error = query_cb(&query, query_cb_pw, + llcache_query_handle_response, object); + } else { + llcache_object_user *user; + llcache_event event; + + /* Inform client(s) that object fetch failed */ + event.type = LLCACHE_EVENT_ERROR; + /** \todo More appropriate error message */ + event.data.msg = messages_get("FetchFailed"); + + for (user = object->users; user != NULL; user = user->next) { + error = user->handle.cb(&user->handle, &event, + user->handle.pw); + if (error != NSERROR_OK) + break; + } + } + + return error; +} + |