From 4b2101ba6ab62ae26d82cc8b86e0e61e9c007156 Mon Sep 17 00:00:00 2001 From: Vincent Sanders Date: Thu, 19 Jun 2014 18:27:24 +0100 Subject: clean up the fetcher factory and improve its API --- content/fetch.c | 313 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 154 insertions(+), 159 deletions(-) (limited to 'content/fetch.c') diff --git a/content/fetch.c b/content/fetch.c index 4736670ff..13c7c6542 100644 --- a/content/fetch.c +++ b/content/fetch.c @@ -19,12 +19,15 @@ */ /** \file - * Fetching of data from a URL (implementation). + * Implementation of fetching of data from a URL. + * + * The implementation is the fetch factory and the generic operations + * around the fetcher specific methods. * * Active fetches are held in the circular linked list ::fetch_ring. There may * be at most ::option_max_fetchers_per_host active requests per Host: header. * There may be at most ::option_max_fetchers active requests overall. Inactive - * fetchers are stored in the ::queue_ring waiting for use. + * fetches are stored in the ::queue_ring waiting for use. */ #include @@ -33,17 +36,9 @@ #include #include #include - #include #include "utils/config.h" -#include "content/fetch.h" -#include "content/fetchers/resource.h" -#include "content/fetchers/about.h" -#include "content/fetchers/curl.h" -#include "content/fetchers/data.h" -#include "content/fetchers/file.h" -#include "content/urldb.h" #include "desktop/netsurf.h" #include "utils/corestrings.h" #include "utils/nsoption.h" @@ -53,27 +48,34 @@ #include "utils/utils.h" #include "utils/ring.h" +#include "content/fetch.h" +#include "content/fetchers.h" +#include "content/fetchers/resource.h" +#include "content/fetchers/about.h" +#include "content/fetchers/curl.h" +#include "content/fetchers/data.h" +#include "content/fetchers/file.h" +#include "content/urldb.h" + /* Define this to turn on verbose fetch logging */ #undef DEBUG_FETCH_VERBOSE -bool fetch_active; /**< Fetches in progress, please call fetch_poll(). */ +/** The maximum number of fetchers that can be added */ +#define MAX_FETCHERS 8 -/** Information about a fetcher for a given scheme. */ +bool fetch_active; /**< Fetches in progress, please call fetch_poll(). */ + +/** + * Information about a fetcher for a given scheme. + */ typedef struct scheme_fetcher_s { - lwc_string *scheme_name; /**< The scheme. */ - fetcher_can_fetch can_fetch; /**< Ensure an URL can be fetched. */ - fetcher_setup_fetch setup_fetch; /**< Set up a fetch. */ - fetcher_start_fetch start_fetch; /**< Start a fetch. */ - fetcher_abort_fetch abort_fetch; /**< Abort a fetch. */ - fetcher_free_fetch free_fetch; /**< Free a fetch. */ - fetcher_poll_fetcher poll_fetcher; /**< Poll this fetcher. */ - fetcher_finalise finaliser; /**< Clean up this fetcher. */ - int refcount; /**< When zero, clean up the fetcher. */ - struct scheme_fetcher_s *next_fetcher; /**< Next fetcher in the list. */ - struct scheme_fetcher_s *prev_fetcher; /**< Prev fetcher in the list. */ + lwc_string *scheme; /**< The scheme. */ + + struct fetcher_operation_table ops; /**< The fetchers operations. */ + int refcount; /**< When zero the fetcher is no longer in use. */ } scheme_fetcher; -static scheme_fetcher *fetchers = NULL; +static scheme_fetcher fetchers[MAX_FETCHERS]; /** Information for a single fetch. */ struct fetch { @@ -85,43 +87,53 @@ struct fetch { void *p; /**< Private data for callback. */ lwc_string *host; /**< Host part of URL, interned */ long http_code; /**< HTTP response code, or 0. */ - scheme_fetcher *ops; /**< Fetcher operations for this fetch, - NULL if not set. */ + int fetcherd; /**< Fetcher descriptor for this fetch */ void *fetcher_handle; /**< The handle for the fetcher. */ bool fetch_is_active; /**< This fetch is active. */ struct fetch *r_prev; /**< Previous active fetch in ::fetch_ring. */ struct fetch *r_next; /**< Next active fetch in ::fetch_ring. */ }; -static struct fetch *fetch_ring = 0; /**< Ring of active fetches. */ -static struct fetch *queue_ring = 0; /**< Ring of queued fetches */ - -#define fetch_ref_fetcher(F) F->refcount++ +static struct fetch *fetch_ring = NULL; /**< Ring of active fetches. */ +static struct fetch *queue_ring = NULL; /**< Ring of queued fetches */ /****************************************************************************** * fetch internals * ******************************************************************************/ -static void fetch_unref_fetcher(scheme_fetcher *fetcher) +static inline void fetch_ref_fetcher(int fetcherd) { - if (--fetcher->refcount == 0) { - fetcher->finaliser(fetcher->scheme_name); - lwc_string_unref(fetcher->scheme_name); - if (fetcher == fetchers) { - fetchers = fetcher->next_fetcher; - if (fetchers) - fetchers->prev_fetcher = NULL; - } else { - fetcher->prev_fetcher->next_fetcher = - fetcher->next_fetcher; - if (fetcher->next_fetcher != NULL) - fetcher->next_fetcher->prev_fetcher = - fetcher->prev_fetcher; - } - free(fetcher); + fetchers[fetcherd].refcount++; +} + +static inline void fetch_unref_fetcher(int fetcherd) +{ + fetchers[fetcherd].refcount--; + if (fetchers[fetcherd].refcount == 0) { + fetchers[fetcherd].ops.finalise(fetchers[fetcherd].scheme); + lwc_string_unref(fetchers[fetcherd].scheme); } } +/** + * Find a suitable fetcher for a scheme. + */ +static int get_fetcher_for_scheme(lwc_string *scheme) +{ + int fetcherd; + bool match; + + for (fetcherd = 0; fetcherd < MAX_FETCHERS; fetcherd++) { + if ((fetchers[fetcherd].refcount > 0) && + (lwc_string_isequal(fetchers[fetcherd].scheme, + scheme, &match) == lwc_error_ok) && + (match == true)) { + return fetcherd; + } + } + return -1; +} + /** * Dispatch a single job */ @@ -132,7 +144,7 @@ static bool fetch_dispatch_job(struct fetch *fetch) LOG(("Attempting to start fetch %p, fetcher %p, url %s", fetch, fetch->fetcher_handle, nsurl_access(fetch->url))); #endif - if (!fetch->ops->start_fetch(fetch->fetcher_handle)) { + if (!fetchers[fetch->fetcherd].ops.start(fetch->fetcher_handle)) { RING_INSERT(queue_ring, fetch); /* Put it back on the end of the queue */ return false; } else { @@ -236,7 +248,7 @@ static void fetch_dispatch_jobs(void) ******************************************************************************/ /* exported interface documented in content/fetch.h */ -nserror fetch_init(void) +nserror fetcher_init(void) { fetch_curl_register(); fetch_data_register(); @@ -249,76 +261,81 @@ nserror fetch_init(void) return NSERROR_OK; } -/* exported interface documented in content/fetch.h */ -void fetch_quit(void) +/* exported interface documented in content/fetchers.h */ +void fetcher_quit(void) { - while (fetchers != NULL) { - if (fetchers->refcount != 1) { - LOG(("Fetcher for scheme %s still active?!", - lwc_string_data(fetchers->scheme_name))); - /* We shouldn't do this, but... */ - fetchers->refcount = 1; + int fetcherd; /* fetcher index */ + for (fetcherd = 0; fetcherd < MAX_FETCHERS; fetcherd++) { + if (fetchers[fetcherd].refcount > 0) { + /* assert if the fetcher is active at quit */ + assert(fetchers[fetcherd].refcount == 1); + + fetch_unref_fetcher(fetcherd); } - fetch_unref_fetcher(fetchers); } } -/* exported interface documented in content/fetch.h */ -bool fetch_add_fetcher(lwc_string *scheme, - fetcher_initialise initialiser, - fetcher_can_fetch can_fetch, - fetcher_setup_fetch setup_fetch, - fetcher_start_fetch start_fetch, - fetcher_abort_fetch abort_fetch, - fetcher_free_fetch free_fetch, - fetcher_poll_fetcher poll_fetcher, - fetcher_finalise finaliser) +/* exported interface documented in content/fetchers.h */ +nserror +fetcher_add(lwc_string *scheme, const struct fetcher_operation_table *ops) { - scheme_fetcher *new_fetcher; - if (!initialiser(scheme)) - return false; - new_fetcher = malloc(sizeof(scheme_fetcher)); - if (new_fetcher == NULL) { - finaliser(scheme); - return false; + int fetcherd; + + /* find unused fetcher descriptor */ + for (fetcherd = 0; fetcherd < MAX_FETCHERS; fetcherd++) { + if (fetchers[fetcherd].refcount == 0) { + break; + } } - new_fetcher->scheme_name = scheme; - new_fetcher->refcount = 0; - new_fetcher->can_fetch = can_fetch; - new_fetcher->setup_fetch = setup_fetch; - new_fetcher->start_fetch = start_fetch; - new_fetcher->abort_fetch = abort_fetch; - new_fetcher->free_fetch = free_fetch; - new_fetcher->poll_fetcher = poll_fetcher; - new_fetcher->finaliser = finaliser; - new_fetcher->next_fetcher = fetchers; - fetchers = new_fetcher; - fetch_ref_fetcher(new_fetcher); - - return true; + if (fetcherd == MAX_FETCHERS) { + return NSERROR_INIT_FAILED; + } + + if (!ops->initialise(scheme)) { + return NSERROR_INIT_FAILED; + } + + fetchers[fetcherd].scheme = scheme; + fetchers[fetcherd].ops = *ops; + + fetch_ref_fetcher(fetcherd); + + return NSERROR_OK; } /* exported interface documented in content/fetch.h */ -struct fetch * fetch_start(nsurl *url, nsurl *referer, - fetch_callback callback, - void *p, bool only_2xx, const char *post_urlenc, - const struct fetch_multipart_data *post_multipart, - bool verifiable, bool downgrade_tls, - const char *headers[]) +struct fetch * +fetch_start(nsurl *url, + nsurl *referer, + fetch_callback callback, + void *p, + bool only_2xx, + const char *post_urlenc, + const struct fetch_multipart_data *post_multipart, + bool verifiable, + bool downgrade_tls, + const char *headers[]) { struct fetch *fetch; - scheme_fetcher *fetcher = fetchers; lwc_string *scheme; bool match; fetch = malloc(sizeof (*fetch)); - if (fetch == NULL) + if (fetch == NULL) { return NULL; + } /* The URL we're fetching must have a scheme */ scheme = nsurl_get_component(url, NSURL_SCHEME); assert(scheme != NULL); + /* try and obtain a fetcher for this scheme */ + fetch->fetcherd = get_fetcher_for_scheme(scheme); + if (fetch->fetcherd == -1) { + lwc_string_unref(scheme); + return NULL; + } + #ifdef DEBUG_FETCH_VERBOSE LOG(("fetch %p, url '%s'", fetch, nsurl_access(url))); #endif @@ -334,7 +351,6 @@ struct fetch * fetch_start(nsurl *url, nsurl *referer, fetch->referer = NULL; fetch->send_referer = false; fetch->fetcher_handle = NULL; - fetch->ops = NULL; fetch->fetch_is_active = false; fetch->host = nsurl_get_component(url, NSURL_HOST); @@ -378,53 +394,38 @@ struct fetch * fetch_start(nsurl *url, nsurl *referer, lwc_string_unref(ref_scheme); } - /* Pick the scheme ops */ - while (fetcher) { - if ((lwc_string_isequal(fetcher->scheme_name, scheme, - &match) == lwc_error_ok) && (match == true)) { - fetch->ops = fetcher; - break; - } - fetcher = fetcher->next_fetcher; - } + /* these aren't needed past here */ + lwc_string_unref(scheme); - if (fetch->ops == NULL) - goto failed; + /* try and set up the fetch */ + fetch->fetcher_handle = fetchers[fetch->fetcherd].ops.setup(fetch, url, + only_2xx, downgrade_tls, + post_urlenc, post_multipart, + headers); + if (fetch->fetcher_handle == NULL) { - /* Got a scheme fetcher, try and set up the fetch */ - fetch->fetcher_handle = fetch->ops->setup_fetch(fetch, url, - only_2xx, downgrade_tls, - post_urlenc, post_multipart, - headers); + if (fetch->host != NULL) + lwc_string_unref(fetch->host); - if (fetch->fetcher_handle == NULL) - goto failed; + if (fetch->url != NULL) + nsurl_unref(fetch->url); - /* Rah, got it, so ref the fetcher. */ - fetch_ref_fetcher(fetch->ops); + if (fetch->referer != NULL) + nsurl_unref(fetch->referer); - /* these aren't needed past here */ - lwc_string_unref(scheme); + free(fetch); + + return NULL; + } + + /* Rah, got it, so ref the fetcher. */ + fetch_ref_fetcher(fetch->fetcherd); /* Dump us in the queue and ask the queue to run. */ RING_INSERT(queue_ring, fetch); fetch_dispatch_jobs(); return fetch; - -failed: - lwc_string_unref(scheme); - - if (fetch->host != NULL) - lwc_string_unref(fetch->host); - if (fetch->url != NULL) - nsurl_unref(fetch->url); - if (fetch->referer != NULL) - nsurl_unref(fetch->referer); - - free(fetch); - - return NULL; } /* exported interface documented in content/fetch.h */ @@ -435,7 +436,7 @@ void fetch_abort(struct fetch *f) LOG(("fetch %p, fetcher %p, url '%s'", f, f->fetcher_handle, nsurl_access(f->url))); #endif - f->ops->abort_fetch(f->fetcher_handle); + fetchers[f->fetcherd].ops.abort(f->fetcher_handle); } /* exported interface documented in content/fetch.h */ @@ -444,8 +445,10 @@ void fetch_free(struct fetch *f) #ifdef DEBUG_FETCH_VERBOSE LOG(("Freeing fetch %p, fetcher %p", f, f->fetcher_handle)); #endif - f->ops->free_fetch(f->fetcher_handle); - fetch_unref_fetcher(f->ops); + fetchers[f->fetcherd].ops.free(f->fetcher_handle); + + fetch_unref_fetcher(f->fetcherd); + nsurl_unref(f->url); if (f->referer != NULL) nsurl_unref(f->referer); @@ -454,45 +457,37 @@ void fetch_free(struct fetch *f) free(f); } -/* exported interface documented in content/fetch.h */ -void fetch_poll(void) +/* exported interface documented in content/fetchers.h */ +void fetcher_poll(void) { - scheme_fetcher *fetcher = fetchers; - scheme_fetcher *next_fetcher; + int fetcherd; fetch_dispatch_jobs(); - if (!fetch_active) - return; /* No point polling, there's no fetch active. */ - while (fetcher != NULL) { - next_fetcher = fetcher->next_fetcher; - if (fetcher->poll_fetcher != NULL) { - /* LOG(("Polling fetcher for %s", - lwc_string_data(fetcher->scheme_name))); */ - fetcher->poll_fetcher(fetcher->scheme_name); + if (fetch_active) { + for (fetcherd = 0; fetcherd < MAX_FETCHERS; fetcherd++) { + if (fetchers[fetcherd].refcount > 0) { + /* fetcher present */ + fetchers[fetcherd].ops.poll(fetchers[fetcherd].scheme); + } } - fetcher = next_fetcher; } } /* exported interface documented in content/fetch.h */ bool fetch_can_fetch(const nsurl *url) { - scheme_fetcher *fetcher = fetchers; - bool match; lwc_string *scheme = nsurl_get_component(url, NSURL_SCHEME); + int fetcherd; - while (fetcher != NULL) { - if (lwc_string_isequal(fetcher->scheme_name, scheme, &match) == lwc_error_ok && match == true) { - break; - } + fetcherd = get_fetcher_for_scheme(scheme); + lwc_string_unref(scheme); - fetcher = fetcher->next_fetcher; + if (fetcherd == -1) { + return false; } - lwc_string_unref(scheme); - - return fetcher == NULL ? false : fetcher->can_fetch(url); + return fetchers[fetcherd].ops.acceptable(url); } /* exported interface documented in content/fetch.h */ -- cgit v1.2.3