From 960c11de1aa674dff3c60bfd60de29f0591a1330 Mon Sep 17 00:00:00 2001 From: James Bursa Date: Wed, 4 Jul 2007 18:05:16 +0000 Subject: Rename register_curl_fetchers() to fetch_curl_register(). Add declarations for static functions and reorder functions. svn path=/trunk/netsurf/; revision=3379 --- content/fetch.c | 452 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 243 insertions(+), 209 deletions(-) (limited to 'content/fetch.c') diff --git a/content/fetch.c b/content/fetch.c index 2c3f63700..b69f50be1 100644 --- a/content/fetch.c +++ b/content/fetch.c @@ -48,16 +48,16 @@ bool fetch_active; /**< Fetches in progress, please call fetch_poll(). */ /** Information about a fetcher for a given scheme. */ typedef struct scheme_fetcher_s { - char *scheme_name; /**< The scheme. */ - fetcher_setup_fetch setup_fetch; /**< Set up a fetch. */ - fetcher_start_fetch start_fetch; /**< Start a fetch. */ - fetcher_abort_fetch abort_fetch; /**< Abort a fetch. */ - fetcher_free_fetch free_fetch; /**< Free a fetch. */ - fetcher_poll_fetcher poll_fetcher; /**< Poll this fetcher. */ - fetcher_finalise finaliser; /**< Clean up this fetcher. */ - int refcount; /**< When zero, clean up the fetcher. */ - struct scheme_fetcher_s *next_fetcher; /**< Next fetcher in the list. */ - struct scheme_fetcher_s *prev_fetcher; /**< Prev fetcher in the list. */ + char *scheme_name; /**< The scheme. */ + fetcher_setup_fetch setup_fetch; /**< Set up a fetch. */ + fetcher_start_fetch start_fetch; /**< Start a fetch. */ + fetcher_abort_fetch abort_fetch; /**< Abort a fetch. */ + fetcher_free_fetch free_fetch; /**< Free a fetch. */ + fetcher_poll_fetcher poll_fetcher; /**< Poll this fetcher. */ + fetcher_finalise finaliser; /**< Clean up this fetcher. */ + int refcount; /**< When zero, clean up the fetcher. */ + struct scheme_fetcher_s *next_fetcher; /**< Next fetcher in the list. */ + struct scheme_fetcher_s *prev_fetcher; /**< Prev fetcher in the list. */ } scheme_fetcher; static scheme_fetcher *fetchers = NULL; @@ -67,15 +67,16 @@ struct fetch { fetch_callback callback;/**< Callback function. */ bool abort; /**< Abort requested. */ bool stopped; /**< Download stopped on purpose. */ - char *url; /**< URL. */ - char *referer; /**< Referer URL. */ + char *url; /**< URL. */ + char *referer; /**< Referer URL. */ bool send_referer; /**< Valid to send the referer */ void *p; /**< Private data for callback. */ char *host; /**< Host part of URL. */ long http_code; /**< HTTP response code, or 0. */ - scheme_fetcher *ops; /**< Fetcher operations for this fetch. NULL if not set. */ - void *fetcher_handle; /**< The handle for the fetcher. */ - bool fetch_is_active; /**< This fetch is active. */ + scheme_fetcher *ops; /**< Fetcher operations for this fetch, + NULL if not set. */ + void *fetcher_handle; /**< The handle for the fetcher. */ + bool fetch_is_active; /**< This fetch is active. */ struct fetch *r_prev; /**< Previous active fetch in ::fetch_ring. */ struct fetch *r_next; /**< Next active fetch in ::fetch_ring. */ }; @@ -83,65 +84,13 @@ struct fetch { static struct fetch *fetch_ring = 0; /**< Ring of active fetches. */ static struct fetch *queue_ring = 0; /**< Ring of queued fetches */ -static void fetch_free(struct fetch *f); +#define fetch_ref_fetcher(F) F->refcount++ +static void fetch_unref_fetcher(scheme_fetcher *fetcher); static void fetch_dispatch_jobs(void); +static bool fetch_choose_and_dispatch(void); +static bool fetch_dispatch_job(struct fetch *fetch); +static void fetch_free(struct fetch *f); -#define fetch_ref_fetcher(F) F->refcount++; - -static void fetch_unref_fetcher(scheme_fetcher *fetcher) -{ - if (--fetcher->refcount == 0) { - fetcher->finaliser(fetcher->scheme_name); - free(fetcher->scheme_name); - if (fetcher == fetchers) { - fetchers = fetcher->next_fetcher; - if (fetchers) - fetchers->prev_fetcher = NULL; - } else { - fetcher->prev_fetcher->next_fetcher = fetcher->next_fetcher; - if (fetcher->next_fetcher != NULL) - fetcher->next_fetcher->prev_fetcher = fetcher->prev_fetcher; - } - free(fetcher); - } -} - -bool -fetch_add_fetcher(const char *scheme, - fetcher_initialise initialiser, - fetcher_setup_fetch setup_fetch, - fetcher_start_fetch start_fetch, - fetcher_abort_fetch abort_fetch, - fetcher_free_fetch free_fetch, - fetcher_poll_fetcher poll_fetcher, - fetcher_finalise finaliser) -{ - scheme_fetcher *new_fetcher; - if (!initialiser(scheme)) - return false; - new_fetcher = malloc(sizeof(scheme_fetcher)); - if (new_fetcher == NULL) { - finaliser(scheme); - return false; - } - new_fetcher->scheme_name = strdup(scheme); - if (new_fetcher->scheme_name == NULL) { - free(new_fetcher); - finaliser(scheme); - return false; - } - new_fetcher->refcount = 0; - new_fetcher->setup_fetch = setup_fetch; - new_fetcher->start_fetch = start_fetch; - new_fetcher->abort_fetch = abort_fetch; - new_fetcher->free_fetch = free_fetch; - new_fetcher->poll_fetcher = poll_fetcher; - new_fetcher->finaliser = finaliser; - new_fetcher->next_fetcher = fetchers; - fetchers = new_fetcher; - fetch_ref_fetcher(new_fetcher); - return true; -} /** * Initialise the fetcher. @@ -151,8 +100,8 @@ fetch_add_fetcher(const char *scheme, void fetch_init(void) { - register_curl_fetchers(); - fetch_active = false; + fetch_curl_register(); + fetch_active = false; } @@ -164,14 +113,73 @@ void fetch_init(void) void fetch_quit(void) { - while (fetchers != NULL) { - if (fetchers->refcount != 1) { - LOG(("Fetcher for scheme %s still active?!", fetchers->scheme_name)); - /* We shouldn't do this, but... */ - fetchers->refcount = 1; - } - fetch_unref_fetcher(fetchers); - } + while (fetchers != NULL) { + if (fetchers->refcount != 1) { + LOG(("Fetcher for scheme %s still active?!", + fetchers->scheme_name)); + /* We shouldn't do this, but... */ + fetchers->refcount = 1; + } + fetch_unref_fetcher(fetchers); + } +} + + +bool fetch_add_fetcher(const char *scheme, + fetcher_initialise initialiser, + fetcher_setup_fetch setup_fetch, + fetcher_start_fetch start_fetch, + fetcher_abort_fetch abort_fetch, + fetcher_free_fetch free_fetch, + fetcher_poll_fetcher poll_fetcher, + fetcher_finalise finaliser) +{ + scheme_fetcher *new_fetcher; + if (!initialiser(scheme)) + return false; + new_fetcher = malloc(sizeof(scheme_fetcher)); + if (new_fetcher == NULL) { + finaliser(scheme); + return false; + } + new_fetcher->scheme_name = strdup(scheme); + if (new_fetcher->scheme_name == NULL) { + free(new_fetcher); + finaliser(scheme); + return false; + } + new_fetcher->refcount = 0; + new_fetcher->setup_fetch = setup_fetch; + new_fetcher->start_fetch = start_fetch; + new_fetcher->abort_fetch = abort_fetch; + new_fetcher->free_fetch = free_fetch; + new_fetcher->poll_fetcher = poll_fetcher; + new_fetcher->finaliser = finaliser; + new_fetcher->next_fetcher = fetchers; + fetchers = new_fetcher; + fetch_ref_fetcher(new_fetcher); + return true; +} + + +void fetch_unref_fetcher(scheme_fetcher *fetcher) +{ + if (--fetcher->refcount == 0) { + fetcher->finaliser(fetcher->scheme_name); + free(fetcher->scheme_name); + if (fetcher == fetchers) { + fetchers = fetcher->next_fetcher; + if (fetchers) + fetchers->prev_fetcher = NULL; + } else { + fetcher->prev_fetcher->next_fetcher = + fetcher->next_fetcher; + if (fetcher->next_fetcher != NULL) + fetcher->next_fetcher->prev_fetcher = + fetcher->prev_fetcher; + } + free(fetcher); + } } @@ -196,16 +204,17 @@ void fetch_quit(void) */ struct fetch * fetch_start(const char *url, const char *referer, - fetch_callback callback, - void *p, bool only_2xx, const char *post_urlenc, - struct form_successful_control *post_multipart, - bool verifiable, const char *parent_url, char *headers[]) + fetch_callback callback, + void *p, bool only_2xx, const char *post_urlenc, + struct form_successful_control *post_multipart, + bool verifiable, const char *parent_url, + char *headers[]) { char *host; struct fetch *fetch; url_func_result res; char *ref1 = 0, *ref2 = 0; - scheme_fetcher *fetcher = fetchers; + scheme_fetcher *fetcher = fetchers; fetch = malloc(sizeof (*fetch)); if (!fetch) @@ -252,45 +261,45 @@ struct fetch * fetch_start(const char *url, const char *referer, fetch->http_code = 0; fetch->r_prev = 0; fetch->r_next = 0; - fetch->referer = 0; - fetch->ops = 0; - fetch->fetch_is_active = false; - - if (referer != NULL) { - fetch->referer = strdup(referer); - if (fetch->referer == NULL) - goto failed; + fetch->referer = 0; + fetch->ops = 0; + fetch->fetch_is_active = false; + + if (referer != NULL) { + fetch->referer = strdup(referer); + if (fetch->referer == NULL) + goto failed; if (option_send_referer && ref1 && ref2 && strcasecmp(ref1, ref2) == 0) fetch->send_referer = true; - } + } if (!fetch->url) goto failed; - /* Pick the scheme ops */ - while (fetcher) { - if (strcmp(fetcher->scheme_name, ref1) == 0) { - fetch->ops = fetcher; - break; - } - fetcher = fetcher->next_fetcher; - } + /* Pick the scheme ops */ + while (fetcher) { + if (strcmp(fetcher->scheme_name, ref1) == 0) { + fetch->ops = fetcher; + break; + } + fetcher = fetcher->next_fetcher; + } - if (fetch->ops == NULL) - goto failed; + if (fetch->ops == NULL) + goto failed; - /* Got a scheme fetcher, try and set up the fetch */ - fetch->fetcher_handle = - fetch->ops->setup_fetch(fetch, url, only_2xx, post_urlenc, - post_multipart, verifiable, parent_url, - (const char **)headers); + /* Got a scheme fetcher, try and set up the fetch */ + fetch->fetcher_handle = + fetch->ops->setup_fetch(fetch, url, only_2xx, post_urlenc, + post_multipart, verifiable, parent_url, + (const char **)headers); - if (fetch->fetcher_handle == NULL) - goto failed; + if (fetch->fetcher_handle == NULL) + goto failed; - /* Rah, got it, so ref the fetcher. */ - fetch_ref_fetcher(fetch->ops); + /* Rah, got it, so ref the fetcher. */ + fetch_ref_fetcher(fetch->ops); /* these aren't needed past here */ if (ref1) { @@ -298,12 +307,12 @@ struct fetch * fetch_start(const char *url, const char *referer, ref1 = 0; } - if (ref2) { - free(ref2); - ref2 = 0; - } + if (ref2) { + free(ref2); + ref2 = 0; + } - /* Dump us in the queue and ask the queue to run. */ + /* Dump us in the queue and ask the queue to run. */ RING_INSERT(queue_ring, fetch); fetch_dispatch_jobs(); return fetch; @@ -313,37 +322,66 @@ failed: if (ref1) free(ref1); free(fetch->url); - if (fetch->referer) - free(fetch->referer); - free(fetch); + if (fetch->referer) + free(fetch->referer); + free(fetch); return 0; } + /** - * Dispatch a single job + * Dispatch as many jobs as we have room to dispatch. */ -static bool fetch_dispatch_job(struct fetch *fetch) +void fetch_dispatch_jobs(void) { - RING_REMOVE(queue_ring, fetch); - LOG(("Attempting to start fetch %p, fetcher %p, url %s", fetch, - fetch->fetcher_handle, fetch->url)); - if (!fetch->ops->start_fetch(fetch->fetcher_handle)) { - RING_INSERT(queue_ring, fetch); /* Put it back on the end of the queue */ - return false; - } else { - RING_INSERT(fetch_ring, fetch); - fetch->fetch_is_active = true; - return true; + int all_active, all_queued; + + if (!queue_ring) + return; /* Nothing to do, the queue is empty */ + RING_GETSIZE(struct fetch, queue_ring, all_queued); + RING_GETSIZE(struct fetch, fetch_ring, all_active); + + LOG(("queue_ring %i, fetch_ring %i", all_queued, all_active)); + + struct fetch *q = queue_ring; + if (q) { + do { + LOG(("queue_ring: %s", q->url)); + q = q->r_next; + } while (q != queue_ring); } + struct fetch *f = fetch_ring; + if (f) { + do { + LOG(("fetch_ring: %s", f->url)); + f = f->r_next; + } while (f != fetch_ring); + } + + while ( all_queued && all_active < option_max_fetchers ) { + /*LOG(("%d queued, %d fetching", all_queued, all_active));*/ + if (fetch_choose_and_dispatch()) { + all_queued--; + all_active++; + } else { + /* Either a dispatch failed or we ran out. Just stop */ + break; + } + } + fetch_active = (all_active > 0); + LOG(("Fetch ring is now %d elements.", all_active)); + LOG(("Queue ring is now %d elements.", all_queued)); } + /** - * Choose and dispatch a single job. Return false if we failed to dispatch anything. + * Choose and dispatch a single job. Return false if we failed to dispatch + * anything. * * We don't check the overall dispatch size here because we're not called unless * there is room in the fetch queue for us. */ -static bool fetch_choose_and_dispatch(void) +bool fetch_choose_and_dispatch(void) { struct fetch *queueitem; queueitem = queue_ring; @@ -352,7 +390,8 @@ static bool fetch_choose_and_dispatch(void) * fetch ring */ int countbyhost; - RING_COUNTBYHOST(struct fetch, fetch_ring, countbyhost, queueitem->host); + RING_COUNTBYHOST(struct fetch, fetch_ring, countbyhost, + queueitem->host); if (countbyhost < option_max_fetchers_per_host) { /* We can dispatch this item in theory */ return fetch_dispatch_job(queueitem); @@ -362,32 +401,26 @@ static bool fetch_choose_and_dispatch(void) return false; } + /** - * Dispatch as many jobs as we have room to dispatch. + * Dispatch a single job */ -static void fetch_dispatch_jobs(void) +bool fetch_dispatch_job(struct fetch *fetch) { - int all_active, all_queued; - - if (!queue_ring) - return; /* Nothing to do, the queue is empty */ - RING_GETSIZE(struct fetch, queue_ring, all_queued); - RING_GETSIZE(struct fetch, fetch_ring, all_active); - while ( all_queued && all_active < option_max_fetchers ) { - /*LOG(("%d queued, %d fetching", all_queued, all_active));*/ - if (fetch_choose_and_dispatch()) { - all_queued--; - all_active++; - } else { - /* Either a dispatch failed or we ran out. Just stop */ - break; - } + RING_REMOVE(queue_ring, fetch); + LOG(("Attempting to start fetch %p, fetcher %p, url %s", fetch, + fetch->fetcher_handle, fetch->url)); + if (!fetch->ops->start_fetch(fetch->fetcher_handle)) { + RING_INSERT(queue_ring, fetch); /* Put it back on the end of the queue */ + return false; + } else { + RING_INSERT(fetch_ring, fetch); + fetch->fetch_is_active = true; + return true; } - fetch_active = (all_active > 0); - LOG(("Fetch ring is now %d elements.", all_active)); - LOG(("Queue ring is now %d elements.", all_queued)); } + /** * Abort a fetch. */ @@ -396,7 +429,7 @@ void fetch_abort(struct fetch *f) { assert(f); LOG(("fetch %p, fetcher %p, url '%s'", f, f->fetcher_handle, f->url)); - f->ops->abort_fetch(f->fetcher_handle); + f->ops->abort_fetch(f->fetcher_handle); } @@ -406,13 +439,13 @@ void fetch_abort(struct fetch *f) void fetch_free(struct fetch *f) { - LOG(("Freeing fetch %p, fetcher %p", f, f->fetcher_handle)); - f->ops->free_fetch(f->fetcher_handle); - fetch_unref_fetcher(f->ops); + LOG(("Freeing fetch %p, fetcher %p", f, f->fetcher_handle)); + f->ops->free_fetch(f->fetcher_handle); + fetch_unref_fetcher(f->ops); free(f->url); free(f->host); - if (f->referer) - free(f->referer); + if (f->referer) + free(f->referer); free(f); } @@ -425,14 +458,17 @@ void fetch_free(struct fetch *f) void fetch_poll(void) { - scheme_fetcher *fetcher = fetchers; - if (!fetch_active) - return; /* No point polling, there's no fetch active. */ - while (fetcher != NULL) { - /* LOG(("Polling fetcher for %s", fetcher->scheme_name)); */ - fetcher->poll_fetcher(fetcher->scheme_name); - fetcher = fetcher->next_fetcher; - } + scheme_fetcher *fetcher = fetchers; + + fetch_dispatch_jobs(); + + if (!fetch_active) + return; /* No point polling, there's no fetch active. */ + while (fetcher != NULL) { + /* LOG(("Polling fetcher for %s", fetcher->scheme_name)); */ + fetcher->poll_fetcher(fetcher->scheme_name); + fetcher = fetcher->next_fetcher; + } } @@ -447,18 +483,18 @@ bool fetch_can_fetch(const char *url) { const char *semi; size_t len; - scheme_fetcher *fetcher = fetchers; + scheme_fetcher *fetcher = fetchers; if ((semi = strchr(url, ':')) == NULL) return false; len = semi - url; - while (fetcher != NULL) { - if (strlen(fetcher->scheme_name) == len && - strncmp(fetcher->scheme_name, url, len) == 0) - return true; - fetcher = fetcher->next_fetcher; - } + while (fetcher != NULL) { + if (strlen(fetcher->scheme_name) == len && + strncmp(fetcher->scheme_name, url, len) == 0) + return true; + fetcher = fetcher->next_fetcher; + } return false; } @@ -469,8 +505,8 @@ bool fetch_can_fetch(const char *url) */ void fetch_change_callback(struct fetch *fetch, - fetch_callback callback, - void *p) + fetch_callback callback, + void *p) { assert(fetch); fetch->callback = callback; @@ -503,49 +539,47 @@ void fetch_send_callback(fetch_msg msg, struct fetch *fetch, const void *data, unsigned long size) { - LOG(("Fetcher sending callback. Fetch %p, fetcher %p data %p size %lu", - fetch, fetch->fetcher_handle, data, size)); - fetch->callback(msg, fetch->p, data, size); + LOG(("Fetcher sending callback. Fetch %p, fetcher %p data %p size %lu", + fetch, fetch->fetcher_handle, data, size)); + fetch->callback(msg, fetch->p, data, size); } void fetch_can_be_freed(struct fetch *fetch) { - int all_active, all_queued; - - /* Go ahead and free the fetch properly now */ - LOG(("Fetch %p, fetcher %p can be freed", fetch, fetch->fetcher_handle)); - - if (fetch->fetch_is_active) { - RING_REMOVE(fetch_ring, fetch); - } else { - RING_REMOVE(queue_ring, fetch); - } - + int all_active, all_queued; + + /* Go ahead and free the fetch properly now */ + LOG(("Fetch %p, fetcher %p can be freed", fetch, fetch->fetcher_handle)); + + if (fetch->fetch_is_active) { + RING_REMOVE(fetch_ring, fetch); + } else { + RING_REMOVE(queue_ring, fetch); + } + RING_GETSIZE(struct fetch, fetch_ring, all_active); RING_GETSIZE(struct fetch, queue_ring, all_queued); - - fetch_active = (all_active > 0); - - LOG(("Fetch ring is now %d elements.", all_active)); - LOG(("Queue ring is now %d elements.", all_queued)); - - fetch_free(fetch); - - fetch_dispatch_jobs(); + + fetch_active = (all_active > 0); + + LOG(("Fetch ring is now %d elements.", all_active)); + LOG(("Queue ring is now %d elements.", all_queued)); + + fetch_free(fetch); } void fetch_set_http_code(struct fetch *fetch, long http_code) { - LOG(("Setting HTTP code to %ld", http_code)); - fetch->http_code = http_code; + LOG(("Setting HTTP code to %ld", http_code)); + fetch->http_code = http_code; } const char * fetch_get_referer_to_send(struct fetch *fetch) { - if (fetch->send_referer) - return fetch->referer; - return NULL; + if (fetch->send_referer) + return fetch->referer; + return NULL; } -- cgit v1.2.3