diff options
-rw-r--r-- | content/fetch.h | 1 | ||||
-rw-r--r-- | content/fetchers/curl.c | 16 | ||||
-rw-r--r-- | content/llcache.c | 19 | ||||
-rw-r--r-- | content/llcache.h | 3 | ||||
-rw-r--r-- | desktop/netsurf.c | 3 | ||||
-rw-r--r-- | desktop/options.h | 8 | ||||
-rw-r--r-- | utils/nsoption.c | 14 |
7 files changed, 60 insertions, 4 deletions
diff --git a/content/fetch.h b/content/fetch.h index 529a800fa..3c1f1ccae 100644 --- a/content/fetch.h +++ b/content/fetch.h @@ -37,6 +37,7 @@ typedef enum { FETCH_HEADER, FETCH_DATA, FETCH_FINISHED, + FETCH_TIMEDOUT, FETCH_ERROR, FETCH_REDIRECT, FETCH_NOTMODIFIED, diff --git a/content/fetchers/curl.c b/content/fetchers/curl.c index a2c6f2eb4..624cdbf41 100644 --- a/content/fetchers/curl.c +++ b/content/fetchers/curl.c @@ -973,11 +973,19 @@ static void fetch_curl_done(CURL *curl_handle, CURLcode result) msg.data.cert_err.num_certs = i; fetch_send_callback(&msg, f->fetch_handle); } else if (error) { - if (result != CURLE_SSL_CONNECT_ERROR) { + switch (result) { + case CURLE_SSL_CONNECT_ERROR: + msg.type = FETCH_SSL_ERR; + break; + + case CURLE_OPERATION_TIMEDOUT: + msg.type = FETCH_TIMEDOUT; + msg.data.error = curl_easy_strerror(result); + break; + + default: msg.type = FETCH_ERROR; msg.data.error = curl_easy_strerror(result); - } else { - msg.type = FETCH_SSL_ERR; } fetch_send_callback(&msg, f->fetch_handle); @@ -1302,7 +1310,7 @@ nserror fetch_curl_register(void) SETOPT(CURLOPT_LOW_SPEED_LIMIT, 1L); SETOPT(CURLOPT_LOW_SPEED_TIME, 180L); SETOPT(CURLOPT_NOSIGNAL, 1L); - SETOPT(CURLOPT_CONNECTTIMEOUT, 30L); + SETOPT(CURLOPT_CONNECTTIMEOUT, nsoption_uint(curl_fetch_timeout)); if (nsoption_charp(ca_bundle) && strcmp(nsoption_charp(ca_bundle), "")) { diff --git a/content/llcache.c b/content/llcache.c index 851dbbb5f..e95eefbe6 100644 --- a/content/llcache.c +++ b/content/llcache.c @@ -120,6 +120,8 @@ typedef struct { uint32_t redirect_count; /**< Count of redirects followed */ + uint32_t retries_remaining; /**< Number of times to retry on timeout */ + bool tried_with_auth; /**< Whether we've tried with auth */ bool tried_with_tls_downgrade; /**< Whether we've tried TLS <= 1.0 */ @@ -227,6 +229,9 @@ struct llcache_s { /** The target upper bound for the RAM cache size */ uint32_t limit; + /** The number of fetch attempts we make when timing out */ + uint32_t fetch_attempts; + /** Whether or not our users are caught up */ bool all_caught_up; @@ -909,6 +914,7 @@ static nserror llcache_object_fetch(llcache_object *object, uint32_t flags, object->fetch.referer = referer_clone; object->fetch.post = post_clone; object->fetch.redirect_count = redirect_count; + object->fetch.retries_remaining = llcache->fetch_attempts; return llcache_object_refetch(object); } @@ -2653,6 +2659,18 @@ static void llcache_fetch_callback(const fetch_msg *msg, void *p) break; /* Out-of-band information */ + case FETCH_TIMEDOUT: + /* Timed out while trying to fetch. */ + /* The fetch has already been cleaned up by the fetcher but + * we would like to retry if we can. */ + if (object->fetch.retries_remaining > 1) { + object->fetch.retries_remaining--; + error = llcache_object_refetch(object); + break; + } + /* Otherwise fall through to error, setting the message to + * a timeout + */ case FETCH_ERROR: /* An error occurred while fetching */ /* The fetch has has already been cleaned up by the fetcher */ @@ -3303,6 +3321,7 @@ llcache_initialise(const struct llcache_parameters *prm) llcache->minimum_bandwidth = prm->minimum_bandwidth; llcache->maximum_bandwidth = prm->maximum_bandwidth; llcache->time_quantum = prm->time_quantum; + llcache->fetch_attempts = prm->fetch_attempts; llcache->all_caught_up = true; LOG("llcache initialising with a limit of %d bytes", llcache->limit); diff --git a/content/llcache.h b/content/llcache.h index 1b7ba7dae..cce9a79bf 100644 --- a/content/llcache.h +++ b/content/llcache.h @@ -230,6 +230,9 @@ struct llcache_parameters { */ unsigned long time_quantum; + /** The number of fetches to attempt when timing out */ + uint32_t fetch_attempts; + struct llcache_store_parameters store; }; diff --git a/desktop/netsurf.c b/desktop/netsurf.c index 0fd7b7bbd..d6b5abe6e 100644 --- a/desktop/netsurf.c +++ b/desktop/netsurf.c @@ -164,6 +164,9 @@ nserror netsurf_init(const char *store_path) LOG("Setting minimum memory cache size %zd", hlcache_parameters.llcache.limit); } + /* Set up the max attempts made to fetch a timing out resource */ + hlcache_parameters.llcache.fetch_attempts = nsoption_uint(max_retried_fetches); + /* image cache is 25% of total memory cache size */ image_cache_parameters.limit = (hlcache_parameters.llcache.limit * 25) / 100; diff --git a/desktop/options.h b/desktop/options.h index f01261ee1..e734c5294 100644 --- a/desktop/options.h +++ b/desktop/options.h @@ -208,6 +208,14 @@ NSOPTION_INTEGER(max_fetchers_per_host, 5) */ NSOPTION_INTEGER(max_cached_fetch_handles, 6) +/** Number of times to retry timed-out fetches before giving up. */ +NSOPTION_UINT(max_retried_fetches, 3) + +/** Number of seconds to allow for a DNS-resolution+connect() before timing out + * the cURL socket. + */ +NSOPTION_UINT(curl_fetch_timeout, 10) + /** Suppress debug output from cURL. */ NSOPTION_BOOL(suppress_curl_debug, true) diff --git a/utils/nsoption.c b/utils/nsoption.c index 1309ed875..ccdd23ae9 100644 --- a/utils/nsoption.c +++ b/utils/nsoption.c @@ -181,6 +181,20 @@ static void nsoption_validate(struct nsoption_s *opts, struct nsoption_s *defs) opts[cloop].value.c = defs[cloop].value.c; } } + + /* To aid migration and ensure that timeouts don't go crazy, + * ensure that (a) we allow at least 1 attempt and + * (b) the total time that we spend should not exceed 60s + */ + if (opts[NSOPTION_max_retried_fetches].value.u == 0) + opts[NSOPTION_max_retried_fetches].value.u = 1; + if (opts[NSOPTION_curl_fetch_timeout].value.u < 5) + opts[NSOPTION_max_retried_fetches].value.u = 5; + if (opts[NSOPTION_curl_fetch_timeout].value.u > 60) + opts[NSOPTION_max_retried_fetches].value.u = 60; + while ((opts[NSOPTION_curl_fetch_timeout].value.u * + opts[NSOPTION_max_retried_fetches].value.u) > 60) + opts[NSOPTION_max_retried_fetches].value.u--; } /** |