Add support for retrying timed-out cURL fetches.

This is an attempt to amelioriate the situation found in #2384 where
we see the cURL connect() failing to complete.  Based on the pcap
from the bug log, we believe that RISC OS is likely failing to signal
the completion of the connection to cURL.  As such, cURL times out.

This change permits retries of timed out connections in the hope that
a fresh socket FD might subsequently function correctly.  The defaults
chosen mean that the previous behaviour of 30 seconds before timeout
is reported will remain the same, but in that time we will make 3 separate
attempts to connect the socket.

content/fetch.h

@@ -37,6 +37,7 @@ typedef enum {
 	FETCH_HEADER,
 	FETCH_DATA,
 	FETCH_FINISHED,
+	FETCH_TIMEDOUT,
 	FETCH_ERROR,
 	FETCH_REDIRECT,
 	FETCH_NOTMODIFIED,

content/fetchers/curl.c

@@ -973,11 +973,19 @@ static void fetch_curl_done(CURL *curl_handle, CURLcode result)
 		msg.data.cert_err.num_certs = i;
 		fetch_send_callback(&msg, f->fetch_handle);
 	} else if (error) {
-		if (result != CURLE_SSL_CONNECT_ERROR) {
+		switch (result) {
+		case CURLE_SSL_CONNECT_ERROR:
+			msg.type = FETCH_SSL_ERR;
+			break;
+
+		case CURLE_OPERATION_TIMEDOUT:
+			msg.type = FETCH_TIMEDOUT;
+			msg.data.error = curl_easy_strerror(result);
+			break;
+
+		default:
 			msg.type = FETCH_ERROR;
 			msg.data.error = curl_easy_strerror(result);
-		} else {
-			msg.type = FETCH_SSL_ERR;
 		}
 
 		fetch_send_callback(&msg, f->fetch_handle);
@@ -1302,7 +1310,7 @@ nserror fetch_curl_register(void)
 	SETOPT(CURLOPT_LOW_SPEED_LIMIT, 1L);
 	SETOPT(CURLOPT_LOW_SPEED_TIME, 180L);
 	SETOPT(CURLOPT_NOSIGNAL, 1L);
-	SETOPT(CURLOPT_CONNECTTIMEOUT, 30L);
+	SETOPT(CURLOPT_CONNECTTIMEOUT, nsoption_uint(curl_fetch_timeout));
 
 	if (nsoption_charp(ca_bundle) &&
 	    strcmp(nsoption_charp(ca_bundle), "")) {

content/llcache.c

@@ -120,6 +120,8 @@ typedef struct {
 
 	uint32_t redirect_count;	/**< Count of redirects followed */
 
+	uint32_t retries_remaining;     /**< Number of times to retry on timeout */
+
 	bool tried_with_auth;		/**< Whether we've tried with auth */
 
 	bool tried_with_tls_downgrade;	/**< Whether we've tried TLS <= 1.0 */
@@ -227,6 +229,9 @@ struct llcache_s {
 	/** The target upper bound for the RAM cache size */
 	uint32_t limit;
 
+	/** The number of fetch attempts we make when timing out */
+	uint32_t fetch_attempts;
+
 	/** Whether or not our users are caught up */
 	bool all_caught_up;
 
@@ -909,6 +914,7 @@ static nserror llcache_object_fetch(llcache_object *object, uint32_t flags,
 	object->fetch.referer = referer_clone;
 	object->fetch.post = post_clone;
 	object->fetch.redirect_count = redirect_count;
+	object->fetch.retries_remaining = llcache->fetch_attempts;
 
 	return llcache_object_refetch(object);
 }
@@ -2653,6 +2659,18 @@ static void llcache_fetch_callback(const fetch_msg *msg, void *p)
 		break;
 
 	/* Out-of-band information */
+	case FETCH_TIMEDOUT:
+		/* Timed out while trying to fetch. */
+		/* The fetch has already been cleaned up by the fetcher but
+		 * we would like to retry if we can. */
+		if (object->fetch.retries_remaining > 1) {
+			object->fetch.retries_remaining--;
+			error = llcache_object_refetch(object);
+			break;
+		}
+		/* Otherwise fall through to error, setting the message to
+		 * a timeout
+		 */
 	case FETCH_ERROR:
 		/* An error occurred while fetching */
 		/* The fetch has has already been cleaned up by the fetcher */
@@ -3303,6 +3321,7 @@ llcache_initialise(const struct llcache_parameters *prm)
 	llcache->minimum_bandwidth = prm->minimum_bandwidth;
 	llcache->maximum_bandwidth = prm->maximum_bandwidth;
 	llcache->time_quantum = prm->time_quantum;
+	llcache->fetch_attempts = prm->fetch_attempts;
 	llcache->all_caught_up = true;
 
 	LOG("llcache initialising with a limit of %d bytes", llcache->limit);

content/llcache.h

@@ -230,6 +230,9 @@ struct llcache_parameters {
 	 */
 	unsigned long time_quantum;
 
+	/** The number of fetches to attempt when timing out */
+	uint32_t fetch_attempts;
+
 	struct llcache_store_parameters store;
 };
 

desktop/netsurf.c

@@ -164,6 +164,9 @@ nserror netsurf_init(const char *store_path)
 		LOG("Setting minimum memory cache size %zd", hlcache_parameters.llcache.limit);
 	} 
 
+	/* Set up the max attempts made to fetch a timing out resource */
+	hlcache_parameters.llcache.fetch_attempts = nsoption_uint(max_retried_fetches);
+
 	/* image cache is 25% of total memory cache size */
 	image_cache_parameters.limit = (hlcache_parameters.llcache.limit * 25) / 100;
 

desktop/options.h

@@ -208,6 +208,14 @@ NSOPTION_INTEGER(max_fetchers_per_host, 5)
  */
 NSOPTION_INTEGER(max_cached_fetch_handles, 6)
 
+/** Number of times to retry timed-out fetches before giving up. */
+NSOPTION_UINT(max_retried_fetches, 3)
+
+/** Number of seconds to allow for a DNS-resolution+connect() before timing out
+ * the cURL socket.
+ */
+NSOPTION_UINT(curl_fetch_timeout, 10)
+
 /** Suppress debug output from cURL. */
 NSOPTION_BOOL(suppress_curl_debug, true)
 

utils/nsoption.c

@@ -181,6 +181,20 @@ static void nsoption_validate(struct nsoption_s *opts, struct nsoption_s *defs)
 			opts[cloop].value.c = defs[cloop].value.c;
 		}
 	}
+
+	/* To aid migration and ensure that timeouts don't go crazy,
+	 * ensure that (a) we allow at least 1 attempt and
+	 * (b) the total time that we spend should not exceed 60s
+	 */
+	if (opts[NSOPTION_max_retried_fetches].value.u == 0)
+		opts[NSOPTION_max_retried_fetches].value.u = 1;
+	if (opts[NSOPTION_curl_fetch_timeout].value.u < 5)
+		opts[NSOPTION_max_retried_fetches].value.u = 5;
+	if (opts[NSOPTION_curl_fetch_timeout].value.u > 60)
+		opts[NSOPTION_max_retried_fetches].value.u = 60;
+	while ((opts[NSOPTION_curl_fetch_timeout].value.u *
+		opts[NSOPTION_max_retried_fetches].value.u) > 60)
+		opts[NSOPTION_max_retried_fetches].value.u--;
 }
 
 /**