2003-06-30 05:44:03 -07:00
|
|
|
/*
|
2007-06-10 10:46:44 -07:00
|
|
|
* Copyright 2006,2007 Daniel Silverstone <dsilvers@digital-scurf.org>
|
2007-05-01 19:38:38 -07:00
|
|
|
* Copyright 2007 James Bursa <bursa@users.sourceforge.net>
|
2003-06-30 05:44:03 -07:00
|
|
|
* Copyright 2003 Phil Mellor <monkeyson@users.sourceforge.net>
|
2007-08-08 09:16:03 -07:00
|
|
|
*
|
|
|
|
* This file is part of NetSurf, http://www.netsurf-browser.org/
|
|
|
|
*
|
|
|
|
* NetSurf is free software; you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License as published by
|
|
|
|
* the Free Software Foundation; version 2 of the License.
|
|
|
|
*
|
|
|
|
* NetSurf is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
2003-06-30 05:44:03 -07:00
|
|
|
*/
|
|
|
|
|
2014-11-10 09:00:15 -08:00
|
|
|
/**
|
|
|
|
* \file
|
2014-06-19 10:27:24 -07:00
|
|
|
* Implementation of fetching of data from a URL.
|
|
|
|
*
|
|
|
|
* The implementation is the fetch factory and the generic operations
|
|
|
|
* around the fetcher specific methods.
|
2003-04-17 14:35:02 -07:00
|
|
|
*
|
2007-01-02 05:07:28 -08:00
|
|
|
* Active fetches are held in the circular linked list ::fetch_ring. There may
|
2017-02-19 03:32:17 -08:00
|
|
|
* be at most nsoption max_fetchers_per_host active requests per Host: header.
|
|
|
|
* There may be at most nsoption max_fetchers active requests overall. Inactive
|
2014-06-19 10:27:24 -07:00
|
|
|
* fetches are stored in the ::queue_ring waiting for use.
|
2003-02-09 04:58:15 -08:00
|
|
|
*/
|
|
|
|
|
2016-04-21 14:36:21 -07:00
|
|
|
#include <stdlib.h>
|
2003-02-09 04:58:15 -08:00
|
|
|
#include <assert.h>
|
2007-05-01 19:38:38 -07:00
|
|
|
#include <errno.h>
|
2003-08-29 05:57:14 -07:00
|
|
|
#include <stdbool.h>
|
2003-03-15 07:53:20 -08:00
|
|
|
#include <string.h>
|
2003-04-17 14:35:02 -07:00
|
|
|
#include <strings.h>
|
2003-02-09 04:58:15 -08:00
|
|
|
#include <time.h>
|
2010-12-20 02:32:15 -08:00
|
|
|
#include <libwapcaplet/libwapcaplet.h>
|
|
|
|
|
2007-05-30 15:39:54 -07:00
|
|
|
#include "utils/config.h"
|
2014-01-24 11:25:07 -08:00
|
|
|
#include "utils/corestrings.h"
|
2013-05-25 14:46:27 -07:00
|
|
|
#include "utils/nsoption.h"
|
2007-05-30 15:39:54 -07:00
|
|
|
#include "utils/log.h"
|
|
|
|
#include "utils/messages.h"
|
2011-09-27 04:07:32 -07:00
|
|
|
#include "utils/nsurl.h"
|
2007-06-10 10:46:44 -07:00
|
|
|
#include "utils/ring.h"
|
2016-05-30 03:23:32 -07:00
|
|
|
#include "netsurf/misc.h"
|
2014-10-16 01:48:09 -07:00
|
|
|
#include "desktop/gui_internal.h"
|
2003-09-17 16:27:33 -07:00
|
|
|
|
2014-06-19 10:27:24 -07:00
|
|
|
#include "content/fetch.h"
|
|
|
|
#include "content/fetchers.h"
|
|
|
|
#include "content/fetchers/resource.h"
|
2020-09-20 02:54:48 -07:00
|
|
|
#include "content/fetchers/about/about.h"
|
2014-06-19 10:27:24 -07:00
|
|
|
#include "content/fetchers/curl.h"
|
|
|
|
#include "content/fetchers/data.h"
|
2020-07-03 10:07:21 -07:00
|
|
|
#include "content/fetchers/file/file.h"
|
2014-08-25 15:58:56 -07:00
|
|
|
#include "javascript/fetcher.h"
|
2014-06-19 10:27:24 -07:00
|
|
|
#include "content/urldb.h"
|
|
|
|
|
2014-07-02 09:07:05 -07:00
|
|
|
/** The maximum number of fetchers that can be added */
|
2014-09-04 16:40:52 -07:00
|
|
|
#define MAX_FETCHERS 10
|
2014-07-02 09:07:05 -07:00
|
|
|
|
|
|
|
/** The time in ms between polling the fetchers.
|
|
|
|
*
|
|
|
|
* \todo The schedule timeout should be profiled to see if there is a
|
|
|
|
* better value or even if it needs to be dynamic.
|
|
|
|
*/
|
|
|
|
#define SCHEDULE_TIME 10
|
|
|
|
|
|
|
|
/** The fdset timeout in ms */
|
|
|
|
#define FDSET_TIMEOUT 1000
|
|
|
|
|
2014-06-19 10:27:24 -07:00
|
|
|
/**
|
|
|
|
* Information about a fetcher for a given scheme.
|
|
|
|
*/
|
2007-06-10 10:46:44 -07:00
|
|
|
typedef struct scheme_fetcher_s {
|
2014-06-19 10:27:24 -07:00
|
|
|
lwc_string *scheme; /**< The scheme. */
|
|
|
|
|
|
|
|
struct fetcher_operation_table ops; /**< The fetchers operations. */
|
|
|
|
int refcount; /**< When zero the fetcher is no longer in use. */
|
2007-06-10 10:46:44 -07:00
|
|
|
} scheme_fetcher;
|
|
|
|
|
2014-06-19 10:27:24 -07:00
|
|
|
static scheme_fetcher fetchers[MAX_FETCHERS];
|
2006-02-23 07:06:54 -08:00
|
|
|
|
2003-09-17 16:27:33 -07:00
|
|
|
/** Information for a single fetch. */
|
|
|
|
struct fetch {
|
2007-06-10 10:46:44 -07:00
|
|
|
fetch_callback callback;/**< Callback function. */
|
2011-09-27 04:07:32 -07:00
|
|
|
nsurl *url; /**< URL. */
|
|
|
|
nsurl *referer; /**< Referer URL. */
|
2008-01-30 11:56:41 -08:00
|
|
|
bool verifiable; /**< Transaction is verifiable */
|
2003-09-17 16:27:33 -07:00
|
|
|
void *p; /**< Private data for callback. */
|
2011-09-27 04:07:32 -07:00
|
|
|
lwc_string *host; /**< Host part of URL, interned */
|
2007-01-12 16:19:02 -08:00
|
|
|
long http_code; /**< HTTP response code, or 0. */
|
2014-06-19 10:27:24 -07:00
|
|
|
int fetcherd; /**< Fetcher descriptor for this fetch */
|
2007-07-04 11:05:16 -07:00
|
|
|
void *fetcher_handle; /**< The handle for the fetcher. */
|
|
|
|
bool fetch_is_active; /**< This fetch is active. */
|
2020-02-24 01:47:52 -08:00
|
|
|
fetch_msg_type last_msg;/**< The last message sent for this fetch */
|
2006-03-14 06:21:01 -08:00
|
|
|
struct fetch *r_prev; /**< Previous active fetch in ::fetch_ring. */
|
|
|
|
struct fetch *r_next; /**< Next active fetch in ::fetch_ring. */
|
|
|
|
};
|
|
|
|
|
2014-06-19 10:27:24 -07:00
|
|
|
static struct fetch *fetch_ring = NULL; /**< Ring of active fetches. */
|
|
|
|
static struct fetch *queue_ring = NULL; /**< Ring of queued fetches */
|
2006-03-14 06:21:01 -08:00
|
|
|
|
2014-01-19 10:17:32 -08:00
|
|
|
/******************************************************************************
|
|
|
|
* fetch internals *
|
|
|
|
******************************************************************************/
|
|
|
|
|
2014-06-19 10:27:24 -07:00
|
|
|
static inline void fetch_ref_fetcher(int fetcherd)
|
2014-01-19 10:17:32 -08:00
|
|
|
{
|
2014-06-19 10:27:24 -07:00
|
|
|
fetchers[fetcherd].refcount++;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void fetch_unref_fetcher(int fetcherd)
|
|
|
|
{
|
|
|
|
fetchers[fetcherd].refcount--;
|
|
|
|
if (fetchers[fetcherd].refcount == 0) {
|
|
|
|
fetchers[fetcherd].ops.finalise(fetchers[fetcherd].scheme);
|
|
|
|
lwc_string_unref(fetchers[fetcherd].scheme);
|
2014-01-19 10:17:32 -08:00
|
|
|
}
|
|
|
|
}
|
2003-02-09 04:58:15 -08:00
|
|
|
|
2014-06-19 10:27:24 -07:00
|
|
|
/**
|
|
|
|
* Find a suitable fetcher for a scheme.
|
|
|
|
*/
|
|
|
|
static int get_fetcher_for_scheme(lwc_string *scheme)
|
|
|
|
{
|
|
|
|
int fetcherd;
|
|
|
|
bool match;
|
|
|
|
|
|
|
|
for (fetcherd = 0; fetcherd < MAX_FETCHERS; fetcherd++) {
|
|
|
|
if ((fetchers[fetcherd].refcount > 0) &&
|
|
|
|
(lwc_string_isequal(fetchers[fetcherd].scheme,
|
|
|
|
scheme, &match) == lwc_error_ok) &&
|
|
|
|
(match == true)) {
|
|
|
|
return fetcherd;
|
2014-06-26 11:04:14 -07:00
|
|
|
}
|
2014-06-19 10:27:24 -07:00
|
|
|
}
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2003-02-09 04:58:15 -08:00
|
|
|
/**
|
2014-01-19 10:17:32 -08:00
|
|
|
* Dispatch a single job
|
2003-02-09 04:58:15 -08:00
|
|
|
*/
|
2014-01-19 10:17:32 -08:00
|
|
|
static bool fetch_dispatch_job(struct fetch *fetch)
|
|
|
|
{
|
|
|
|
RING_REMOVE(queue_ring, fetch);
|
2017-09-05 06:08:48 -07:00
|
|
|
NSLOG(fetch, DEBUG,
|
|
|
|
"Attempting to start fetch %p, fetcher %p, url %s", fetch,
|
|
|
|
fetch->fetcher_handle,
|
|
|
|
nsurl_access(fetch->url));
|
2014-06-26 11:04:14 -07:00
|
|
|
|
2014-06-19 10:27:24 -07:00
|
|
|
if (!fetchers[fetch->fetcherd].ops.start(fetch->fetcher_handle)) {
|
2014-01-19 10:17:32 -08:00
|
|
|
RING_INSERT(queue_ring, fetch); /* Put it back on the end of the queue */
|
|
|
|
return false;
|
|
|
|
} else {
|
|
|
|
RING_INSERT(fetch_ring, fetch);
|
|
|
|
fetch->fetch_is_active = true;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
2003-02-09 04:58:15 -08:00
|
|
|
|
2014-01-19 10:17:32 -08:00
|
|
|
/**
|
|
|
|
* Choose and dispatch a single job. Return false if we failed to dispatch
|
|
|
|
* anything.
|
|
|
|
*
|
|
|
|
* We don't check the overall dispatch size here because we're not called unless
|
|
|
|
* there is room in the fetch queue for us.
|
|
|
|
*/
|
|
|
|
static bool fetch_choose_and_dispatch(void)
|
2003-02-09 04:58:15 -08:00
|
|
|
{
|
2014-01-19 10:17:32 -08:00
|
|
|
bool same_host;
|
|
|
|
struct fetch *queueitem;
|
|
|
|
queueitem = queue_ring;
|
|
|
|
do {
|
|
|
|
/* We can dispatch the selected item if there is room in the
|
|
|
|
* fetch ring
|
|
|
|
*/
|
|
|
|
int countbyhost;
|
|
|
|
RING_COUNTBYLWCHOST(struct fetch, fetch_ring, countbyhost,
|
|
|
|
queueitem->host);
|
|
|
|
if (countbyhost < nsoption_int(max_fetchers_per_host)) {
|
|
|
|
/* We can dispatch this item in theory */
|
|
|
|
return fetch_dispatch_job(queueitem);
|
|
|
|
}
|
|
|
|
/* skip over other items with the same host */
|
|
|
|
same_host = true;
|
|
|
|
while (same_host == true && queueitem->r_next != queue_ring) {
|
|
|
|
if (lwc_string_isequal(queueitem->host,
|
|
|
|
queueitem->r_next->host, &same_host) ==
|
|
|
|
lwc_error_ok && same_host == true) {
|
|
|
|
queueitem = queueitem->r_next;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
queueitem = queueitem->r_next;
|
|
|
|
} while (queueitem != queue_ring);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2014-06-26 11:04:14 -07:00
|
|
|
static void dump_rings(void)
|
2014-01-19 10:17:32 -08:00
|
|
|
{
|
|
|
|
struct fetch *q;
|
|
|
|
struct fetch *f;
|
|
|
|
|
|
|
|
q = queue_ring;
|
|
|
|
if (q) {
|
|
|
|
do {
|
2017-09-05 06:08:48 -07:00
|
|
|
NSLOG(fetch, DEBUG, "queue_ring: %s",
|
Use coccinelle to change logging macro calls in c files
for F in $(git ls-files '*.c');do spatch --sp-file foo.cocci --in-place ${F};done
@@ expression E; @@
-LOG(E);
+NSLOG(netsurf, INFO, E);
@@ expression E, E1; @@
-LOG(E, E1);
+NSLOG(netsurf, INFO, E, E1);
@@ expression E, E1, E2; @@
-LOG(E, E1, E2);
+NSLOG(netsurf, INFO, E, E1, E2);
@@ expression E, E1, E2, E3; @@
-LOG(E, E1, E2, E3);
+NSLOG(netsurf, INFO, E, E1, E2, E3);
@@ expression E, E1, E2, E3, E4; @@
-LOG(E, E1, E2, E3, E4);
+NSLOG(netsurf, INFO, E, E1, E2, E3, E4);
@@ expression E, E1, E2, E3, E4, E5; @@
-LOG(E, E1, E2, E3, E4, E5);
+NSLOG(netsurf, INFO, E, E1, E2, E3, E4, E5);
@@ expression E, E1, E2, E3, E4, E5, E6; @@
-LOG(E, E1, E2, E3, E4, E5, E6);
+NSLOG(netsurf, INFO, E, E1, E2, E3, E4, E5, E6);
@@ expression E, E1, E2, E3, E4, E5, E6, E7; @@
-LOG(E, E1, E2, E3, E4, E5, E6, E7);
+NSLOG(netsurf, INFO, E, E1, E2, E3, E4, E5, E6, E7);
2017-09-06 10:28:12 -07:00
|
|
|
nsurl_access(q->url));
|
2014-01-19 10:17:32 -08:00
|
|
|
q = q->r_next;
|
|
|
|
} while (q != queue_ring);
|
|
|
|
}
|
|
|
|
f = fetch_ring;
|
|
|
|
if (f) {
|
|
|
|
do {
|
2017-09-05 06:08:48 -07:00
|
|
|
NSLOG(fetch, DEBUG, "fetch_ring: %s",
|
Use coccinelle to change logging macro calls in c files
for F in $(git ls-files '*.c');do spatch --sp-file foo.cocci --in-place ${F};done
@@ expression E; @@
-LOG(E);
+NSLOG(netsurf, INFO, E);
@@ expression E, E1; @@
-LOG(E, E1);
+NSLOG(netsurf, INFO, E, E1);
@@ expression E, E1, E2; @@
-LOG(E, E1, E2);
+NSLOG(netsurf, INFO, E, E1, E2);
@@ expression E, E1, E2, E3; @@
-LOG(E, E1, E2, E3);
+NSLOG(netsurf, INFO, E, E1, E2, E3);
@@ expression E, E1, E2, E3, E4; @@
-LOG(E, E1, E2, E3, E4);
+NSLOG(netsurf, INFO, E, E1, E2, E3, E4);
@@ expression E, E1, E2, E3, E4, E5; @@
-LOG(E, E1, E2, E3, E4, E5);
+NSLOG(netsurf, INFO, E, E1, E2, E3, E4, E5);
@@ expression E, E1, E2, E3, E4, E5, E6; @@
-LOG(E, E1, E2, E3, E4, E5, E6);
+NSLOG(netsurf, INFO, E, E1, E2, E3, E4, E5, E6);
@@ expression E, E1, E2, E3, E4, E5, E6, E7; @@
-LOG(E, E1, E2, E3, E4, E5, E6, E7);
+NSLOG(netsurf, INFO, E, E1, E2, E3, E4, E5, E6, E7);
2017-09-06 10:28:12 -07:00
|
|
|
nsurl_access(f->url));
|
2014-01-19 10:17:32 -08:00
|
|
|
f = f->r_next;
|
|
|
|
} while (f != fetch_ring);
|
|
|
|
}
|
2014-06-26 11:04:14 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Dispatch as many jobs as we have room to dispatch.
|
|
|
|
*
|
|
|
|
* @return true if there are active fetchers that require polling else false.
|
|
|
|
*/
|
|
|
|
static bool fetch_dispatch_jobs(void)
|
|
|
|
{
|
|
|
|
int all_active;
|
|
|
|
int all_queued;
|
|
|
|
|
|
|
|
RING_GETSIZE(struct fetch, queue_ring, all_queued);
|
|
|
|
RING_GETSIZE(struct fetch, fetch_ring, all_active);
|
|
|
|
|
2017-09-05 06:08:48 -07:00
|
|
|
NSLOG(fetch, DEBUG,
|
|
|
|
"queue_ring %i, fetch_ring %i",
|
|
|
|
all_queued,
|
|
|
|
all_active);
|
2014-06-26 11:04:14 -07:00
|
|
|
dump_rings();
|
2014-01-19 10:17:32 -08:00
|
|
|
|
2014-06-26 11:04:14 -07:00
|
|
|
while ((all_queued != 0) &&
|
|
|
|
(all_active < nsoption_int(max_fetchers)) &&
|
|
|
|
fetch_choose_and_dispatch()) {
|
2014-01-19 10:17:32 -08:00
|
|
|
all_queued--;
|
|
|
|
all_active++;
|
2017-09-05 06:08:48 -07:00
|
|
|
NSLOG(fetch, DEBUG,
|
|
|
|
"%d queued, %d fetching",
|
|
|
|
all_queued,
|
|
|
|
all_active);
|
2014-06-26 11:04:14 -07:00
|
|
|
}
|
|
|
|
|
2017-09-05 06:08:48 -07:00
|
|
|
NSLOG(fetch, DEBUG, "Fetch ring is now %d elements.", all_active);
|
|
|
|
NSLOG(fetch, DEBUG, "Queue ring is now %d elements.", all_queued);
|
2014-06-26 11:04:14 -07:00
|
|
|
|
|
|
|
return (all_active > 0);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void fetcher_poll(void *unused)
|
|
|
|
{
|
|
|
|
int fetcherd;
|
|
|
|
|
|
|
|
if (fetch_dispatch_jobs()) {
|
2017-09-05 06:08:48 -07:00
|
|
|
NSLOG(fetch, DEBUG, "Polling fetchers");
|
2014-06-26 11:04:14 -07:00
|
|
|
for (fetcherd = 0; fetcherd < MAX_FETCHERS; fetcherd++) {
|
|
|
|
if (fetchers[fetcherd].refcount > 0) {
|
|
|
|
/* fetcher present */
|
|
|
|
fetchers[fetcherd].ops.poll(fetchers[fetcherd].scheme);
|
|
|
|
}
|
2014-01-19 10:17:32 -08:00
|
|
|
}
|
2014-06-26 11:04:14 -07:00
|
|
|
|
|
|
|
/* schedule active fetchers to run again in 10ms */
|
2016-04-15 14:50:28 -07:00
|
|
|
guit->misc->schedule(SCHEDULE_TIME, fetcher_poll, NULL);
|
2014-01-19 10:17:32 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/******************************************************************************
|
|
|
|
* Public API *
|
|
|
|
******************************************************************************/
|
|
|
|
|
|
|
|
/* exported interface documented in content/fetch.h */
|
2014-06-19 10:27:24 -07:00
|
|
|
nserror fetcher_init(void)
|
2014-01-19 10:17:32 -08:00
|
|
|
{
|
2014-10-25 09:10:08 -07:00
|
|
|
nserror ret;
|
|
|
|
|
2016-08-13 05:09:39 -07:00
|
|
|
#ifdef WITH_CURL
|
2014-10-25 09:10:08 -07:00
|
|
|
ret = fetch_curl_register();
|
|
|
|
if (ret != NSERROR_OK) {
|
|
|
|
return ret;
|
|
|
|
}
|
2016-08-13 05:09:39 -07:00
|
|
|
#endif
|
2014-10-25 09:10:08 -07:00
|
|
|
|
|
|
|
ret = fetch_data_register();
|
|
|
|
if (ret != NSERROR_OK) {
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = fetch_file_register();
|
|
|
|
if (ret != NSERROR_OK) {
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = fetch_resource_register();
|
|
|
|
if (ret != NSERROR_OK) {
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = fetch_about_register();
|
|
|
|
if (ret != NSERROR_OK) {
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = fetch_javascript_register();
|
|
|
|
|
|
|
|
return ret;
|
2014-01-19 10:17:32 -08:00
|
|
|
}
|
2003-02-09 04:58:15 -08:00
|
|
|
|
2014-06-19 10:27:24 -07:00
|
|
|
/* exported interface documented in content/fetchers.h */
|
|
|
|
void fetcher_quit(void)
|
2003-02-09 04:58:15 -08:00
|
|
|
{
|
2014-06-19 10:27:24 -07:00
|
|
|
int fetcherd; /* fetcher index */
|
|
|
|
for (fetcherd = 0; fetcherd < MAX_FETCHERS; fetcherd++) {
|
2014-07-02 06:50:23 -07:00
|
|
|
if (fetchers[fetcherd].refcount > 1) {
|
|
|
|
/* fetcher still has reference at quit. This
|
|
|
|
* should not happen as the fetch should have
|
|
|
|
* been aborted in llcache shutdown.
|
|
|
|
*
|
|
|
|
* This appears to be normal behaviour if a
|
|
|
|
* curl operation is still in progress at exit
|
|
|
|
* as the abort waits for curl to complete.
|
|
|
|
*
|
|
|
|
* We could make the user wait for curl to
|
|
|
|
* complete but we are exiting anyway so thats
|
|
|
|
* unhelpful. Instead we just log it and force
|
|
|
|
* the reference count to allow the fetcher to
|
|
|
|
* be stopped.
|
|
|
|
*/
|
2017-09-05 06:08:48 -07:00
|
|
|
NSLOG(fetch, INFO,
|
Use coccinelle to change logging macro calls in c files
for F in $(git ls-files '*.c');do spatch --sp-file foo.cocci --in-place ${F};done
@@ expression E; @@
-LOG(E);
+NSLOG(netsurf, INFO, E);
@@ expression E, E1; @@
-LOG(E, E1);
+NSLOG(netsurf, INFO, E, E1);
@@ expression E, E1, E2; @@
-LOG(E, E1, E2);
+NSLOG(netsurf, INFO, E, E1, E2);
@@ expression E, E1, E2, E3; @@
-LOG(E, E1, E2, E3);
+NSLOG(netsurf, INFO, E, E1, E2, E3);
@@ expression E, E1, E2, E3, E4; @@
-LOG(E, E1, E2, E3, E4);
+NSLOG(netsurf, INFO, E, E1, E2, E3, E4);
@@ expression E, E1, E2, E3, E4, E5; @@
-LOG(E, E1, E2, E3, E4, E5);
+NSLOG(netsurf, INFO, E, E1, E2, E3, E4, E5);
@@ expression E, E1, E2, E3, E4, E5, E6; @@
-LOG(E, E1, E2, E3, E4, E5, E6);
+NSLOG(netsurf, INFO, E, E1, E2, E3, E4, E5, E6);
@@ expression E, E1, E2, E3, E4, E5, E6, E7; @@
-LOG(E, E1, E2, E3, E4, E5, E6, E7);
+NSLOG(netsurf, INFO, E, E1, E2, E3, E4, E5, E6, E7);
2017-09-06 10:28:12 -07:00
|
|
|
"Fetcher for scheme %s still has %d active users at quit.",
|
|
|
|
lwc_string_data(fetchers[fetcherd].scheme),
|
|
|
|
fetchers[fetcherd].refcount);
|
2014-07-02 06:50:23 -07:00
|
|
|
|
2014-07-02 07:01:24 -07:00
|
|
|
fetchers[fetcherd].refcount = 1;
|
2014-07-02 06:50:23 -07:00
|
|
|
}
|
|
|
|
if (fetchers[fetcherd].refcount == 1) {
|
2014-06-19 10:27:24 -07:00
|
|
|
|
|
|
|
fetch_unref_fetcher(fetcherd);
|
2007-07-04 11:05:16 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-06-19 10:27:24 -07:00
|
|
|
/* exported interface documented in content/fetchers.h */
|
|
|
|
nserror
|
|
|
|
fetcher_add(lwc_string *scheme, const struct fetcher_operation_table *ops)
|
2007-07-04 11:05:16 -07:00
|
|
|
{
|
2014-06-19 10:27:24 -07:00
|
|
|
int fetcherd;
|
|
|
|
|
|
|
|
/* find unused fetcher descriptor */
|
|
|
|
for (fetcherd = 0; fetcherd < MAX_FETCHERS; fetcherd++) {
|
|
|
|
if (fetchers[fetcherd].refcount == 0) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (fetcherd == MAX_FETCHERS) {
|
|
|
|
return NSERROR_INIT_FAILED;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!ops->initialise(scheme)) {
|
|
|
|
return NSERROR_INIT_FAILED;
|
2007-07-04 11:05:16 -07:00
|
|
|
}
|
2014-06-19 10:27:24 -07:00
|
|
|
|
|
|
|
fetchers[fetcherd].scheme = scheme;
|
|
|
|
fetchers[fetcherd].ops = *ops;
|
|
|
|
|
|
|
|
fetch_ref_fetcher(fetcherd);
|
|
|
|
|
|
|
|
return NSERROR_OK;
|
2003-02-09 04:58:15 -08:00
|
|
|
}
|
|
|
|
|
2016-06-27 13:09:39 -07:00
|
|
|
/* exported interface documented in content/fetch.h */
|
|
|
|
nserror
|
|
|
|
fetch_fdset(fd_set *read_fd_set,
|
2014-06-26 11:04:14 -07:00
|
|
|
fd_set *write_fd_set,
|
|
|
|
fd_set *except_fd_set,
|
|
|
|
int *maxfd_out)
|
|
|
|
{
|
2016-06-27 13:00:58 -07:00
|
|
|
int maxfd = -1;
|
2014-06-26 11:04:14 -07:00
|
|
|
int fetcherd; /* fetcher index */
|
|
|
|
|
|
|
|
if (!fetch_dispatch_jobs()) {
|
2017-09-05 06:08:48 -07:00
|
|
|
NSLOG(fetch, DEBUG, "No jobs");
|
2014-06-26 11:04:14 -07:00
|
|
|
*maxfd_out = -1;
|
|
|
|
return NSERROR_OK;
|
|
|
|
}
|
|
|
|
|
2017-09-05 06:08:48 -07:00
|
|
|
NSLOG(fetch, DEBUG, "Polling fetchers");
|
2014-06-26 11:04:14 -07:00
|
|
|
|
|
|
|
for (fetcherd = 0; fetcherd < MAX_FETCHERS; fetcherd++) {
|
|
|
|
if (fetchers[fetcherd].refcount > 0) {
|
|
|
|
/* fetcher present */
|
|
|
|
fetchers[fetcherd].ops.poll(fetchers[fetcherd].scheme);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
FD_ZERO(read_fd_set);
|
|
|
|
FD_ZERO(write_fd_set);
|
|
|
|
FD_ZERO(except_fd_set);
|
2016-06-27 13:00:58 -07:00
|
|
|
|
|
|
|
for (fetcherd = 0; fetcherd < MAX_FETCHERS; fetcherd++) {
|
|
|
|
if ((fetchers[fetcherd].refcount > 0) &&
|
|
|
|
(fetchers[fetcherd].ops.fdset != NULL)) {
|
|
|
|
/* fetcher present */
|
|
|
|
int fetcher_maxfd;
|
|
|
|
fetcher_maxfd = fetchers[fetcherd].ops.fdset(
|
|
|
|
fetchers[fetcherd].scheme, read_fd_set,
|
|
|
|
write_fd_set, except_fd_set);
|
|
|
|
if (fetcher_maxfd > maxfd)
|
|
|
|
maxfd = fetcher_maxfd;
|
|
|
|
}
|
|
|
|
}
|
2014-06-26 11:04:14 -07:00
|
|
|
|
|
|
|
if (maxfd >= 0) {
|
|
|
|
/* change the scheduled poll to happen is a 1000ms as
|
|
|
|
* we assume fetching an fdset means the fetchers will
|
|
|
|
* be run by the client waking up on data available on
|
|
|
|
* the fd and re-calling fetcher_fdset() if this does
|
|
|
|
* not happen the fetch polling will continue as
|
|
|
|
* usual.
|
|
|
|
*/
|
|
|
|
/** @note adjusting the schedule time is only done for
|
|
|
|
* curl currently. This is because as it is assumed to
|
|
|
|
* be the only fetcher that can possibly have fd to
|
|
|
|
* select on. All the other fetchers continue to need
|
|
|
|
* polling frequently.
|
|
|
|
*/
|
2016-04-15 14:50:28 -07:00
|
|
|
guit->misc->schedule(FDSET_TIMEOUT, fetcher_poll, NULL);
|
2014-06-26 11:04:14 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
*maxfd_out = maxfd;
|
|
|
|
|
|
|
|
return NSERROR_OK;
|
|
|
|
}
|
|
|
|
|
2014-01-19 10:17:32 -08:00
|
|
|
/* exported interface documented in content/fetch.h */
|
2015-06-24 02:31:13 -07:00
|
|
|
nserror
|
2014-06-19 10:27:24 -07:00
|
|
|
fetch_start(nsurl *url,
|
|
|
|
nsurl *referer,
|
|
|
|
fetch_callback callback,
|
|
|
|
void *p,
|
|
|
|
bool only_2xx,
|
|
|
|
const char *post_urlenc,
|
|
|
|
const struct fetch_multipart_data *post_multipart,
|
|
|
|
bool verifiable,
|
|
|
|
bool downgrade_tls,
|
2015-06-24 02:31:13 -07:00
|
|
|
const char *headers[],
|
|
|
|
struct fetch **fetch_out)
|
2003-02-09 04:58:15 -08:00
|
|
|
{
|
2004-03-26 16:50:58 -08:00
|
|
|
struct fetch *fetch;
|
2011-09-27 04:07:32 -07:00
|
|
|
lwc_string *scheme;
|
2004-03-26 16:50:58 -08:00
|
|
|
|
2020-03-12 10:22:28 -07:00
|
|
|
fetch = calloc(1, sizeof (*fetch));
|
2014-06-19 10:27:24 -07:00
|
|
|
if (fetch == NULL) {
|
2015-06-24 02:31:13 -07:00
|
|
|
return NSERROR_NOMEM;
|
2014-06-19 10:27:24 -07:00
|
|
|
}
|
2004-03-26 16:50:58 -08:00
|
|
|
|
2009-05-28 04:56:56 -07:00
|
|
|
/* The URL we're fetching must have a scheme */
|
2011-09-27 04:07:32 -07:00
|
|
|
scheme = nsurl_get_component(url, NSURL_SCHEME);
|
2012-10-11 06:38:10 -07:00
|
|
|
assert(scheme != NULL);
|
2006-06-29 15:04:56 -07:00
|
|
|
|
2014-06-19 10:27:24 -07:00
|
|
|
/* try and obtain a fetcher for this scheme */
|
|
|
|
fetch->fetcherd = get_fetcher_for_scheme(scheme);
|
2021-03-25 16:31:37 -07:00
|
|
|
lwc_string_unref(scheme);
|
2014-06-19 10:27:24 -07:00
|
|
|
if (fetch->fetcherd == -1) {
|
2014-07-08 07:57:48 -07:00
|
|
|
free(fetch);
|
2015-06-24 02:31:13 -07:00
|
|
|
return NSERROR_NO_FETCH_HANDLER;
|
2014-06-19 10:27:24 -07:00
|
|
|
}
|
|
|
|
|
2017-09-05 06:08:48 -07:00
|
|
|
NSLOG(fetch, DEBUG, "fetch %p, url '%s'", fetch, nsurl_access(url));
|
2003-04-17 14:35:02 -07:00
|
|
|
|
|
|
|
/* construct a new fetch structure */
|
2003-02-09 04:58:15 -08:00
|
|
|
fetch->callback = callback;
|
2011-09-27 04:07:32 -07:00
|
|
|
fetch->url = nsurl_ref(url);
|
2008-01-30 11:56:41 -08:00
|
|
|
fetch->verifiable = verifiable;
|
2003-02-09 04:58:15 -08:00
|
|
|
fetch->p = p;
|
2011-09-27 04:07:32 -07:00
|
|
|
fetch->host = nsurl_get_component(url, NSURL_HOST);
|
2014-01-19 10:17:32 -08:00
|
|
|
|
2007-07-04 11:05:16 -07:00
|
|
|
if (referer != NULL) {
|
2011-09-27 04:07:32 -07:00
|
|
|
fetch->referer = nsurl_ref(referer);
|
2007-07-04 11:05:16 -07:00
|
|
|
}
|
2003-04-17 14:35:02 -07:00
|
|
|
|
2014-06-19 10:27:24 -07:00
|
|
|
/* try and set up the fetch */
|
|
|
|
fetch->fetcher_handle = fetchers[fetch->fetcherd].ops.setup(fetch, url,
|
|
|
|
only_2xx, downgrade_tls,
|
|
|
|
post_urlenc, post_multipart,
|
|
|
|
headers);
|
|
|
|
if (fetch->fetcher_handle == NULL) {
|
2007-06-10 11:08:22 -07:00
|
|
|
|
2014-06-19 10:27:24 -07:00
|
|
|
if (fetch->host != NULL)
|
|
|
|
lwc_string_unref(fetch->host);
|
2007-06-10 11:08:22 -07:00
|
|
|
|
2014-06-19 10:27:24 -07:00
|
|
|
if (fetch->url != NULL)
|
|
|
|
nsurl_unref(fetch->url);
|
2007-06-10 11:08:22 -07:00
|
|
|
|
2014-06-19 10:27:24 -07:00
|
|
|
if (fetch->referer != NULL)
|
|
|
|
nsurl_unref(fetch->referer);
|
2007-06-10 11:08:22 -07:00
|
|
|
|
2014-06-19 10:27:24 -07:00
|
|
|
free(fetch);
|
|
|
|
|
2015-06-24 02:31:13 -07:00
|
|
|
|
|
|
|
/** \todo The fetchers setup should return nserror and that be
|
|
|
|
* passed back rather than assuming a bad url
|
|
|
|
*/
|
|
|
|
return NSERROR_BAD_URL;
|
2014-06-19 10:27:24 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Rah, got it, so ref the fetcher. */
|
|
|
|
fetch_ref_fetcher(fetch->fetcherd);
|
2007-06-10 11:08:22 -07:00
|
|
|
|
2014-06-26 11:04:14 -07:00
|
|
|
/* Dump new fetch in the queue. */
|
2006-03-14 06:21:01 -08:00
|
|
|
RING_INSERT(queue_ring, fetch);
|
2014-06-26 11:04:14 -07:00
|
|
|
|
|
|
|
/* Ask the queue to run. */
|
|
|
|
if (fetch_dispatch_jobs()) {
|
2017-09-05 06:08:48 -07:00
|
|
|
NSLOG(fetch, DEBUG, "scheduling poll");
|
2014-06-26 11:04:14 -07:00
|
|
|
/* schedule active fetchers to run again in 10ms */
|
2016-04-15 14:50:28 -07:00
|
|
|
guit->misc->schedule(10, fetcher_poll, NULL);
|
2014-06-26 11:04:14 -07:00
|
|
|
}
|
2009-05-28 04:56:56 -07:00
|
|
|
|
2015-06-24 02:31:13 -07:00
|
|
|
*fetch_out = fetch;
|
|
|
|
return NSERROR_OK;
|
2004-03-26 16:50:58 -08:00
|
|
|
}
|
2003-10-25 12:20:13 -07:00
|
|
|
|
2014-01-19 10:17:32 -08:00
|
|
|
/* exported interface documented in content/fetch.h */
|
2003-02-09 04:58:15 -08:00
|
|
|
void fetch_abort(struct fetch *f)
|
2004-06-22 10:37:51 -07:00
|
|
|
{
|
|
|
|
assert(f);
|
2020-02-24 02:24:23 -08:00
|
|
|
f->last_msg = FETCH__INTERNAL_ABORTED;
|
2017-09-05 06:08:48 -07:00
|
|
|
NSLOG(fetch, DEBUG,
|
|
|
|
"fetch %p, fetcher %p, url '%s'", f, f->fetcher_handle,
|
2015-05-28 09:06:18 -07:00
|
|
|
nsurl_access(f->url));
|
2014-06-19 10:27:24 -07:00
|
|
|
fetchers[f->fetcherd].ops.abort(f->fetcher_handle);
|
2004-03-26 16:50:58 -08:00
|
|
|
}
|
|
|
|
|
2014-01-19 10:17:32 -08:00
|
|
|
/* exported interface documented in content/fetch.h */
|
2004-03-26 16:50:58 -08:00
|
|
|
void fetch_free(struct fetch *f)
|
|
|
|
{
|
2020-02-24 01:47:52 -08:00
|
|
|
if (f->last_msg < FETCH_MIN_FINISHED_MSG) {
|
|
|
|
/* We didn't finish, so tell our user that an error occurred */
|
|
|
|
fetch_msg msg;
|
|
|
|
|
|
|
|
msg.type = FETCH_ERROR;
|
|
|
|
msg.data.error = "FetchFailedToFinish";
|
|
|
|
|
|
|
|
NSLOG(fetch, CRITICAL,
|
|
|
|
"During the fetch of %s, the fetcher did not finish.",
|
|
|
|
nsurl_access(f->url));
|
|
|
|
|
|
|
|
fetch_send_callback(&msg, f);
|
|
|
|
}
|
|
|
|
|
2017-09-05 06:08:48 -07:00
|
|
|
NSLOG(fetch, DEBUG,
|
|
|
|
"Freeing fetch %p, fetcher %p",
|
|
|
|
f,
|
|
|
|
f->fetcher_handle);
|
2014-06-26 11:04:14 -07:00
|
|
|
|
2014-06-19 10:27:24 -07:00
|
|
|
fetchers[f->fetcherd].ops.free(f->fetcher_handle);
|
|
|
|
|
|
|
|
fetch_unref_fetcher(f->fetcherd);
|
|
|
|
|
2011-09-27 04:07:32 -07:00
|
|
|
nsurl_unref(f->url);
|
2014-06-26 11:04:14 -07:00
|
|
|
if (f->referer != NULL) {
|
2011-09-27 04:07:32 -07:00
|
|
|
nsurl_unref(f->referer);
|
2014-06-26 11:04:14 -07:00
|
|
|
}
|
|
|
|
if (f->host != NULL) {
|
2011-09-27 04:07:32 -07:00
|
|
|
lwc_string_unref(f->host);
|
2014-06-26 11:04:14 -07:00
|
|
|
}
|
2014-01-19 10:17:32 -08:00
|
|
|
free(f);
|
2003-02-09 04:58:15 -08:00
|
|
|
}
|
|
|
|
|
2007-07-04 11:05:16 -07:00
|
|
|
|
2003-10-25 09:22:11 -07:00
|
|
|
|
2014-01-19 10:17:32 -08:00
|
|
|
/* exported interface documented in content/fetch.h */
|
2011-09-27 07:42:45 -07:00
|
|
|
bool fetch_can_fetch(const nsurl *url)
|
2004-04-02 05:51:13 -08:00
|
|
|
{
|
2011-09-27 04:07:32 -07:00
|
|
|
lwc_string *scheme = nsurl_get_component(url, NSURL_SCHEME);
|
2014-06-19 10:27:24 -07:00
|
|
|
int fetcherd;
|
2011-09-26 14:07:19 -07:00
|
|
|
|
2014-06-19 10:27:24 -07:00
|
|
|
fetcherd = get_fetcher_for_scheme(scheme);
|
|
|
|
lwc_string_unref(scheme);
|
2011-11-27 06:14:36 -08:00
|
|
|
|
2014-06-19 10:27:24 -07:00
|
|
|
if (fetcherd == -1) {
|
|
|
|
return false;
|
2007-07-04 11:05:16 -07:00
|
|
|
}
|
2007-06-10 11:08:22 -07:00
|
|
|
|
2014-06-19 10:27:24 -07:00
|
|
|
return fetchers[fetcherd].ops.acceptable(url);
|
2004-04-02 05:51:13 -08:00
|
|
|
}
|
|
|
|
|
2014-01-19 10:17:32 -08:00
|
|
|
/* exported interface documented in content/fetch.h */
|
2004-06-27 16:24:11 -07:00
|
|
|
void fetch_change_callback(struct fetch *fetch,
|
2007-07-04 11:05:16 -07:00
|
|
|
fetch_callback callback,
|
|
|
|
void *p)
|
2004-06-27 16:24:11 -07:00
|
|
|
{
|
|
|
|
assert(fetch);
|
|
|
|
fetch->callback = callback;
|
|
|
|
fetch->p = p;
|
|
|
|
}
|
|
|
|
|
2014-01-19 10:17:32 -08:00
|
|
|
/* exported interface documented in content/fetch.h */
|
2007-01-12 16:19:02 -08:00
|
|
|
long fetch_http_code(struct fetch *fetch)
|
|
|
|
{
|
|
|
|
return fetch->http_code;
|
|
|
|
}
|
|
|
|
|
2009-02-16 12:24:54 -08:00
|
|
|
|
2014-01-19 10:17:32 -08:00
|
|
|
/* exported interface documented in content/fetch.h */
|
|
|
|
struct fetch_multipart_data *
|
|
|
|
fetch_multipart_data_clone(const struct fetch_multipart_data *list)
|
2010-03-28 05:56:39 -07:00
|
|
|
{
|
|
|
|
struct fetch_multipart_data *clone, *last = NULL;
|
|
|
|
struct fetch_multipart_data *result = NULL;
|
|
|
|
|
|
|
|
for (; list != NULL; list = list->next) {
|
|
|
|
clone = malloc(sizeof(struct fetch_multipart_data));
|
|
|
|
if (clone == NULL) {
|
|
|
|
if (result != NULL)
|
|
|
|
fetch_multipart_data_destroy(result);
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
clone->file = list->file;
|
|
|
|
|
|
|
|
clone->name = strdup(list->name);
|
|
|
|
if (clone->name == NULL) {
|
|
|
|
free(clone);
|
|
|
|
if (result != NULL)
|
|
|
|
fetch_multipart_data_destroy(result);
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
clone->value = strdup(list->value);
|
|
|
|
if (clone->value == NULL) {
|
|
|
|
free(clone->name);
|
|
|
|
free(clone);
|
|
|
|
if (result != NULL)
|
|
|
|
fetch_multipart_data_destroy(result);
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2014-01-04 13:49:03 -08:00
|
|
|
if (clone->file) {
|
|
|
|
clone->rawfile = strdup(list->rawfile);
|
|
|
|
if (clone->rawfile == NULL) {
|
|
|
|
free(clone->value);
|
|
|
|
free(clone->name);
|
|
|
|
free(clone);
|
|
|
|
if (result != NULL)
|
|
|
|
fetch_multipart_data_destroy(result);
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
clone->rawfile = NULL;
|
|
|
|
}
|
|
|
|
|
2010-03-28 05:56:39 -07:00
|
|
|
clone->next = NULL;
|
|
|
|
|
|
|
|
if (result == NULL)
|
|
|
|
result = clone;
|
|
|
|
else
|
|
|
|
last->next = clone;
|
|
|
|
|
|
|
|
last = clone;
|
|
|
|
}
|
|
|
|
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2019-08-06 03:25:11 -07:00
|
|
|
|
|
|
|
/* exported interface documented in content/fetch.h */
|
|
|
|
const char *
|
|
|
|
fetch_multipart_data_find(const struct fetch_multipart_data *list,
|
|
|
|
const char *name)
|
|
|
|
{
|
|
|
|
while (list != NULL) {
|
|
|
|
if (strcmp(list->name, name) == 0) {
|
|
|
|
return list->value;
|
|
|
|
}
|
|
|
|
list = list->next;
|
|
|
|
}
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2014-01-19 10:17:32 -08:00
|
|
|
/* exported interface documented in content/fetch.h */
|
2010-03-28 05:56:39 -07:00
|
|
|
void fetch_multipart_data_destroy(struct fetch_multipart_data *list)
|
|
|
|
{
|
|
|
|
struct fetch_multipart_data *next;
|
|
|
|
|
|
|
|
for (; list != NULL; list = next) {
|
|
|
|
next = list->next;
|
|
|
|
free(list->name);
|
|
|
|
free(list->value);
|
2014-01-04 13:15:52 -08:00
|
|
|
if (list->file) {
|
2017-09-05 06:08:48 -07:00
|
|
|
NSLOG(fetch, DEBUG,
|
|
|
|
"Freeing rawfile: %s", list->rawfile);
|
2014-01-04 11:34:04 -08:00
|
|
|
free(list->rawfile);
|
2014-01-04 13:15:52 -08:00
|
|
|
}
|
2010-03-28 05:56:39 -07:00
|
|
|
free(list);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-08-06 03:25:11 -07:00
|
|
|
|
|
|
|
/* exported interface documented in content/fetch.h */
|
|
|
|
nserror
|
|
|
|
fetch_multipart_data_new_kv(struct fetch_multipart_data **list,
|
|
|
|
const char *name,
|
|
|
|
const char *value)
|
|
|
|
{
|
|
|
|
struct fetch_multipart_data *newdata;
|
|
|
|
|
|
|
|
assert(list);
|
|
|
|
|
|
|
|
newdata = calloc(sizeof(*newdata), 1);
|
|
|
|
|
|
|
|
if (newdata == NULL) {
|
|
|
|
return NSERROR_NOMEM;
|
|
|
|
}
|
|
|
|
|
|
|
|
newdata->name = strdup(name);
|
|
|
|
if (newdata->name == NULL) {
|
|
|
|
free(newdata);
|
|
|
|
return NSERROR_NOMEM;
|
|
|
|
}
|
|
|
|
|
|
|
|
newdata->value = strdup(value);
|
|
|
|
if (newdata->value == NULL) {
|
|
|
|
free(newdata->name);
|
|
|
|
free(newdata);
|
|
|
|
return NSERROR_NOMEM;
|
|
|
|
}
|
|
|
|
|
|
|
|
newdata->next = *list;
|
|
|
|
*list = newdata;
|
|
|
|
|
|
|
|
return NSERROR_OK;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2014-01-19 10:17:32 -08:00
|
|
|
/* exported interface documented in content/fetch.h */
|
2007-06-10 10:46:44 -07:00
|
|
|
void
|
2011-11-08 13:51:42 -08:00
|
|
|
fetch_send_callback(const fetch_msg *msg, struct fetch *fetch)
|
2006-02-23 07:06:54 -08:00
|
|
|
{
|
2020-02-24 09:43:21 -08:00
|
|
|
/* Bump the last_msg to the greatest seen msg */
|
|
|
|
if (msg->type > fetch->last_msg)
|
|
|
|
fetch->last_msg = msg->type;
|
2011-11-08 13:51:42 -08:00
|
|
|
fetch->callback(msg, fetch->p);
|
2006-02-23 07:06:54 -08:00
|
|
|
}
|
|
|
|
|
2007-07-04 11:44:13 -07:00
|
|
|
|
2014-01-19 10:17:32 -08:00
|
|
|
/* exported interface documented in content/fetch.h */
|
2007-07-04 11:44:13 -07:00
|
|
|
void fetch_remove_from_queues(struct fetch *fetch)
|
2006-02-23 07:06:54 -08:00
|
|
|
{
|
2017-09-05 06:08:48 -07:00
|
|
|
int all_active;
|
|
|
|
int all_queued;
|
|
|
|
|
|
|
|
NSLOG(fetch, DEBUG,
|
|
|
|
"Fetch %p, fetcher %p can be freed",
|
|
|
|
fetch,
|
|
|
|
fetch->fetcher_handle);
|
2007-07-04 11:05:16 -07:00
|
|
|
|
2014-01-19 10:17:32 -08:00
|
|
|
/* Go ahead and free the fetch properly now */
|
2007-07-04 11:05:16 -07:00
|
|
|
if (fetch->fetch_is_active) {
|
|
|
|
RING_REMOVE(fetch_ring, fetch);
|
|
|
|
} else {
|
|
|
|
RING_REMOVE(queue_ring, fetch);
|
|
|
|
}
|
|
|
|
|
2014-01-19 10:17:32 -08:00
|
|
|
|
2014-06-26 11:04:14 -07:00
|
|
|
RING_GETSIZE(struct fetch, fetch_ring, all_active);
|
2014-01-19 10:17:32 -08:00
|
|
|
RING_GETSIZE(struct fetch, queue_ring, all_queued);
|
|
|
|
|
2017-09-05 06:08:48 -07:00
|
|
|
NSLOG(fetch, DEBUG, "Fetch ring is now %d elements.", all_active);
|
|
|
|
NSLOG(fetch, DEBUG, "Queue ring is now %d elements.", all_queued);
|
2006-02-23 07:06:54 -08:00
|
|
|
}
|
2003-02-09 04:58:15 -08:00
|
|
|
|
2007-07-04 11:44:13 -07:00
|
|
|
|
2014-01-19 10:17:32 -08:00
|
|
|
/* exported interface documented in content/fetch.h */
|
|
|
|
void fetch_set_http_code(struct fetch *fetch, long http_code)
|
2003-02-09 04:58:15 -08:00
|
|
|
{
|
2017-09-05 06:08:48 -07:00
|
|
|
NSLOG(fetch, DEBUG, "Setting HTTP code to %ld", http_code);
|
2014-06-26 11:04:14 -07:00
|
|
|
|
2007-07-04 11:05:16 -07:00
|
|
|
fetch->http_code = http_code;
|
2003-02-09 04:58:15 -08:00
|
|
|
}
|
|
|
|
|
2008-01-30 11:56:41 -08:00
|
|
|
|
2014-01-19 10:17:32 -08:00
|
|
|
/* exported interface documented in content/fetch.h */
|
|
|
|
void fetch_set_cookie(struct fetch *fetch, const char *data)
|
2008-01-30 11:56:41 -08:00
|
|
|
{
|
|
|
|
assert(fetch && data);
|
|
|
|
|
2010-04-08 04:47:05 -07:00
|
|
|
/* If the fetch is unverifiable err on the side of caution and
|
|
|
|
* do not set the cookie */
|
2008-01-30 11:56:41 -08:00
|
|
|
|
2010-04-08 04:47:05 -07:00
|
|
|
if (fetch->verifiable) {
|
2008-01-30 11:56:41 -08:00
|
|
|
/* If the transaction's verifiable, we don't require
|
|
|
|
* that the request uri and the parent domain match,
|
2010-04-08 04:47:05 -07:00
|
|
|
* so don't pass in any referer/parent in this case. */
|
2012-10-08 12:32:57 -07:00
|
|
|
urldb_set_cookie(data, fetch->url, NULL);
|
2010-04-10 05:24:25 -07:00
|
|
|
} else if (fetch->referer != NULL) {
|
|
|
|
/* Permit the cookie to be set if the fetch is unverifiable
|
|
|
|
* and the fetch URI domain matches the referer. */
|
|
|
|
/** \todo Long-term, this needs to be replaced with a
|
|
|
|
* comparison against the origin fetch URI. In the case
|
|
|
|
* where a nested object requests a fetch, the origin URI
|
|
|
|
* is the nested object's parent URI, whereas the referer
|
|
|
|
* for the fetch will be the nested object's URI. */
|
2012-10-08 12:32:57 -07:00
|
|
|
urldb_set_cookie(data, fetch->url, fetch->referer);
|
2008-01-30 11:56:41 -08:00
|
|
|
}
|
|
|
|
}
|