eukara
/
WebSurf
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

hoist the Referer header generation logic up to the low level cache 

This removes the need for the fetchers to have any interaction with
 the Referer header. It has not been completely removed from the
 fetch interface as fetch.c:fetch_set_cookie() still uses it for
 unverifiable cookie decision logic. (There is an anchient todo here)
This commit is contained in:
Vincent Sanders 2021-03-25 23:31:37 +00:00
parent 537f131ee3
commit 5a2f69388c
5 changed files with 152 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
content/fetch.c
View File

@ -90,7 +90,6 @@ struct fetch {
fetch_callback callback;/**< Callback function. */
nsurl *url; /**< URL. */
nsurl *referer; /**< Referer URL. */
bool send_referer; /**< Valid to send the referer */
bool verifiable; /**< Transaction is verifiable */
void *p; /**< Private data for callback. */
lwc_string *host; /**< Host part of URL, interned */
@ -461,7 +460,6 @@ fetch_start(nsurl *url,
{
struct fetch *fetch;
lwc_string *scheme;
bool match;
fetch = calloc(1, sizeof (*fetch));
if (fetch == NULL) {
@ -474,8 +472,8 @@ fetch_start(nsurl *url,
/* try and obtain a fetcher for this scheme */
fetch->fetcherd = get_fetcher_for_scheme(scheme);
lwc_string_unref(scheme);
if (fetch->fetcherd == -1) {
lwc_string_unref(scheme);
free(fetch);
return NSERROR_NO_FETCH_HANDLER;
}
@ -490,48 +488,9 @@ fetch_start(nsurl *url,
fetch->host = nsurl_get_component(url, NSURL_HOST);
if (referer != NULL) {
lwc_string *ref_scheme;
fetch->referer = nsurl_ref(referer);
ref_scheme = nsurl_get_component(referer, NSURL_SCHEME);
/* Not a problem if referer has no scheme */
/* Determine whether to send the Referer header */
if (nsoption_bool(send_referer) && ref_scheme != NULL) {
/* User permits us to send the header
* Only send it if:
* 1) The fetch and referer schemes match
* or 2) The fetch is https and the referer is http
*
* This ensures that referer information is only sent
* across schemes in the special case of an https
* request from a page served over http. The inverse
* (https -> http) should not send the referer (15.1.3)
*/
bool match1;
bool match2;
if (lwc_string_isequal(scheme, ref_scheme,
&match) != lwc_error_ok) {
match = false;
}
if (lwc_string_isequal(scheme, corestring_lwc_https,
&match1) != lwc_error_ok) {
match1 = false;
}
if (lwc_string_isequal(ref_scheme, corestring_lwc_http,
&match2) != lwc_error_ok) {
match2= false;
}
if (match == true || (match1 == true && match2 == true))
fetch->send_referer = true;
}
if (ref_scheme != NULL)
lwc_string_unref(ref_scheme);
}
/* these aren't needed past here */
lwc_string_unref(scheme);
/* try and set up the fetch */
fetch->fetcher_handle = fetchers[fetch->fetcherd].ops.setup(fetch, url,
only_2xx, downgrade_tls,
@ -839,13 +798,6 @@ void fetch_set_http_code(struct fetch *fetch, long http_code)
fetch->http_code = http_code;
}
/* exported interface documented in content/fetch.h */
const char *fetch_get_referer_to_send(struct fetch *fetch)
{
if (fetch->send_referer)
return nsurl_access(fetch->referer);
return NULL;
}
/* exported interface documented in content/fetch.h */
void fetch_set_cookie(struct fetch *fetch, const char *data)

5
content/fetch.h
View File

@ -229,11 +229,6 @@ void fetch_free(struct fetch *f);
*/
void fetch_set_http_code(struct fetch *fetch, long http_code);
/**
* get the referer from the fetch
*/
const char *fetch_get_referer_to_send(struct fetch *fetch);
/**
* set cookie data on a fetch
*/

1
content/fetchers/curl.c
View File

@ -889,7 +889,6 @@ static CURLcode fetch_curl_set_options(struct curl_fetch_info *f)
SETOPT(CURLOPT_WRITEDATA, f);
SETOPT(CURLOPT_WRITEHEADER, f);
SETOPT(CURLOPT_PROGRESSDATA, f);
SETOPT(CURLOPT_REFERER, fetch_get_referer_to_send(f->fetch_handle));
SETOPT(CURLOPT_HTTPHEADER, f->headers);
if (f->post_urlenc) {
SETOPT(CURLOPT_HTTPPOST, NULL);

14
content/hlcache.c
View File

@ -673,11 +673,15 @@ void hlcache_finalise(void)
}
/* See hlcache.h for documentation */
nserror hlcache_handle_retrieve(nsurl *url, uint32_t flags,
nsurl *referer, llcache_post_data *post,
hlcache_handle_callback cb, void *pw,
hlcache_child_context *child,
content_type accepted_types, hlcache_handle **result)
nserror
hlcache_handle_retrieve(nsurl *url,
uint32_t flags,
nsurl *referer,
llcache_post_data *post,
hlcache_handle_callback cb, void *pw,
hlcache_child_context *child,
content_type accepted_types,
hlcache_handle **result)
{
hlcache_retrieval_ctx *ctx;
nserror error;

186
content/llcache.c
View File

@ -47,6 +47,7 @@
#include "utils/utils.h"
#include "utils/time.h"
#include "utils/http.h"
#include "utils/nsoption.h"
#include "netsurf/misc.h"
#include "desktop/gui_internal.h"
@ -806,6 +807,87 @@ static nserror llcache_fetch_process_header(llcache_object *object,
return NSERROR_OK;
}
/**
* construct a Referer header appropriate for the request
*
* \param url The url being navigated to
* \param referer The referring url
* \param header_out A pointer to receive the header. The buffer must
* be freed by the caller.
* \return NSERROR_OK and \a header_out updated on success else error code
*/
static nserror get_referer_header(nsurl *url, nsurl *referer, char **header_out)
{
nserror res = NSERROR_INVALID;
lwc_string *ref_scheme;
lwc_string *scheme;
bool match;
bool match1;
bool match2;
char *header;
/* Determine whether to send the Referer header */
if (!nsoption_bool(send_referer)) {
return NSERROR_INVALID;
}
scheme = nsurl_get_component(url, NSURL_SCHEME);
if (scheme == NULL) {
return NSERROR_BAD_URL;
}
ref_scheme = nsurl_get_component(referer, NSURL_SCHEME);
if (ref_scheme == NULL) {
/* referer has no scheme so no header */
lwc_string_unref(scheme);
return NSERROR_INVALID;
}
/* User permits us to send the header
* Only send it if:
* 1) The fetch and referer schemes match
* or 2) The fetch is https and the referer is http
*
* This ensures that referer information is only sent
* across schemes in the special case of an https
* request from a page served over http. The inverse
* (https -> http) should not send the referer (15.1.3)
*/
if (lwc_string_isequal(scheme, ref_scheme,
&match) != lwc_error_ok) {
match = false;
}
if (lwc_string_isequal(scheme, corestring_lwc_https,
&match1) != lwc_error_ok) {
match1 = false;
}
if (lwc_string_isequal(ref_scheme, corestring_lwc_http,
&match2) != lwc_error_ok) {
match2 = false;
}
if (match == true || (match1 == true && match2 == true)) {
const size_t len = SLEN("Referer: ") +
nsurl_length(referer) + 1;
header = malloc(len);
if (header == NULL) {
res = NSERROR_NOMEM;
} else {
snprintf(header, len, "Referer: %s",
nsurl_access(referer));
*header_out = header;
res = NSERROR_OK;
}
}
lwc_string_unref(scheme);
lwc_string_unref(ref_scheme);
return res;
}
/**
* (Re)fetch an object
*
@ -834,12 +916,13 @@ static nserror llcache_object_refetch(llcache_object *object)
}
}
/* Generate cache-control headers */
headers = malloc(3 * sizeof(char *));
/* Generate headers */
headers = malloc(4 * sizeof(char *));
if (headers == NULL) {
return NSERROR_NOMEM;
}
/* cache-control header for etag */
if (object->cache.etag != NULL) {
const size_t len = SLEN("If-None-Match: ") +
strlen(object->cache.etag) + 1;
@ -856,6 +939,7 @@ static nserror llcache_object_refetch(llcache_object *object)
header_idx++;
}
/* cache-control header for modification time */
if (object->cache.last_modified != 0) {
/* Maximum length of an RFC 1123 date is 29 bytes */
const size_t len = SLEN("If-Modified-Since: ") + 29 + 1;
@ -873,6 +957,15 @@ static nserror llcache_object_refetch(llcache_object *object)
header_idx++;
}
/* Referer header */
if (object->fetch.referer != NULL) {
if (get_referer_header(object->url,
object->fetch.referer,
&headers[header_idx]) == NSERROR_OK) {
header_idx++;
}
}
headers[header_idx] = NULL;
/* Reset cache control data */
@ -3571,6 +3664,46 @@ total_object_size(llcache_object *object)
return tot;
}
/**
* Catch up the cache users with state changes from fetchers.
*
* \param ignored We ignore this because all our state comes from llcache.
*/
static void llcache_catch_up_all_users(void *ignored)
{
llcache_object *object;
/* Assume after this we'll be all caught up. If any user of a handle
* defers then we'll invalidate all_caught_up and reschedule via
* llcache_users_not_caught_up()
*/
llcache->all_caught_up = true;
/* Catch new users up with state of objects */
for (object = llcache->cached_objects; object != NULL;
object = object->next) {
llcache_object_notify_users(object);
}
for (object = llcache->uncached_objects; object != NULL;
object = object->next) {
llcache_object_notify_users(object);
}
}
/**
* Ask for ::llcache_catch_up_all_users to be scheduled ASAP to pump the
* user state machines.
*/
static void llcache_users_not_caught_up(void)
{
if (llcache->all_caught_up) {
llcache->all_caught_up = false;
guit->misc->schedule(0, llcache_catch_up_all_users, NULL);
}
}
/******************************************************************************
* Public API *
******************************************************************************/
@ -3843,51 +3976,16 @@ void llcache_finalise(void)
llcache = NULL;
}
/**
* Catch up the cache users with state changes from fetchers.
*
* \param ignored We ignore this because all our state comes from llcache.
*/
static void llcache_catch_up_all_users(void *ignored)
{
llcache_object *object;
/* Assume after this we'll be all caught up. If any user of a handle
* defers then we'll invalidate all_caught_up and reschedule via
* llcache_users_not_caught_up()
*/
llcache->all_caught_up = true;
/* Catch new users up with state of objects */
for (object = llcache->cached_objects; object != NULL;
object = object->next) {
llcache_object_notify_users(object);
}
for (object = llcache->uncached_objects; object != NULL;
object = object->next) {
llcache_object_notify_users(object);
}
}
/**
* Ask for ::llcache_catch_up_all_users to be scheduled ASAP to pump the
* user state machines.
*/
static void llcache_users_not_caught_up(void)
{
if (llcache->all_caught_up) {
llcache->all_caught_up = false;
guit->misc->schedule(0, llcache_catch_up_all_users, NULL);
}
}
/* Exported interface documented in content/llcache.h */
nserror llcache_handle_retrieve(nsurl *url, uint32_t flags,
nsurl *referer, const llcache_post_data *post,
llcache_handle_callback cb, void *pw,
llcache_handle **result)
nserror
llcache_handle_retrieve(nsurl *url,
uint32_t flags,
nsurl *referer,
const llcache_post_data *post,
llcache_handle_callback cb, void *pw,
llcache_handle **result)
{
nserror error;
llcache_object_user *user;