eukara
/
WebSurf
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Extend low level source data cache with persistant storage

This commit is contained in:
Vincent Sanders 2014-03-22 14:54:51 +00:00
parent 657abbd245
commit 4a49ff5266
5 changed files with 1031 additions and 231 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
content/hlcache.c
View File

@ -339,9 +339,10 @@ static nserror hlcache_migrate_ctx(hlcache_retrieval_ctx *ctx,
ctx->migrate_target = true;
if (effective_type != NULL &&
hlcache_type_is_acceptable(effective_type,
ctx->accepted_types, &type)) {
if ((effective_type != NULL) &&
hlcache_type_is_acceptable(effective_type,
ctx->accepted_types,
&type)) {
error = hlcache_find_content(ctx, effective_type);
if (error != NSERROR_OK && error != NSERROR_NEED_DATA) {
if (ctx->handle->cb != NULL) {
@ -524,9 +525,7 @@ hlcache_initialise(const struct hlcache_parameters *hlcache_parameters)
return NSERROR_NOMEM;
}
ret = llcache_initialise(hlcache_parameters->cb,
hlcache_parameters->cb_ctx,
hlcache_parameters->limit);
ret = llcache_initialise(&hlcache_parameters->llcache);
if (ret != NSERROR_OK) {
free(hlcache);
hlcache = NULL;

39
content/hlcache.h
View File

@ -23,11 +23,12 @@
#ifndef NETSURF_CONTENT_HLCACHE_H_
#define NETSURF_CONTENT_HLCACHE_H_
#include "content/content.h"
#include "content/llcache.h"
#include "utils/errors.h"
#include "utils/nsurl.h"
#include "content/content.h"
#include "content/llcache.h"
/** High-level cache handle */
typedef struct hlcache_handle hlcache_handle;
@ -44,18 +45,10 @@ typedef struct {
} hlcache_event;
struct hlcache_parameters {
llcache_query_callback cb; /**< Query handler for llcache */
void *cb_ctx; /**< Pointer to llcache query handler data */
/** How frequently the background cache clean process is run (ms) */
unsigned int bg_clean_time;
/** The target upper bound for the cache size */
size_t limit;
/** The hysteresis allowed round the target size */
size_t hysteresis;
struct llcache_parameters llcache;
};
/**
@ -67,13 +60,13 @@ struct hlcache_parameters {
* \return NSERROR_OK on success, appropriate error otherwise.
*/
typedef nserror (*hlcache_handle_callback)(hlcache_handle *handle,
const hlcache_event *event, void *pw);
const hlcache_event *event, void *pw);
/** Flags for high-level cache object retrieval */
enum hlcache_retrieve_flag {
/* Note: low-level cache retrieval flags occupy the bottom 16 bits of
* the flags word. High-level cache flags occupy the top 16 bits.
* To avoid confusion, high-level flags are allocated from bit 31 down.
/* Note: low-level cache retrieval flags occupy the bottom 16 bits of
* the flags word. High-level cache flags occupy the top 16 bits.
* To avoid confusion, high-level flags are allocated from bit 31 down.
*/
/** It's permitted to convert this request into a download */
HLCACHE_RETRIEVE_MAY_DOWNLOAD = (1 << 31),
@ -84,7 +77,7 @@ enum hlcache_retrieve_flag {
/**
* Initialise the high-level cache, preparing the llcache also.
*
* \param hlcache_parameters Settings to initialise cache with
* \param hlcache_parameters Settings to initialise cache with
* \return NSERROR_OK on success, appropriate error otherwise.
*/
nserror hlcache_initialise(const struct hlcache_parameters *hlcache_parameters);
@ -133,7 +126,7 @@ nserror hlcache_poll(void);
nserror hlcache_handle_retrieve(nsurl *url, uint32_t flags,
nsurl *referer, llcache_post_data *post,
hlcache_handle_callback cb, void *pw,
hlcache_child_context *child,
hlcache_child_context *child,
content_type accepted_types, hlcache_handle **result);
/**
@ -169,13 +162,13 @@ nserror hlcache_handle_replace_callback(hlcache_handle *handle,
* \param handle Cache handle to dereference
* \return Pointer to content object, or NULL if there is none
*
* \todo This may not be correct. Ideally, the client should never need to
* directly access a content object. It may, therefore, be better to provide a
* bunch of veneers here that take a hlcache_handle and invoke the
* \todo This may not be correct. Ideally, the client should never need to
* directly access a content object. It may, therefore, be better to provide a
* bunch of veneers here that take a hlcache_handle and invoke the
* corresponding content_ API. If there's no content object associated with the
* hlcache_handle (e.g. because the source data is still being fetched, so it
* doesn't exist yet), then these veneers would behave as a NOP. The important
* thing being that the client need not care about this possibility and can
* hlcache_handle (e.g. because the source data is still being fetched, so it
* doesn't exist yet), then these veneers would behave as a NOP. The important
* thing being that the client need not care about this possibility and can
* just call the functions with impugnity.
*/
struct content *hlcache_handle_get_content(const hlcache_handle *handle);

1011
content/llcache.c
View File

File diff suppressed because it is too large Load Diff

92
content/llcache.h
View File

@ -76,7 +76,7 @@ typedef struct {
} data; /**< Event data */
} llcache_event;
/**
/**
* Client callback for low-level cache events
*
* \param handle Handle for which event is issued
@ -84,18 +84,18 @@ typedef struct {
* \param pw Pointer to client-specific data
* \return NSERROR_OK on success, appropriate error otherwise.
*/
typedef nserror (*llcache_handle_callback)(llcache_handle *handle,
typedef nserror (*llcache_handle_callback)(llcache_handle *handle,
const llcache_event *event, void *pw);
/** Flags for low-level cache object retrieval */
enum llcache_retrieve_flag {
/* Note: We're permitted a maximum of 16 flags which must reside in the
* bottom 16 bits of the flags word. See hlcache.h for further details.
* bottom 16 bits of the flags word. See hlcache.h for further details.
*/
/** Force a new fetch */
LLCACHE_RETRIEVE_FORCE_FETCH = (1 << 0),
LLCACHE_RETRIEVE_FORCE_FETCH = (1 << 0),
/** Requested URL was verified */
LLCACHE_RETRIEVE_VERIFIABLE = (1 << 1),
LLCACHE_RETRIEVE_VERIFIABLE = (1 << 1),
/**< No error pages */
LLCACHE_RETRIEVE_NO_ERROR_PAGES = (1 << 2),
/**< Stream data (implies that object is not cacheable) */
@ -149,13 +149,81 @@ typedef nserror (*llcache_query_response)(bool proceed, void *cbpw);
* \param cbpw Opaque value to pass into \a cb
* \return NSERROR_OK on success, appropriate error otherwise
*
* \note This callback should return immediately. Once a suitable answer to
* the query has been obtained, the provided response callback should be
* \note This callback should return immediately. Once a suitable answer to
* the query has been obtained, the provided response callback should be
* called. This is intended to be an entirely asynchronous process.
*/
typedef nserror (*llcache_query_callback)(const llcache_query *query, void *pw,
llcache_query_response cb, void *cbpw);
/**
* Parameters to configure the low level cache backing store.
*/
struct llcache_store_parameters {
const char *path; /**< The path to the backing store */
size_t limit; /**< The backing store upper bound target size */
size_t hysteresis; /**< The hysteresis around the target size */
/** log2 of the default maximum number of entries the cache
* can track.
*
* If unset this defaults to 16 (65536 entries) The cache
* control file takes precedence so cache data remains
* portable between builds with differing defaults.
*/
unsigned int entry_size;
/** log2 of the default number of entries in the mapping between
* the url and cache entries.
*
* @note This is exposing an internal implementation detail of
* the filesystem based default backing store implementation.
* However it is likely any backing store implementation will
* need some way to map url to cache entries so it is a
* generally useful configuration value.
*
* Too small a value will cause unecessary collisions and
* cache misses and larger values cause proportionaly larger
* amounts of memory to be used.
*
* The "birthday paradox" means that the hash will experience
* a collision in every 2^(address_size/2) urls the cache
* stores.
*
* A value of 20 means one object stored in every 1024 will
* cause a collion and a cache miss while using two megabytes
* of storage.
*
* If unset this defaults to 20 (1048576 entries using two
* megabytes) The cache control file takes precedence so cache
* data remains portable between builds with differing
* defaults.
*/
unsigned int address_size;
};
/**
* Parameters to configure the low level cache.
*/
struct llcache_parameters {
llcache_query_callback cb; /**< Query handler for llcache */
void *cb_ctx; /**< Pointer to llcache query handler data */
size_t limit; /**< The target upper bound for the RAM cache size */
size_t hysteresis; /**< The hysteresis around the target size */
int minimum_lifetime; /**< The minimum lifetime to consider
* sending objects to backing store.
*/
size_t bandwidth; /**< The maximum bandwidth to allow the
* backing store to use.
*/
struct llcache_store_parameters store;
};
/**
* Initialise the low-level cache
*
@ -163,7 +231,7 @@ typedef nserror (*llcache_query_callback)(const llcache_query *query, void *pw,
* \param pw Pointer to query handler data
* \return NSERROR_OK on success, appropriate error otherwise.
*/
nserror llcache_initialise(llcache_query_callback cb, void *pw, uint32_t llcache_limit);
nserror llcache_initialise(const struct llcache_parameters *parameters);
/**
* Finalise the low-level cache
@ -280,12 +348,12 @@ const uint8_t *llcache_handle_get_source_data(const llcache_handle *handle,
* \return Header value, or NULL if header does not exist
*
* \todo Make the key an enumeration, to avoid needless string comparisons
* \todo Forcing the client to parse the header value seems wrong.
* Better would be to return the actual value part and an array of
* \todo Forcing the client to parse the header value seems wrong.
* Better would be to return the actual value part and an array of
* key-value pairs for any additional parameters.
* \todo Deal with multiple headers of the same key (e.g. Set-Cookie)
*/
const char *llcache_handle_get_header(const llcache_handle *handle,
const char *llcache_handle_get_header(const llcache_handle *handle,
const char *key);
/**
@ -295,7 +363,7 @@ const char *llcache_handle_get_header(const llcache_handle *handle,
* \param b Second handle
* \return True if handles reference the same object, false otherwise
*/
bool llcache_handle_references_same_object(const llcache_handle *a,
bool llcache_handle_references_same_object(const llcache_handle *a,
const llcache_handle *b);
#endif

109
desktop/netsurf.c
View File

@ -67,11 +67,23 @@
*/
#define SPECULATE_SMALL 4096
/* the time between cache clean runs in ms */
/** the time between image cache clean runs in ms. */
#define IMAGE_CACHE_CLEAN_TIME (10 * 1000)
/** default time between content cache cleans. */
#define HL_CACHE_CLEAN_TIME (2 * IMAGE_CACHE_CLEAN_TIME)
/** default minimum object time before object is pushed to backing store. */
#define LLCACHE_MIN_DISC_LIFETIME (60 * 30)
/** default maximum bandwidth for backing store writeout. */
#define LLCACHE_MAX_DISC_BANDWIDTH (128 * 1024)
/** ensure there is a minimal amount of memory for source objetcs and
* decoded bitmaps.
*/
#define MINIMUM_MEMORY_CACHE_SIZE (2 * 1024 * 1024)
bool netsurf_quit = false;
static void netsurf_lwc_iterator(lwc_string *str, void *pw)
@ -108,8 +120,6 @@ static nserror netsurf_llcache_query_handler(const llcache_query *query,
return NSERROR_OK;
}
#define MINIMUM_MEMORY_CACHE_SIZE (2 * 1024 * 1024)
/* exported interface documented in desktop/netsurf.h */
nserror netsurf_register(struct netsurf_table *table)
{
@ -120,12 +130,15 @@ nserror netsurf_register(struct netsurf_table *table)
/* exported interface documented in desktop/netsurf.h */
nserror netsurf_init(const char *messages)
{
nserror error;
nserror ret;
struct utsname utsname;
nserror ret = NSERROR_OK;
struct hlcache_parameters hlcache_parameters = {
.bg_clean_time = HL_CACHE_CLEAN_TIME,
.cb = netsurf_llcache_query_handler,
.llcache = {
.cb = netsurf_llcache_query_handler,
.minimum_lifetime = LLCACHE_MIN_DISC_LIFETIME,
.bandwidth = LLCACHE_MAX_DISC_BANDWIDTH,
}
};
struct image_cache_parameters image_cache_parameters = {
.bg_clean_time = IMAGE_CACHE_CLEAN_TIME,
@ -155,75 +168,87 @@ nserror netsurf_init(const char *messages)
messages_load(messages);
/* corestrings init */
error = corestrings_init();
if (error != NSERROR_OK)
return error;
ret = corestrings_init();
if (ret != NSERROR_OK)
return ret;
/* set up cache limits based on the memory cache size option */
hlcache_parameters.limit = nsoption_int(memory_cache_size);
hlcache_parameters.llcache.limit = nsoption_int(memory_cache_size);
if (hlcache_parameters.limit < MINIMUM_MEMORY_CACHE_SIZE) {
hlcache_parameters.limit = MINIMUM_MEMORY_CACHE_SIZE;
LOG(("Setting minimum memory cache size to %d",
hlcache_parameters.limit));
if (hlcache_parameters.llcache.limit < MINIMUM_MEMORY_CACHE_SIZE) {
hlcache_parameters.llcache.limit = MINIMUM_MEMORY_CACHE_SIZE;
LOG(("Setting minimum memory cache size %d",
hlcache_parameters.llcache.limit));
}
/* image cache is 25% of total memory cache size */
image_cache_parameters.limit = (hlcache_parameters.limit * 25) / 100;
image_cache_parameters.limit = (hlcache_parameters.llcache.limit * 25) / 100;
/* image cache hysteresis is 20% of the image cache size */
image_cache_parameters.hysteresis = (image_cache_parameters.limit * 20) / 100;
/* account for image cache use from total */
hlcache_parameters.limit -= image_cache_parameters.limit;
hlcache_parameters.llcache.limit -= image_cache_parameters.limit;
/* set backing store target limit */
hlcache_parameters.llcache.store.limit = nsoption_int(disc_cache_size);
/* set backing store hysterissi to 20% */
hlcache_parameters.llcache.store.hysteresis = (hlcache_parameters.llcache.store.limit * 20) / 100;;
/* set the path to the backing store */
/** \todo set the backing store path properly */
hlcache_parameters.llcache.store.path = "/tmp/ns";
/* image handler bitmap cache */
error = image_cache_init(&image_cache_parameters);
if (error != NSERROR_OK)
return error;
ret = image_cache_init(&image_cache_parameters);
if (ret != NSERROR_OK)
return ret;
/* content handler initialisation */
error = nscss_init();
if (error != NSERROR_OK)
return error;
ret = nscss_init();
if (ret != NSERROR_OK)
return ret;
error = html_init();
if (error != NSERROR_OK)
return error;
ret = html_init();
if (ret != NSERROR_OK)
return ret;
error = image_init();
if (error != NSERROR_OK)
return error;
ret = image_init();
if (ret != NSERROR_OK)
return ret;
error = textplain_init();
if (error != NSERROR_OK)
return error;
ret = textplain_init();
if (ret != NSERROR_OK)
return ret;
error = mimesniff_init();
if (error != NSERROR_OK)
return error;
ret = mimesniff_init();
if (ret != NSERROR_OK)
return ret;
url_init();
setlocale(LC_ALL, "C");
/* initialise the fetchers */
error = fetch_init();
if (error != NSERROR_OK)
return error;
ret = fetch_init();
if (ret != NSERROR_OK)
return ret;
/* Initialise the hlcache and allow it to init the llcache for us */
hlcache_initialise(&hlcache_parameters);
ret = hlcache_initialise(&hlcache_parameters);
if (ret != NSERROR_OK)
return ret;
/* Initialize system colours */
error = ns_system_colour_init();
if (error != NSERROR_OK)
return error;
ret = ns_system_colour_init();
if (ret != NSERROR_OK)
return ret;
js_initialise();
return ret;
return NSERROR_OK;
}