tech
/
engine
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Preliminary attempt at loading zips concatenated onto the end of bsp files, instead of having servers load 5000+ different pk3s up front. 

git-svn-id: https://svn.code.sf.net/p/fteqw/code/trunk@5876 fc73d0e0-1445-4013-8a0c-d673dee63da5
This commit is contained in:
Spoike 2021-05-28 07:07:48 +00:00
parent b4092ab903
commit 1979150076
8 changed files with 121 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
engine/common/common.h
View File

@ -812,7 +812,8 @@ void FS_FreeFile(void *file);
qbyte *COM_LoadFile (const char *path, unsigned int locateflags, int usehunk, size_t *filesize);
qboolean COM_LoadMapPackFile(const char *name, qofs_t offset);
qboolean FS_LoadMapPackFile (const char *filename, searchpathfuncs_t *archive);
void FS_CloseMapPackFile (searchpathfuncs_t *archive);
void COM_FlushTempoaryPacks(void);
void COM_EnumerateFiles (const char *match, int (QDECL *func)(const char *fname, qofs_t fsize, time_t mtime, void *parm, searchpathfuncs_t *spath), void *parm);

64
engine/common/fs.c
View File

@ -2793,14 +2793,14 @@ searchpathfuncs_t *COM_EnumerateFilesPackage (char *matches, const char *package
if (com_homepathenabled)
{ //try the homedir
Q_snprintfz(syspath, sizeof(syspath), "%s%s", com_homepath, package);
handle = FS_OpenPackByExtension(VFSOS_Open(package, "rb"), NULL, package, package);
handle = FS_OpenPackByExtension(VFSOS_Open(syspath, "rb"), NULL, package, package);
}
else
handle = NULL;
if (!handle)
{ //now go for the basedir to see if ther.
Q_snprintfz(syspath, sizeof(syspath), "%s%s", com_gamepath, package);
handle = FS_OpenPackByExtension(VFSOS_Open(package, "rb"), NULL, package, package);
handle = FS_OpenPackByExtension(VFSOS_Open(syspath, "rb"), NULL, package, package);
}
if (handle)
@ -2835,17 +2835,20 @@ void COM_EnumerateFiles (const char *match, int (QDECL *func)(const char *, qofs
}
}
void COM_FlushTempoaryPacks(void)
void COM_FlushTempoaryPacks(void) //flush all temporary packages
{
#if 0
searchpath_t *sp, **link;
COM_WorkerLock(); //make sure no workers are poking files...
Sys_LockMutex(fs_thread_mutex);
link = &com_searchpaths;
while (*link)
{
sp = *link;
if (sp->flags & SPF_TEMPORARY)
{
FS_FlushFSHashReally();
FS_FlushFSHashFull();
*link = sp->next;
@ -2856,15 +2859,52 @@ void COM_FlushTempoaryPacks(void)
link = &sp->next;
}
com_purepaths = NULL;
#endif
}
qboolean COM_LoadMapPackFile (const char *filename, qofs_t ofs)
{
return false;
Sys_UnlockMutex(fs_thread_mutex);
COM_WorkerUnlock(); //workers can continue now
}
static searchpath_t *FS_AddPathHandle(searchpath_t **oldpaths, const char *purepath, const char *probablepath, searchpathfuncs_t *handle, const char *prefix, unsigned int flags, unsigned int loadstuff);
qboolean FS_LoadMapPackFile (const char *filename, searchpathfuncs_t *archive)
{
if (!archive->AddReference)
return false; //nope...
archive->AddReference(archive);
if (FS_AddPathHandle(NULL, filename, filename, archive, "", SPF_TEMPORARY, 0))
return true;
return false;
}
void FS_CloseMapPackFile (searchpathfuncs_t *archive)
{
searchpath_t *sp, **link;
COM_WorkerLock(); //make sure no workers are poking files...
Sys_LockMutex(fs_thread_mutex);
link = &com_searchpaths;
while (*link)
{
sp = *link;
if (sp->handle == archive)
{
FS_FlushFSHashFull();
*link = sp->next;
sp->handle->ClosePath(sp->handle);
Z_Free (sp);
break;
}
else
link = &sp->next;
}
com_purepaths = NULL;
Sys_UnlockMutex(fs_thread_mutex);
COM_WorkerUnlock(); //workers can continue now
archive->ClosePath(archive);
}
static searchpathfuncs_t *FS_GetOldPath(searchpath_t **oldpaths, const char *dir, unsigned int *keepflags)
{
searchpath_t *p;
@ -3356,7 +3396,7 @@ static searchpath_t *FS_AddPathHandle(searchpath_t **oldpaths, const char *purep
flags &= ~SPF_WRITABLE;
//temp packages also do not nest
// if (!(flags & SPF_TEMPORARY))
if (!(flags & SPF_TEMPORARY))
FS_AddDataFiles(oldpaths, purepath, logicalpath, search, flags&(SPF_COPYPROTECTED|SPF_UNTRUSTED|SPF_TEMPORARY|SPF_PRIVATE|SPF_QSHACK), loadstuff);
if (flags & SPF_TEMPORARY)

5
engine/common/fs.h
View File

@ -8,7 +8,7 @@ Files may be shared between threads, but not simultaneously.
The filesystem driver is responsible for closing the pak/pk3 once all files are closed, and must ensure that opens+reads+closes as well as archive closure are thread safe.
*/
#define FSVER 2
#define FSVER 3
#define FF_NOTFOUND (0u) //file wasn't found
@ -32,7 +32,8 @@ struct searchpath_s;
struct searchpathfuncs_s
{
int fsver;
void (QDECL *ClosePath)(searchpathfuncs_t *handle);
void (QDECL *ClosePath)(searchpathfuncs_t *handle); //removes a reference, kills it when it reaches 0. the package can only actually be killed once all contained files are closed.
void (QDECL *AddReference)(searchpathfuncs_t *handle); //adds an extra reference, so we survive closes better.
void (QDECL *GetPathDetails)(searchpathfuncs_t *handle, char *outdetails, size_t sizeofdetails);
void (QDECL *BuildHash)(searchpathfuncs_t *handle, int depth, void (QDECL *FS_AddFileHash)(int depth, const char *fname, fsbucket_t *filehandle, void *pathhandle));

9
engine/common/fs_zip.c
View File

@ -596,6 +596,14 @@ static void QDECL FSZIP_GetPathDetails(searchpathfuncs_t *handle, char *out, siz
else
*out = '\0';
}
static void QDECL FSZIP_UnclosePath(searchpathfuncs_t *handle)
{
zipfile_t *zip = (void*)handle;
if (!Sys_LockMutex(zip->mutex))
return; //ohnoes
zip->references++;
Sys_UnlockMutex(zip->mutex);
}
static void QDECL FSZIP_ClosePath(searchpathfuncs_t *handle)
{
size_t s;
@ -2105,6 +2113,7 @@ searchpathfuncs_t *QDECL FSZIP_LoadArchive (vfsfile_t *packhandle, searchpathfun
zip->pub.fsver = FSVER;
zip->pub.GetPathDetails = FSZIP_GetPathDetails;
zip->pub.ClosePath = FSZIP_ClosePath;
zip->pub.AddReference = FSZIP_UnclosePath;
zip->pub.BuildHash = FSZIP_BuildHash;
zip->pub.FindFile = FSZIP_FLocate;
zip->pub.ReadFile = FSZIP_ReadFile;

1
engine/common/gl_q2bsp.c
View File

@ -4914,6 +4914,7 @@ static cmodel_t *CM_LoadMap (model_t *mod, qbyte *filein, size_t filelen, qboole
Q_snprintfz (name, sizeof(name), "*%i:%s", i, wmod->publicname);
mod = Mod_FindName (name);
*mod = *wmod;
mod->archive = NULL;
mod->entities_raw = NULL;
mod->submodelof = wmod;
Q_strncpyz(mod->publicname, name, sizeof(mod->publicname));

60
engine/common/q1bsp.c
View File

@ -2201,6 +2201,7 @@ Init stuff
BSPX Stuff
*/
#include "fs.h"
typedef struct {
char lumpname[24]; // up to 23 chars, zero-padded
@ -2240,10 +2241,10 @@ void *BSPX_FindLump(bspx_header_t *bspxheader, void *mod_base, char *lumpname, i
}
return NULL;
}
bspx_header_t *BSPX_Setup(model_t *mod, char *filebase, unsigned int filelen, lump_t *lumps, int numlumps)
bspx_header_t *BSPX_Setup(model_t *mod, char *filebase, size_t filelen, lump_t *lumps, size_t numlumps)
{
int i;
int offs = 0;
size_t i;
size_t offs = 0;
bspx_header_t *h;
for (i = 0; i < numlumps; i++, lumps++)
@ -2253,22 +2254,45 @@ bspx_header_t *BSPX_Setup(model_t *mod, char *filebase, unsigned int filelen, lu
}
offs = (offs + 3) & ~3;
if (offs + sizeof(*h) > filelen)
return NULL; /*no space for it*/
h = (bspx_header_t*)(filebase + offs);
i = LittleLong(h->numlumps);
/*verify the header*/
if (*(int*)h->id != (('B'<<0)|('S'<<8)|('P'<<16)|('X'<<24)) ||
i < 0 ||
offs + sizeof(*h) + sizeof(h->lumps[0])*(i-1) > filelen)
return NULL;
h->numlumps = i;
while(i-->0)
h = NULL; /*no space for it*/
else
{
h->lumps[i].fileofs = LittleLong(h->lumps[i].fileofs);
h->lumps[i].filelen = LittleLong(h->lumps[i].filelen);
if (h->lumps[i].fileofs + h->lumps[i].filelen > filelen)
return NULL;
h = (bspx_header_t*)(filebase + offs);
i = LittleLong(h->numlumps);
/*verify the header*/
if (*(int*)h->id != (('B'<<0)|('S'<<8)|('P'<<16)|('X'<<24)) ||
i < 0 ||
offs + sizeof(*h) + sizeof(h->lumps[0])*(i-1) > filelen)
h = NULL;
else
{
h->numlumps = i;
while(i-->0)
{
h->lumps[i].fileofs = LittleLong(h->lumps[i].fileofs);
h->lumps[i].filelen = LittleLong(h->lumps[i].filelen);
if (h->lumps[i].fileofs + h->lumps[i].filelen > filelen)
return NULL; //some sort of corruption/truncation.
if (offs < lumps->fileofs + lumps->filelen)
offs = lumps->fileofs + lumps->filelen;
}
}
}
if (offs < filelen && !mod->archive)
{ //we have some sort of trailing junk... is it a zip?...
vfsfile_t *f = VFSPIPE_Open(1,true);
if (f)
{
VFS_WRITE(f, filebase+offs, filelen-offs);
mod->archive = FSZIP_LoadArchive(f, NULL, mod->name, mod->name, NULL);
if (mod->archive)
FS_LoadMapPackFile(mod->name, mod->archive); //give it to the filesystem to use.
else
VFS_CLOSE(f); //give up.
}
}
return h;

12
engine/gl/gl_model.c
View File

@ -23,7 +23,7 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
// on the same machine.
#include "quakedef.h"
#include "fs.h"
#include "com_bih.h"
#if 1//ndef SERVERONLY //FIXME
#include "glquake.h"
@ -541,8 +541,13 @@ qboolean Mod_PurgeModel(model_t *mod, enum mod_purge_e ptype)
#endif
//and obliterate anything else remaining in memory.
ZG_FreeGroup(&mod->memgroup);
mod->meshinfo = NULL;
if (mod->archive)
{
FS_CloseMapPackFile(mod->archive);
mod->archive = NULL;
}
ZG_FreeGroup(&mod->memgroup);
mod->loadstate = MLS_NOTLOADED;
mod->submodelof = NULL;
@ -5496,7 +5501,10 @@ TRACE(("LoadBrushModel %i\n", __LINE__));
submod->pvsbytes = ((submod->numclusters+31)>>3)&~3;
if (i)
{
submod->entities_raw = NULL;
submod->archive = NULL;
}
memset(&submod->batches, 0, sizeof(submod->batches));
submod->vbos = NULL;

3
engine/gl/gl_model.h
View File

@ -585,7 +585,7 @@ void Q1BSP_Init(void);
void BSPX_LoadEnvmaps(struct model_s *mod, bspx_header_t *bspx, void *mod_base);
void *BSPX_FindLump(bspx_header_t *bspxheader, void *mod_base, char *lumpname, int *lumpsize);
bspx_header_t *BSPX_Setup(struct model_s *mod, char *filebase, unsigned int filelen, lump_t *lumps, int numlumps);
bspx_header_t *BSPX_Setup(struct model_s *mod, char *filebase, size_t filelen, lump_t *lumps, size_t numlumps);
typedef struct fragmentdecal_s fragmentdecal_t;
void Fragment_ClipPoly(fragmentdecal_t *dec, int numverts, float *inverts, shader_t *surfshader);
@ -1084,6 +1084,7 @@ typedef struct model_s
// additional model data
//
void *meshinfo; //data allocated within the memgroup allocations, will be nulled out when the model is flushed
searchpathfuncs_t *archive; //some bsp formats have an embedded zip...
zonegroup_t memgroup;
} model_t;