tech
/
engine
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add support for maps/foo.bsp.xz too, for better compression. 

Seeking within files inside zips is now slightly more efficient.
Fix issue with servers decompressing .gz before download...

git-svn-id: https://svn.code.sf.net/p/fteqw/code/trunk@5883 fc73d0e0-1445-4013-8a0c-d673dee63da5
This commit is contained in:
Spoike 2021-06-02 15:29:44 +00:00
parent 5ceb4730ad
commit c035801161
21 changed files with 172 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
engine/client/cl_main.c
View File

@ -5461,7 +5461,7 @@ unsigned int Host_GuessFileType(const char *mimetype, const char *filename)
ext = COM_GetFileExtension(filename, stop);
if (!Q_strstopcasecmp(ext, stop, ".php")) //deal with extra extensions the easy way
ext = COM_GetFileExtension(filename, stop=ext);
if (!Q_strstopcasecmp(ext, stop, ".gz")) //deal with extra extensions the easy way
if (!Q_strstopcasecmp(ext, stop, ".gz") || !Q_strstopcasecmp(ext, stop, ".xz")) //deal with extra extensions the easy way
ext = COM_GetFileExtension(filename, ext);
if (*ext == '.')
ext++;

2
engine/client/image.c
View File

@ -14088,7 +14088,7 @@ static void Image_LoadHiResTextureWorker(void *ctx, void *data, size_t a, size_t
if (Image_LocateHighResTexture(tex, &loc, fname, sizeof(fname), &locflags))
{
f = FS_OpenReadLocation(&loc);
f = FS_OpenReadLocation(fname, &loc);
if (f)
{
fsize = VFS_GETLEN(f);

2
engine/client/m_single.c
View File

@ -57,7 +57,7 @@ static void M_ScanSave(unsigned int slot, const char *name, qboolean savable)
if (!FS_FLocateFile(line, FSLF_DONTREFERENCE|FSLF_IGNOREPURE, &loc))
return; //not found
}
f = FS_OpenReadLocation(&loc);
f = FS_OpenReadLocation(line, &loc);
if (f)
{
VFS_GETS(f, line, sizeof(line));

7
engine/client/pr_csqc.c
View File

@ -7829,8 +7829,11 @@ static void *CSQC_FindMainProgs(size_t *sz, const char *name, unsigned int check
if (!found && *progsname && cls.state)
found = FS_FLocateFile(progsname, FSLF_IGNOREPURE, &loc);
if (!found && strcmp(progsname, "csprogs.dat"))
found = FS_FLocateFile("csprogs.dat", FSLF_IGNOREPURE, &loc);
if (found && (f=FS_OpenReadLocation(&loc)))
{
progsname = "csprogs.dat";
found = FS_FLocateFile(progsname, FSLF_IGNOREPURE, &loc);
}
if (found && (f=FS_OpenReadLocation(progsname, &loc)))
{
*sz = VFS_GETLEN(f);
file = Hunk_TempAlloc (*sz);

2
engine/client/pr_menu.c
View File

@ -2988,7 +2988,7 @@ static void *PDECL MP_PRReadFile (const char *path, qbyte *(PDECL *buf_get)(void
if (FS_FLocateFile(path, FSLF_IFFOUND|FSLF_SECUREONLY, &loc))
{
qbyte *buffer = NULL;
vfsfile_t *file = FS_OpenReadLocation(&loc);
vfsfile_t *file = FS_OpenReadLocation(path, &loc);
if (file)
{
*size = loc.len;

1
engine/common/bothdefs.h
View File

@ -132,6 +132,7 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#undef PACKAGE_DOOMWAD
#undef PACKAGE_VPK
#undef PACKAGE_DZIP
#undef AVAIL_XZDEC
#undef AVAIL_GZDEC
#else
#if defined(SERVERONLY) && defined(CLIENTONLY)

2
engine/common/cmd.c
View File

@ -733,7 +733,7 @@ static void Cmd_Exec_f (void)
Con_TPrintf ("couldn't exec %s\n", name);
return;
}
file = FS_OpenReadLocation(&loc);
file = FS_OpenReadLocation(name, &loc);
if (!file)
{
Con_TPrintf ("couldn't exec %s. check permissions.\n", name);

4
engine/common/common.c
View File

@ -2433,7 +2433,7 @@ void QDECL COM_StripExtension (const char *in, char *out, int outlen)
*s = 0;
//some extensions don't really count, strip the next one too...
if (!strcmp(s+1,"gz"))
if (!strcmp(s+1,"gz") || !strcmp(s+1,"xz"))
;
else
break;
@ -2575,7 +2575,7 @@ void COM_FileBase (const char *in, char *out, int outlen)
while (s > in)
{
if ((*s == '.'&&strcmp(s+1,"gz")) || *s == '/')
if ((*s == '.'&&strcmp(s+1,"gz")&&strcmp(s+1,"xz")) || *s == '/')
break;
s--;
}

2
engine/common/common.h
View File

@ -574,7 +574,7 @@ struct vfsfile_s;
//if loc is valid, loc->search is always filled in, the others are filled on success.
//standard return value is 0 on failure, or depth on success.
int FS_FLocateFile(const char *filename, unsigned int flags, flocation_t *loc);
struct vfsfile_s *FS_OpenReadLocation(flocation_t *location);
struct vfsfile_s *FS_OpenReadLocation(const char *fname, flocation_t *location); //fname used for extension-based filters
#define WP_REFERENCE 1
#define WP_FULLPATH 2
#define WP_FORCE 4

24
engine/common/fs.c
View File

@ -1053,7 +1053,7 @@ static int QDECL COM_Dir_List(const char *name, qofs_t size, time_t mtime, void
colour = "^2";
ext = COM_GetFileExtension(name, NULL);
if (!Q_strcasecmp(ext, ".gz"))
if (!Q_strcasecmp(ext, ".gz") || !Q_strcasecmp(ext, ".xz"))
ext = COM_GetFileExtension(name, ext);
if (*ext == '.')
{
@ -2005,10 +2005,22 @@ static const char *FS_GetCleanPath(const char *pattern, qboolean silent, char *o
static vfsfile_t *VFS_Filter(const char *filename, vfsfile_t *handle)
{
// char *ext;
if (!handle || !handle->ReadBytes || handle->seekstyle == SS_SLOW || handle->seekstyle == SS_UNSEEKABLE) //only on readonly files for which we can undo any header read damage
if (!filename)
return handle; //block any filtering (so we don't do stupid stuff like having servers pre-decompressing when downloading)
if (!handle || !handle->ReadBytes || handle->seekstyle == SS_UNSEEKABLE) //only on readonly files for which we can undo any header read damage
return handle;
// ext = COM_FileExtension (filename);
// if (handle->seekstyle == SS_SLOW)
// return handle; //we only peek at the header, so rewinding shouldn't be too expensive at least...
// const char *ext = COM_GetFileExtension(filename, NULL);
#ifdef AVAIL_XZDEC
// if (!Q_strcasecmp(ext, ".xz"))
{
vfsfile_t *nh;
nh = FS_XZ_DecompressReadFilter(handle);
if (nh!=handle)
return nh;
}
#endif
#ifdef AVAIL_GZDEC
// if (!Q_strcasecmp(ext, ".gz"))
{
@ -2373,11 +2385,11 @@ qboolean FS_GetLocMTime(flocation_t *location, time_t *modtime)
}
/*opens a vfsfile from an already discovered location*/
vfsfile_t *FS_OpenReadLocation(flocation_t *location)
vfsfile_t *FS_OpenReadLocation(const char *fname, flocation_t *location)
{
if (location->search)
{
return VFS_Filter(NULL, location->search->handle->OpenVFS(location->search->handle, location, "rb"));
return VFS_Filter(fname, location->search->handle->OpenVFS(location->search->handle, location, "rb"));
}
return NULL;
}

1
engine/common/fs.h
View File

@ -105,6 +105,7 @@ qboolean FS_LoadPackageFromFile(vfsfile_t *vfs, char *pname, char *localname, in
#ifdef AVAIL_XZDEC
vfsfile_t *FS_XZ_DecompressWriteFilter(vfsfile_t *infile);
vfsfile_t *FS_XZ_DecompressReadFilter(vfsfile_t *srcfile);
#endif
#ifdef AVAIL_GZDEC
vfsfile_t *FS_GZ_WriteFilter(vfsfile_t *outfile, qboolean autoclosefile, qboolean compress);

35
engine/common/fs_xz.c
View File

@ -3052,11 +3052,11 @@ static int QDECL FS_XZ_Dec_Write(vfsfile_t *f, const void *buffer, int len)
break;
case XZ_MEM_ERROR:
Con_Printf("Memory allocation failed\n");
Con_Printf("XZ: Memory allocation failed\n");
break;
case XZ_MEMLIMIT_ERROR:
Con_Printf("Memory usage limit reached\n");
Con_Printf("XZ: Memory usage limit reached\n");
break;
case XZ_FORMAT_ERROR:
@ -3069,7 +3069,7 @@ static int QDECL FS_XZ_Dec_Write(vfsfile_t *f, const void *buffer, int len)
case XZ_DATA_ERROR:
case XZ_BUF_ERROR:
Con_Printf("File is corrupt\n");
Con_Printf("XZ: File is corrupt\n");
break;
default:
@ -3082,6 +3082,7 @@ static int QDECL FS_XZ_Dec_Write(vfsfile_t *f, const void *buffer, int len)
return n->b.in_pos;
}
//return a write-only wrapper around another file. .xz data written to the wrapper will be seen as decompressed for the wrapee.
vfsfile_t *FS_XZ_DecompressWriteFilter(vfsfile_t *outfile)
{
vf_xz_dec_t *n = Z_Malloc(sizeof(*n));
@ -3116,5 +3117,33 @@ vfsfile_t *FS_XZ_DecompressWriteFilter(vfsfile_t *outfile)
return &n->vf;
}
//returns a read-only filter around a compressed .xz file
vfsfile_t *FS_XZ_DecompressReadFilter(vfsfile_t *srcfile)
{
char block[65536];
int blocksize = VFS_READ(srcfile, block, HEADER_MAGIC_SIZE);
if (blocksize == HEADER_MAGIC_SIZE && !memcmp(block, HEADER_MAGIC, HEADER_MAGIC_SIZE))
{ //okay, looks like an xz.
vfsfile_t *pipe = VFSPIPE_Open(2, false);
vfsfile_t *xzpipe = FS_XZ_DecompressWriteFilter(pipe);
for (;blocksize;)
{
if (blocksize < 0 || blocksize != VFS_WRITE(xzpipe, block, blocksize))
{
VFS_CLOSE(pipe);
pipe = NULL;
break;
}
blocksize = VFS_READ(srcfile, block, sizeof(block));
}
VFS_CLOSE(srcfile);
VFS_CLOSE(xzpipe);
return pipe;
}
VFS_SEEK(srcfile, 0);
return srcfile;
}
#endif

122
engine/common/fs_zip.c
View File

@ -760,6 +760,7 @@ struct decompressstate
{
struct decompressstate *(*Reinit)(zipfile_t *source, qofs_t start, qofs_t csize, qofs_t usize, char *filename, char *password, unsigned int crc);
qofs_t (*Read)(struct decompressstate *st, qbyte *buffer, qofs_t bytes);
qboolean (*Seek)(struct decompressstate *st, qofs_t *fileofs, qofs_t newofs); //may fail, file will be fully decompressed.
void (*Destroy)(struct decompressstate *st);
zipfile_t *source;
@ -846,6 +847,34 @@ static void FSZIP_DecryptBlock(struct decompressstate *st, char *block, size_t b
#endif
#ifdef AVAIL_ZLIB
//if the offset is still within our decompressed block then we can just rewind a smidge
static qboolean FSZIP_Deflate_Seek(struct decompressstate *st, qofs_t *offset, qofs_t newoffset)
{
qofs_t dist;
if (newoffset <= *offset)
{ //rewinding
dist = *offset-newoffset;
if (st->readoffset >= dist)
{
st->readoffset -= dist;
*offset -= dist;
return true;
}
//went too far back, we lost that data.
}
else
{ //seek forward... mneh
dist = newoffset - *offset;
if (st->readoffset+dist <= st->strm.total_out)
{
st->readoffset += dist;
*offset += dist;
return true;
}
//fail. we could just decompress more, but we're probably better off copying to a temp file instead.
}
return false;
}
static qofs_t FSZIP_Deflate_Read(struct decompressstate *st, qbyte *buffer, qofs_t bytes)
{
qboolean eof = false;
@ -925,6 +954,7 @@ static struct decompressstate *FSZIP_Deflate_Init(zipfile_t *source, qofs_t star
st = Z_Malloc(sizeof(*st));
st->Reinit = FSZIP_Deflate_Init;
st->Read = FSZIP_Deflate_Read;
st->Seek = FSZIP_Deflate_Seek;
st->Destroy = FSZIP_Deflate_Destroy;
st->source = source;
@ -960,6 +990,35 @@ static struct decompressstate *FSZIP_Deflate_Init(zipfile_t *source, qofs_t star
#endif
#ifdef AVAIL_BZLIB
//if the offset is still within our decompressed block then we can just rewind a smidge
static qboolean FSZIP_BZip2_Seek(struct decompressstate *st, qofs_t *offset, qofs_t newoffset)
{
qofs_t dist;
if (newoffset <= *offset)
{ //rewinding
dist = *offset-newoffset;
if (st->readoffset >= dist)
{
st->readoffset -= dist;
*offset -= dist;
return true;
}
//went too far back, we lost that data.
}
else
{ //seek forward... mneh
dist = newoffset - *offset;
if (st->readoffset+dist <= st->bstrm.total_out_lo32)
{
st->readoffset += dist;
*offset += dist;
return true;
}
//fail. we could just decompress more, but we're probably better off copying to a temp file instead.
}
return false;
}
//decompress in chunks.
static qofs_t FSZIP_BZip2_Read(struct decompressstate *st, qbyte *buffer, qofs_t bytes)
{
qboolean eof = false;
@ -1040,6 +1099,7 @@ static struct decompressstate *FSZIP_BZip2_Init(zipfile_t *source, qofs_t start,
st = Z_Malloc(sizeof(*st));
st->Reinit = FSZIP_BZip2_Init;
st->Read = FSZIP_BZip2_Read;
st->Seek = FSZIP_BZip2_Seek;
st->Destroy = FSZIP_BZip2_Destroy;
st->source = source;
@ -1073,50 +1133,55 @@ static struct decompressstate *FSZIP_BZip2_Init(zipfile_t *source, qofs_t start,
}
#endif
static vfsfile_t *FSZIP_Decompress_ToTempFile(struct decompressstate *decompress)
{ //if they're going to seek on a file in a zip, let's just copy it out
struct decompressstate *FSZIP_Decompress_Rewind(struct decompressstate *decompress)
{
qofs_t cstart = decompress->cstart, csize = decompress->cend - cstart;
qofs_t upos = 0, usize = decompress->usize;
qofs_t chunk;
struct decompressstate *nc;
qbyte buffer[16384];
vfsfile_t *defer;
qofs_t usize = decompress->usize;
struct decompressstate *(*Reinit)(zipfile_t *source, qofs_t start, qofs_t csize, qofs_t usize, char *filename, char *password, unsigned int crc) = decompress->Reinit;
zipfile_t *source = decompress->source;
#ifdef ZIPCRYPT //we need to preserve any crypto stuff if we're restarting the stream
qboolean encrypted = decompress->encrypted;
unsigned int cryptkeys[3];
const z_crc_t *crctab = decompress->crctable;
memcpy(cryptkeys, decompress->initialkey, sizeof(cryptkeys));
#endif
decompress->Destroy(decompress);
decompress = Reinit(source, cstart, csize, usize, NULL, NULL, 0);
#ifdef ZIPCRYPT
decompress->encrypted = encrypted;
decompress->crctable = crctab;
memcpy(decompress->initialkey, cryptkeys, sizeof(decompress->initialkey));
memcpy(decompress->cryptkey, cryptkeys, sizeof(decompress->cryptkey));
#endif
return decompress;
}
static vfsfile_t *FSZIP_Decompress_ToTempFile(struct decompressstate *decompress)
{ //if they're going to seek on a file in a zip, let's just copy it out
qofs_t upos = 0, usize = decompress->usize;
qofs_t chunk;
qbyte buffer[16384];
vfsfile_t *defer;
defer = FS_OpenTemp();
if (defer)
{
decompress->Destroy(decompress);
decompress = NULL;
nc = Reinit(source, cstart, csize, usize, NULL, NULL, 0);
#ifdef ZIPCRYPT
nc->encrypted = encrypted;
nc->crctable = crctab;
memcpy(nc->initialkey, cryptkeys, sizeof(nc->initialkey));
memcpy(nc->cryptkey, cryptkeys, sizeof(nc->cryptkey));
#endif
decompress = FSZIP_Decompress_Rewind(decompress);
while (upos < usize)
{
chunk = usize - upos;
if (chunk > sizeof(buffer))
chunk = sizeof(buffer);
if (!nc->Read(nc, buffer, chunk))
if (!decompress->Read(decompress, buffer, chunk))
break;
if (VFS_WRITE(defer, buffer, chunk) != chunk)
break;
upos += chunk;
}
nc->Destroy(nc);
decompress->Destroy(decompress);
return defer;
}
@ -1183,11 +1248,18 @@ static qboolean QDECL VFSZIP_Seek (struct vfsfile_s *file, qofs_t pos)
//This is *really* inefficient
if (vfsz->decompress)
{ //if they're going to seek on a file in a zip, let's just copy it out
vfsz->defer = FSZIP_Decompress_ToTempFile(vfsz->decompress);
vfsz->decompress = NULL;
if (vfsz->defer)
return VFS_SEEK(vfsz->defer, pos);
return false;
if (vfsz->decompress->Seek(vfsz->decompress, &vfsz->pos, pos))
return true;
else if (pos == 0)
vfsz->decompress = FSZIP_Decompress_Rewind(vfsz->decompress);
else
{
vfsz->defer = FSZIP_Decompress_ToTempFile(vfsz->decompress);
vfsz->decompress = NULL;
if (vfsz->defer)
return VFS_SEEK(vfsz->defer, pos);
return false;
}
}
if (pos > vfsz->length)

2
engine/common/pr_bgcmd.c
View File

@ -2514,7 +2514,7 @@ static int PF_fopen_search (pubprogfuncs_t *prinst, const char *name, flocation_
Q_strncpyz(pf_fopen_files[i].name, fallbackread, sizeof(pf_fopen_files[i].name));
if (loc->search->handle)
pf_fopen_files[i].file = FS_OpenReadLocation(loc);
pf_fopen_files[i].file = FS_OpenReadLocation(name, loc);
else
pf_fopen_files[i].file = FS_OpenVFS(loc->rawname, "rb", FS_ROOT);

2
engine/gl/gl_font.c
View File

@ -1721,7 +1721,7 @@ qboolean Font_LoadFreeTypeFont(struct font_s *f, int height, const char *fontfil
{
/*File is inside an archive, we need to read it and pass it as memory (and keep it available)*/
vfsfile_t *f;
f = FS_OpenReadLocation(&loc);
f = FS_OpenReadLocation(loc.rawname, &loc);
if (f && loc.len > 0)
{
fbase = BZ_Malloc(loc.len);

2
engine/server/pr_cmds.c
View File

@ -757,7 +757,7 @@ static void *PDECL SSQC_PRReadFile (const char *path, qbyte *(PDECL *buf_get)(vo
if (FS_FLocateFile(path, FSLF_IFFOUND, &loc))
{
qbyte *buffer = NULL;
vfsfile_t *file = FS_OpenReadLocation(&loc);
vfsfile_t *file = FS_OpenReadLocation(path, &loc);
if (file)
{
*size = loc.len;

2
engine/server/savegame.c
View File

@ -2099,7 +2099,7 @@ qboolean SV_Loadgame (const char *unsafe_savename)
}
Q_snprintfz (filename, sizeof(filename), savefiles[best].pattern, savename);
f = FS_OpenReadLocation(&savefiles[best].loc);
f = FS_OpenReadLocation(filename, &savefiles[best].loc);
if (!f)
{
Con_TPrintf ("ERROR: couldn't open %s.\n", filename);

4
engine/server/sv_ccmds.c
View File

@ -452,6 +452,7 @@ static void SV_MapList_f(void)
//FIXME: maps/mapname#modifier.ent
COM_EnumerateFiles("maps/*.bsp", ShowMapList, "");
COM_EnumerateFiles("maps/*.bsp.gz", ShowMapList, ".bsp.gz");
COM_EnumerateFiles("maps/*.bsp.xz", ShowMapList, ".bsp.xz");
COM_EnumerateFiles("maps/*.map", ShowMapList, ".map");
COM_EnumerateFiles("maps/*.map.gz", ShowMapList, ".gz");
COM_EnumerateFiles("maps/*.cm", ShowMapList, ".cm");
@ -485,6 +486,7 @@ static void SV_Map_c(int argn, const char *partial, struct xcommandargcompletion
//FIXME: maps/mapname#modifier.ent
COM_EnumerateFiles(va("maps/%s*.bsp", partial), CompleteMapList, ctx);
COM_EnumerateFiles(va("maps/%s*.bsp.gz", partial), CompleteMapListExt, ctx);
COM_EnumerateFiles(va("maps/%s*.bsp.xz", partial), CompleteMapListExt, ctx);
COM_EnumerateFiles(va("maps/%s*.map", partial), CompleteMapListExt, ctx);
COM_EnumerateFiles(va("maps/%s*.map.gz", partial), CompleteMapListExt, ctx);
COM_EnumerateFiles(va("maps/%s*.cm", partial), CompleteMapList, ctx);
@ -742,7 +744,7 @@ void SV_Map_f (void)
else
#endif
{
char *exts[] = {"maps/%s", "maps/%s.bsp", "maps/%s.bsp.gz", "maps/%s.cm", "maps/%s.hmp", /*"maps/%s.map",*/ /*"maps/%s.ent",*/ NULL};
char *exts[] = {"maps/%s", "maps/%s.bsp", "maps/%s.bsp.gz", "maps/%s.bsp.xz", "maps/%s.cm", "maps/%s.hmp", /*"maps/%s.map",*/ /*"maps/%s.ent",*/ NULL};
int i, j;
for (i = 0; exts[i]; i++)

2
engine/server/sv_init.c
View File

@ -1000,7 +1000,7 @@ void SV_SpawnServer (const char *server, const char *startspot, qboolean noents,
{
//.map is commented out because quite frankly, they're a bit annoying when the engine loads the gpled start.map when really you wanted to just play the damn game intead of take it apart.
//if you want to load a .map, just use 'map foo.map' instead.
char *exts[] = {"maps/%s", "maps/%s.bsp", "maps/%s.cm", "maps/%s.hmp", /*"maps/%s.map",*/ "maps/%s.bsp.gz", NULL};
char *exts[] = {"maps/%s", "maps/%s.bsp", "maps/%s.cm", "maps/%s.hmp", /*"maps/%s.map",*/ "maps/%s.bsp.gz", "maps/%s.bsp.xz", NULL};
int depth, bestdepth;
flocation_t loc;
time_t filetime;

4
engine/server/sv_mvd.c
View File

@ -2811,9 +2811,7 @@ char *SV_MVDName2Txt(char *name)
Q_strncpyz(s, name, MAX_OSPATH);
ext = COM_GetFileExtension(s, NULL);
if (!Q_strcasecmp(ext, ".gz"))
ext = COM_GetFileExtension(s, ext);
else if (!Q_strcasecmp(ext, ".xz"))
if (!Q_strcasecmp(ext, ".gz") || !Q_strcasecmp(ext, ".xz"))
ext = COM_GetFileExtension(s, ext);
if (!ext || !*ext) //if there's no extension on there, then make sure we're pointing to the end of the string.
ext = s+strlen(s);

2
engine/server/sv_user.c
View File

@ -3681,7 +3681,7 @@ void SV_BeginDownload_f(void)
if (result == 0)
{ //if we are allowed and could find it
host_client->download = FS_OpenReadLocation(&loc);
host_client->download = FS_OpenReadLocation(NULL, &loc);
}
}