From 219b3fb9f93eef7f8a6367141ccd6d7f8143ecad Mon Sep 17 00:00:00 2001 From: Christoph Oelckers Date: Sat, 19 Aug 2023 09:24:33 +0200 Subject: [PATCH] - uncoupled directory loader from the rest of the engine. --- src/CMakeLists.txt | 1 + src/common/filesystem/file_directory.cpp | 108 ++++++----- src/common/filesystem/files.cpp | 9 +- src/common/filesystem/fs_findfile.cpp | 232 +++++++++++++++++++++++ src/common/filesystem/fs_findfile.h | 98 ++++++++++ src/common/filesystem/resourcefile.h | 2 + src/common/filesystem/w_zip.h | 15 +- src/d_main.cpp | 20 ++ 8 files changed, 430 insertions(+), 55 deletions(-) create mode 100644 src/common/filesystem/fs_findfile.cpp create mode 100644 src/common/filesystem/fs_findfile.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 518235f66..a35b048bf 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1221,6 +1221,7 @@ set( ZDOOM_SOURCES common/filesystem/resourcefile.cpp common/filesystem/files.cpp common/filesystem/files_decompress.cpp + common/filesystem/fs_findfile.cpp ) diff --git a/src/common/filesystem/file_directory.cpp b/src/common/filesystem/file_directory.cpp index 688a45aa0..51801c166 100644 --- a/src/common/filesystem/file_directory.cpp +++ b/src/common/filesystem/file_directory.cpp @@ -37,8 +37,11 @@ #include #include "resourcefile.h" -#include "cmdlib.h" -#include "findfile.h" +#include "fs_findfile.h" + +#ifdef _WIN32 +std::wstring toWide(const char* str); +#endif //========================================================================== // @@ -51,7 +54,7 @@ struct FDirectoryLump : public FResourceLump FileReader NewReader() override; int FillCache() override; - FString mFullPath; + std::string mFullPath; }; @@ -66,7 +69,7 @@ class FDirectory : public FResourceFile TArray Lumps; const bool nosubdir; - int AddDirectory(const char* dirpath, FileSystemMessageFunc Printf); + int AddDirectory(const char* dirpath, LumpFilterInfo* filter, FileSystemMessageFunc Printf); void AddEntry(const char *fullpath, int size); public: @@ -86,39 +89,47 @@ public: FDirectory::FDirectory(const char * directory, bool nosubdirflag) : FResourceFile(""), nosubdir(nosubdirflag) { - FString dirname; - - #ifdef _WIN32 - directory = _fullpath(NULL, directory, _MAX_PATH); - #else - // Todo for Linux: Resolve the path before using it - #endif - dirname = directory; -#ifdef _WIN32 - free((void *)directory); -#endif - dirname.Substitute("\\", "/"); - if (dirname[dirname.Len()-1] != '/') dirname += '/'; - FileName = dirname; + FileName = FS_FullPath(directory); + if (FileName[FileName.length()-1] != '/') FileName += '/'; } - //========================================================================== // // Windows version // //========================================================================== -int FDirectory::AddDirectory(const char *dirpath, FileSystemMessageFunc Printf) +static bool FS_GetFileInfo(const char* pathname, size_t* size) +{ +#ifndef _WIN32 + struct stat info; + bool res = stat(pathname, &info) == 0; +#else + // Windows must use the wide version of stat to preserve non-ASCII paths. + struct _stat64 info; + bool res = _wstat64(toWide(pathname).c_str(), &info) == 0; +#endif + if (!res || (info.st_mode & S_IFDIR)) return false; + if (size) *size = (size_t)info.st_size; + return res; +} + +//========================================================================== +// +// Windows version +// +//========================================================================== + +int FDirectory::AddDirectory(const char *dirpath, LumpFilterInfo* filter, FileSystemMessageFunc Printf) { void * handle; int count = 0; - FString dirmatch = dirpath; - findstate_t find; + std::string dirmatch = dirpath; dirmatch += '*'; + fs_findstate_t find; - handle = I_FindFirst(dirmatch.GetChars(), &find); + handle = FS_FindFirst(dirmatch.c_str(), &find); if (handle == ((void *)(-1))) { Printf(FSMessageLevel::Error, "Could not scan '%s': %s\n", dirpath, strerror(errno)); @@ -127,15 +138,15 @@ int FDirectory::AddDirectory(const char *dirpath, FileSystemMessageFunc Printf) { do { - // I_FindName only returns the file's name and not its full path - auto attr = I_FindAttr(&find); + // FS_FindName only returns the file's name and not its full path + auto attr = FS_FindAttr(&find); if (attr & FA_HIDDEN) { - // Skip hidden files and directories. (Prevents SVN bookkeeping + // Skip hidden files and directories. (Prevents SVN/Git bookkeeping // info from being included.) continue; } - FString fi = I_FindName(&find); + const char* fi = FS_FindName(&find); if (attr & FA_DIREC) { if (nosubdir || (fi[0] == '.' && @@ -145,9 +156,10 @@ int FDirectory::AddDirectory(const char *dirpath, FileSystemMessageFunc Printf) // Do not record . and .. directories. continue; } - FString newdir = dirpath; - newdir << fi << '/'; - count += AddDirectory(newdir, Printf); + std::string newdir = dirpath; + newdir += fi; + newdir += '/'; + count += AddDirectory(newdir.c_str(), filter, Printf); } else { @@ -157,31 +169,25 @@ int FDirectory::AddDirectory(const char *dirpath, FileSystemMessageFunc Printf) continue; } size_t size = 0; - FString fn = FString(dirpath) + fi; + std::string fn = dirpath; + fn += fi; - // The next one is courtesy of EDuke32. :( - // Putting cache files in the application directory is very bad style. - // Unfortunately, having a garbage file named "texture" present will cause serious problems down the line. - if (!stricmp(fi, "textures")) + if (filter->filenamecheck == nullptr || filter->filenamecheck(fi, fn.c_str())) { - FILE* f = fopen(fn, "rb"); - if (f) + if (FS_GetFileInfo(fn.c_str(), &size)) { - char check[3]{}; - fread(check, 1, 3, f); - if (!memcmp(check, "LZ4", 3)) continue; + if (size > 0x7fffffff) + { + Printf(FSMessageLevel::Warning, "%s is larger than 2GB and will be ignored\n", fn.c_str()); + } + AddEntry(fn.c_str(), (int)size); + count++; } } - - if (GetFileInfo(fn, &size, nullptr)) - { - AddEntry(fn, (int)size); - count++; - } } - } while (I_FindNext (handle, &find) == 0); - I_FindClose (handle); + } while (FS_FindNext (handle, &find) == 0); + FS_FindClose (handle); } return count; } @@ -194,7 +200,7 @@ int FDirectory::AddDirectory(const char *dirpath, FileSystemMessageFunc Printf) bool FDirectory::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf) { - NumLumps = AddDirectory(FileName.c_str(), Printf); + NumLumps = AddDirectory(FileName.c_str(), filter, Printf); PostProcessArchive(&Lumps[0], sizeof(FDirectoryLump), filter); return true; } @@ -234,7 +240,7 @@ void FDirectory::AddEntry(const char *fullpath, int size) FileReader FDirectoryLump::NewReader() { FileReader fr; - fr.OpenFile(mFullPath); + fr.OpenFile(mFullPath.c_str()); return fr; } @@ -248,7 +254,7 @@ int FDirectoryLump::FillCache() { FileReader fr; Cache = new char[LumpSize]; - if (!fr.OpenFile(mFullPath)) + if (!fr.OpenFile(mFullPath.c_str())) { throw FileSystemException("unable to open file"); } diff --git a/src/common/filesystem/files.cpp b/src/common/filesystem/files.cpp index cbefc7bbc..094bd8a99 100644 --- a/src/common/filesystem/files.cpp +++ b/src/common/filesystem/files.cpp @@ -33,18 +33,21 @@ ** */ +#include #include "files.h" -#include "utf8.h" #include "stb_sprintf.h" +#ifdef _WIN32 +std::wstring toWide(const char* str); +#endif FILE *myfopen(const char *filename, const char *flags) { #ifndef _WIN32 return fopen(filename, flags); #else - auto widename = WideString(filename); - auto wideflags = WideString(flags); + auto widename = toWide(filename); + auto wideflags = toWide(flags); return _wfopen(widename.c_str(), wideflags.c_str()); #endif } diff --git a/src/common/filesystem/fs_findfile.cpp b/src/common/filesystem/fs_findfile.cpp new file mode 100644 index 000000000..7e47b33f4 --- /dev/null +++ b/src/common/filesystem/fs_findfile.cpp @@ -0,0 +1,232 @@ +/* +** findfile.cpp +** Wrapper around the native directory scanning APIs +** +**--------------------------------------------------------------------------- +** Copyright 1998-2016 Randy Heit +** Copyright 2005-2020 Christoph Oelckers +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions +** are met: +** +** 1. Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** 2. Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** 3. The name of the author may not be used to endorse or promote products +** derived from this software without specific prior written permission. +** +** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +** IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +** OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +** IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +** INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +** NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +** THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**--------------------------------------------------------------------------- +** +*/ + +#include "fs_findfile.h" + +#ifndef _WIN32 + +#include +#include +#include + +static const char *pattern; + +static int matchfile(const struct dirent *ent) +{ + return fnmatch(pattern, ent->d_name, FNM_NOESCAPE) == 0; +} + +void *FS_FindFirst(const char *const filespec, fs_findstate_t *const fileinfo) +{ + const char* dir; + + const char *const slash = strrchr(filespec, '/'); + + if (slash) + { + pattern = slash + 1; + fileinfo->path = std::string(filespec, slash - filespec + 1); + dir = fileinfo->path.c_str(); + } + else + { + pattern = filespec; + dir = "."; + } + + fileinfo->current = 0; + fileinfo->count = scandir(dir, &fileinfo->namelist, matchfile, alphasort); + + if (fileinfo->count > 0) + { + return fileinfo; + } + + return (void *)-1; +} + +int FS_FindNext(void *const handle, fs_findstate_t *const fileinfo) +{ + fs_findstate_t *const state = static_cast(handle); + + if (state->current < fileinfo->count) + { + return ++state->current < fileinfo->count ? 0 : -1; + } + + return -1; +} + +int FS_FindClose(void *const handle) +{ + fs_findstate_t *const state = static_cast(handle); + + if (handle != (void *)-1 && state->count > 0) + { + for (int i = 0; i < state->count; ++i) + { + free(state->namelist[i]); + } + + free(state->namelist); + state->namelist = nullptr; + state->count = 0; + } + + return 0; +} + +static bool DirEntryExists(const char* pathname, bool* isdir) +{ + if (isdir) *isdir = false; + struct stat info; + bool res = stat(pathname, &info) == 0; + if (isdir) *isdir = !!(info.st_mode & S_IFDIR); + return res; +} + +int FS_FindAttr(fs_findstate_t *const fileinfo) +{ + dirent *const ent = fileinfo->namelist[fileinfo->current]; + const std::string path = fileinfo->path + ent->d_name; + bool isdir; + + if (DirEntryExists(path.c_str(), &isdir)) + { + return isdir ? FA_DIREC : 0; + } + + return 0; +} + +std::string FS_FullPath(const char* directory) +{ + // todo + return directory +} + +#else + +#include +#include + +std::wstring toWide(const char* str) +{ + int len = (int)strlen(str); + int size_needed = MultiByteToWideChar(CP_UTF8, 0, str, len, nullptr, 0); + std::wstring wide; + wide.resize(size_needed); + MultiByteToWideChar(CP_UTF8, 0, str, len, &wide.front(), size_needed); + return wide; +} + +static std::string toUtf8(const wchar_t* str) +{ + auto len = wcslen(str); + int size_needed = WideCharToMultiByte(CP_UTF8, 0, str, (int)len, nullptr, 0, nullptr, nullptr); + std::string utf8; + utf8.resize(size_needed); + WideCharToMultiByte(CP_UTF8, 0, str, (int)len, &utf8.front(), size_needed, nullptr, nullptr); + return utf8; +} + +//========================================================================== +// +// FS_FindFirst +// +// Start a pattern matching sequence. +// +//========================================================================== + + +void *FS_FindFirst(const char *filespec, fs_findstate_t *fileinfo) +{ + static_assert(sizeof(WIN32_FIND_DATAW) == sizeof(fileinfo->FindData), "FindData size mismatch"); + fileinfo->UTF8Name = ""; + return FindFirstFileW(toWide(filespec).c_str(), (LPWIN32_FIND_DATAW)&fileinfo->FindData); +} + +//========================================================================== +// +// FS_FindNext +// +// Return the next file in a pattern matching sequence. +// +//========================================================================== + +int FS_FindNext(void *handle, fs_findstate_t *fileinfo) +{ + fileinfo->UTF8Name = ""; + return FindNextFileW((HANDLE)handle, (LPWIN32_FIND_DATAW)&fileinfo->FindData) == 0; +} + +//========================================================================== +// +// FS_FindClose +// +// Finish a pattern matching sequence. +// +//========================================================================== + +int FS_FindClose(void *handle) +{ + return FindClose((HANDLE)handle); +} + +//========================================================================== +// +// FS_FindName +// +// Returns the name for an entry +// +//========================================================================== + +const char *FS_FindName(fs_findstate_t *fileinfo) +{ + if (fileinfo->UTF8Name.empty()) + { + fileinfo->UTF8Name = toUtf8(fileinfo->FindData.Name); + } + return fileinfo->UTF8Name.c_str(); +} + +std::string FS_FullPath(const char* directory) +{ + auto wdirectory = _wfullpath(nullptr, toWide(directory).c_str(), _MAX_PATH); + std::string sdirectory = toUtf8(wdirectory); + free((void*)wdirectory); + for (auto& c : sdirectory) if (c == '\\') c = '/'; + return sdirectory; +} +#endif diff --git a/src/common/filesystem/fs_findfile.h b/src/common/filesystem/fs_findfile.h new file mode 100644 index 000000000..528c428c3 --- /dev/null +++ b/src/common/filesystem/fs_findfile.h @@ -0,0 +1,98 @@ +#pragma once +// Directory searching routines + +#include +#include + +enum +{ + ZPATH_MAX = 260 +}; + +#ifndef _WIN32 + +#include + +struct fs_findstate_t +{ +private: + std::string path; + struct dirent **namelist; + int current; + int count; + + friend void *FS_FindFirst(const char *filespec, fs_findstate_t *fileinfo); + friend int FS_FindNext(void *handle, fs_findstate_t *fileinfo); + friend const char *FS_FindName(fs_findstate_t *fileinfo); + friend int FS_FindAttr(fs_findstate_t *fileinfo); + friend int FS_FindClose(void *handle); +}; + +int FS_FindAttr (fs_findstate_t *fileinfo); + +inline const char *FS_FindName(fs_findstate_t *fileinfo) +{ + return (fileinfo->namelist[fileinfo->current]->d_name); +} + +enum +{ + FA_RDONLY = 1, + FA_HIDDEN = 2, + FA_SYSTEM = 4, + FA_DIREC = 8, + FA_ARCH = 16, +}; + + +#else + + +struct fs_findstate_t +{ +private: + struct FileTime + { + uint32_t lo, hi; + }; + // Mirror WIN32_FIND_DATAW in . We cannot pull in the Windows header here as it would pollute all consumers' symbol space. + struct WinData + { + uint32_t Attribs; + FileTime Times[3]; + uint32_t Size[2]; + uint32_t Reserved[2]; + wchar_t Name[ZPATH_MAX]; + wchar_t AltName[14]; + }; + WinData FindData; + std::string UTF8Name; + + friend void *FS_FindFirst(const char *filespec, fs_findstate_t *fileinfo); + friend int FS_FindNext(void *handle, fs_findstate_t *fileinfo); + friend const char *FS_FindName(fs_findstate_t *fileinfo); + friend int FS_FindAttr(fs_findstate_t *fileinfo); +}; + + +const char *FS_FindName(fs_findstate_t *fileinfo); +inline int FS_FindAttr(fs_findstate_t *fileinfo) +{ + return fileinfo->FindData.Attribs; +} + +enum +{ + FA_RDONLY = 1, + FA_HIDDEN = 2, + FA_SYSTEM = 4, + FA_DIREC = 16, + FA_ARCH = 32, +}; + +#endif + +void *FS_FindFirst (const char *filespec, fs_findstate_t *fileinfo); +int FS_FindNext (void *handle, fs_findstate_t *fileinfo); +int FS_FindClose (void *handle); +std::string FS_FullPath(const char* directory); diff --git a/src/common/filesystem/resourcefile.h b/src/common/filesystem/resourcefile.h index ff785747b..b2a81ac26 100644 --- a/src/common/filesystem/resourcefile.h +++ b/src/common/filesystem/resourcefile.h @@ -21,6 +21,8 @@ struct LumpFilterInfo std::vector reservedFolders; std::vector requiredPrefixes; std::vector embeddings; + std::vector blockednames; // File names that will never be accepted (e.g. dehacked.exe for Doom) + std::function filenamecheck; // for scanning directories, this allows to eliminate unwanted content. std::function postprocessFunc; }; diff --git a/src/common/filesystem/w_zip.h b/src/common/filesystem/w_zip.h index ddc065f07..c0905cd20 100644 --- a/src/common/filesystem/w_zip.h +++ b/src/common/filesystem/w_zip.h @@ -1,7 +1,11 @@ #ifndef __W_ZIP #define __W_ZIP -#include "basics.h" +#if defined(__GNUC__) +#define FORCE_PACKED __attribute__((__packed__)) +#else +#define FORCE_PACKED +#endif #pragma pack(1) // FZipCentralInfo @@ -85,6 +89,15 @@ struct FZipLocalFileHeader #pragma pack() +#ifndef MAKE_ID +#ifndef __BIG_ENDIAN__ +#define MAKE_ID(a,b,c,d) ((uint32_t)((a)|((b)<<8)|((c)<<16)|((d)<<24))) +#else +#define MAKE_ID(a,b,c,d) ((uint32_t)((d)|((c)<<8)|((b)<<16)|((a)<<24))) +#endif +#endif + + #define ZIP_LOCALFILE MAKE_ID('P','K',3,4) #define ZIP_CENTRALFILE MAKE_ID('P','K',1,2) #define ZIP_ENDOFDIR MAKE_ID('P','K',5,6) diff --git a/src/d_main.cpp b/src/d_main.cpp index 937f04bc8..d96467158 100644 --- a/src/d_main.cpp +++ b/src/d_main.cpp @@ -3034,6 +3034,26 @@ static FILE* D_GetHashFile() return hashfile; } +// checks if a file within a directory is allowed to be added to the file system. +static bool FileNameCheck(const char* base, const char* path) +{ + // This one is courtesy of EDuke32. :( + // Putting cache files in the application directory is very bad style. + // Unfortunately, having a garbage file named "textures" present will cause serious problems down the line. + if (!strnicmp(base, "textures", 8)) + { + // do not use fopen. The path may contain non-ASCII characters. + FileReader f; + if (f.OpenFile(path)) + { + char check[3]{}; + f.Read(check, 3); + if (!memcmp(check, "LZ4", 3)) return false; + } + } + return true; +} + static int FileSystemPrintf(FSMessageLevel level, const char* fmt, ...) { va_list arg;