diff --git a/src/common/filesystem/include/fs_files.h b/src/common/filesystem/include/fs_files.h index a3f71f159..34536200d 100644 --- a/src/common/filesystem/include/fs_files.h +++ b/src/common/filesystem/include/fs_files.h @@ -85,6 +85,13 @@ enum METHOD_PPMD = 98, METHOD_LZSS = 1337, // not used in Zips - this is for Console Doom compression METHOD_ZLIB = 1338, // Zlib stream with header, used by compressed nodes. + METHOD_IMPLODE_MIN = 1000, // having discrete types for these avoids keeping around the GPFlags word in Zips. + METHOD_IMPLODE_0 = 1000, + METHOD_IMPLODE_2 = 1002, + METHOD_IMPLODE_4 = 1004, + METHOD_IMPLODE_6 = 1006, + METHOD_IMPLODE_MAX = 1006, + METHOD_INVALID = 0x7fff, METHOD_TRANSFEROWNER = 0x8000, }; diff --git a/src/common/filesystem/include/resourcefile.h b/src/common/filesystem/include/resourcefile.h index 180bed931..9c1691db6 100644 --- a/src/common/filesystem/include/resourcefile.h +++ b/src/common/filesystem/include/resourcefile.h @@ -82,6 +82,14 @@ enum ELumpFlags LUMPF_EMBEDDED = 4, // marks an embedded resource file for later processing. LUMPF_SHORTNAME = 8, // the stored name is a short extension-less name LUMPF_COMPRESSED = 16, // compressed or encrypted, i.e. cannot be read with the container file's reader. + + RESFF_MAYBEFLAT = 1, // might be a flat inside a WAD outside F_START/END + RESFF_FULLPATH = 2, // contains a full path. This will trigger extended namespace checks when looking up short names. + RESFF_EMBEDDED = 4, // marks an embedded resource file for later processing. + RESFF_SHORTNAME = 8, // the stored name is a short extension-less name + RESFF_COMPRESSED = 16, // compressed or encrypted, i.e. cannot be read with the container file's reader. + RESFF_NEEDFILESTART = 32, // The real position is not known yet and needs to be calculated on access + }; // This holds a compresed Zip entry with all needed info to decompress it. @@ -172,12 +180,26 @@ protected: }; +struct FResourceEntry +{ + size_t Length; + size_t CompressedSize; + const char* FileName; + size_t Position; + int ResourceID; + uint32_t CRC32; + uint16_t Flags; + uint16_t Method; + int16_t Namespace; +}; + class FResourceFile { public: protected: FileReader Reader; const char* FileName; + FResourceEntry* Entries = nullptr; uint32_t NumLumps; char Hash[48]; StringPool* stringpool; @@ -185,6 +207,9 @@ protected: FResourceFile(const char *filename, StringPool* sp); FResourceFile(const char *filename, FileReader &r, StringPool* sp); + const char* NormalizeFileName(const char* fn, int fallbackcp = 0); + void AllocateEntries(int count); + // for archives that can contain directories void GenerateHash(); void PostProcessArchive(void *lumps, size_t lumpsize, LumpFilterInfo *filter); @@ -214,15 +239,13 @@ public: int EntryCount() const { return NumLumps; } int FindEntry(const char* name); - size_t Length(int entry) + size_t Length(uint32_t entry) { - auto l = GetLump(entry); - return l ? l->LumpSize : -1; + return (entry < NumLumps) ? Entries[entry].Length : 0; } - size_t Offset(int entry) + size_t Offset(uint32_t entry) { - auto l = GetLump(entry); - return l ? l->GetFileOffset() : -1; + return (entry < NumLumps) ? Entries[entry].Position : 0; } FileReader GetEntryReader(int entry, bool newreader = true) @@ -231,22 +254,24 @@ public: return l ? l->NewReader() : FileReader(); } - int GetEntryFlags(int entry) + int GetEntryFlags(uint32_t entry) { - auto l = GetLump(entry); - return l ? l->Flags : 0; + return (entry < NumLumps) ? Entries[entry].Flags : 0; } - int GetEntryNamespace(int entry) + int GetEntryNamespace(uint32_t entry) { - auto l = GetLump(entry); - return l ? l->GetNamespace() : 0; + return (entry < NumLumps) ? Entries[entry].Namespace : ns_hidden; } - int GetEntryResourceID(int entry) + int GetEntryResourceID(uint32_t entry) { - auto l = GetLump(entry); - return l ? l->GetIndexNum() : 0; + return (entry < NumLumps) ? Entries[entry].ResourceID : -1; + } + + const char* getName(uint32_t entry) + { + return (entry < NumLumps) ? Entries[entry].FileName : nullptr; } FileData Read(int entry) @@ -255,11 +280,6 @@ public: return fr.Read(); } - const char* getName(int entry) - { - auto l = GetLump(entry); - return l ? l->FullName : nullptr; - } FCompressedBuffer GetRawData(int entry) { auto l = GetLump(entry); diff --git a/src/common/filesystem/source/file_7z.cpp b/src/common/filesystem/source/file_7z.cpp index 0740ceca1..42bb053db 100644 --- a/src/common/filesystem/source/file_7z.cpp +++ b/src/common/filesystem/source/file_7z.cpp @@ -38,6 +38,7 @@ #include "7zCrc.h" #include "resourcefile.h" #include "fs_findfile.h" +#include "unicode.h" namespace FileSys { @@ -247,12 +248,13 @@ bool F7ZFile::Open(LumpFilterInfo *filter, FileSystemMessageFunc Printf) CSzArEx* const archPtr = &Archive->DB; + AllocateEntries(archPtr->NumFiles); NumLumps = archPtr->NumFiles; Lumps = new F7ZLump[NumLumps]; F7ZLump *lump_p = Lumps; std::u16string nameUTF16; - std::string nameASCII; + std::vector nameASCII; for (uint32_t i = 0; i < NumLumps; ++i) { @@ -273,15 +275,19 @@ bool F7ZFile::Open(LumpFilterInfo *filter, FileSystemMessageFunc Printf) nameUTF16.resize((unsigned)nameLength); nameASCII.resize((unsigned)nameLength); - // note that the file system is not equipped to handle non-ASCII, so don't bother with proper Unicode conversion here. - SzArEx_GetFileNameUtf16(archPtr, i, (UInt16*)nameUTF16.data()); - for (size_t c = 0; c < nameLength; ++c) - { - nameASCII[c] = tolower(static_cast(nameUTF16[c])); - } - FixPathSeparator(&nameASCII.front()); - lump_p->LumpNameSetup(nameASCII.c_str(), stringpool); + SzArEx_GetFileNameUtf16(archPtr, i, (UInt16*)nameUTF16.data()); + utf16_to_utf8((uint16_t*)nameUTF16.data(), nameASCII); + + Entries[i].FileName = NormalizeFileName(nameASCII.data()); + Entries[i].Length = SzArEx_GetFileSize(archPtr, i); + Entries[i].Flags = RESFF_FULLPATH|RESFF_COMPRESSED; + Entries[i].ResourceID = -1; + Entries[i].Namespace = ns_global; + Entries[i].Method = METHOD_INVALID; + Entries[i].Position = i; + + lump_p->LumpNameSetup(nameASCII.data(), stringpool); lump_p->LumpSize = static_cast(SzArEx_GetFileSize(archPtr, i)); lump_p->Owner = this; lump_p->Flags = LUMPF_FULLPATH|LUMPF_COMPRESSED; diff --git a/src/common/filesystem/source/file_directory.cpp b/src/common/filesystem/source/file_directory.cpp index 89b9625fc..9d96eacad 100644 --- a/src/common/filesystem/source/file_directory.cpp +++ b/src/common/filesystem/source/file_directory.cpp @@ -56,10 +56,10 @@ std::wstring toWide(const char* str); struct FDirectoryLump : public FResourceLump { + const char* mBasePath; FileReader NewReader() override; int FillCache() override; - const char* mFullPath; }; @@ -73,9 +73,10 @@ class FDirectory : public FResourceFile { TArray Lumps; const bool nosubdir; + const char* mBasePath; int AddDirectory(const char* dirpath, LumpFilterInfo* filter, FileSystemMessageFunc Printf); - void AddEntry(const char *fullpath, const char* relpath, int size); + void AddEntry(const char* relpath, int size); public: FDirectory(const char * dirname, StringPool* sp, bool nosubdirflag = false); @@ -116,8 +117,17 @@ int FDirectory::AddDirectory(const char *dirpath, LumpFilterInfo* filter, FileSy } else { + mBasePath = nullptr; + AllocateEntries(list.size()); for(auto& entry : list) { + if (mBasePath == nullptr) + { + // extract the base path from the first entry to cover changes made in ScanDirectory. + auto full = entry.FilePath.find(entry.FilePathRel); + std::string path(entry.FilePath, full); + mBasePath = stringpool->Strdup(path.c_str()); + } if (!entry.isDirectory) { auto fi = entry.FileName; @@ -128,6 +138,7 @@ int FDirectory::AddDirectory(const char *dirpath, LumpFilterInfo* filter, FileSy continue; } + if (filter->filenamecheck == nullptr || filter->filenamecheck(fi.c_str(), entry.FilePath.c_str())) { if (entry.Length > 0x7fffffff) @@ -135,7 +146,17 @@ int FDirectory::AddDirectory(const char *dirpath, LumpFilterInfo* filter, FileSy Printf(FSMessageLevel::Warning, "%s is larger than 2GB and will be ignored\n", entry.FilePath.c_str()); continue; } - AddEntry(entry.FilePath.c_str(), entry.FilePathRel.c_str(), (int)entry.Length); + AddEntry(entry.FilePathRel.c_str(), (int)entry.Length); + int index = count; + // for internal access we use the normalized form of the relative path. + Entries[index].FileName = NormalizeFileName(entry.FilePathRel.c_str()); + Entries[index].Length = entry.Length; + Entries[index].Flags = RESFF_FULLPATH; + Entries[index].ResourceID = -1; + Entries[index].Method = METHOD_STORED; + Entries[index].Namespace = ns_global; + index++; + count++; } } @@ -163,13 +184,10 @@ bool FDirectory::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf) // //========================================================================== -void FDirectory::AddEntry(const char *fullpath, const char* relpath, int size) +void FDirectory::AddEntry(const char* relpath, int size) { FDirectoryLump *lump_p = &Lumps[Lumps.Reserve(1)]; - // Store the full path here so that we can access the file later, even if it is from a filter directory. - lump_p->mFullPath = stringpool->Strdup(fullpath); - // [mxd] Convert name to lowercase std::string name = relpath; for (auto& c : name) c = tolower(c); @@ -192,7 +210,8 @@ void FDirectory::AddEntry(const char *fullpath, const char* relpath, int size) FileReader FDirectoryLump::NewReader() { FileReader fr; - fr.OpenFile(mFullPath); + std::string fn = mBasePath; fn += this->FullName; + fr.OpenFile(fn.c_str()); return fr; } @@ -206,7 +225,8 @@ int FDirectoryLump::FillCache() { FileReader fr; Cache = new char[LumpSize]; - if (!fr.OpenFile(mFullPath)) + std::string fn = mBasePath; fn += this->FullName; + if (!fr.OpenFile(fn.c_str())) { throw FileSystemException("unable to open file"); } diff --git a/src/common/filesystem/source/file_grp.cpp b/src/common/filesystem/source/file_grp.cpp index 05299f940..3af7746a5 100644 --- a/src/common/filesystem/source/file_grp.cpp +++ b/src/common/filesystem/source/file_grp.cpp @@ -101,6 +101,7 @@ bool FGrpFile::Open(LumpFilterInfo* filter) GrpHeader header; Reader.Read(&header, sizeof(header)); + AllocateEntries(header.NumLumps); NumLumps = LittleLong(header.NumLumps); GrpLump *fileinfo = new GrpLump[NumLumps]; @@ -112,12 +113,21 @@ bool FGrpFile::Open(LumpFilterInfo* filter) for(uint32_t i = 0; i < NumLumps; i++) { + Entries[i].Position = Position; + Entries[i].Length = LittleLong(fileinfo[i].Size); + Position += fileinfo[i].Size; + Entries[i].Flags = 0; + Entries[i].Namespace = ns_global; + fileinfo[i].NameWithZero[12] = '\0'; // Be sure filename is null-terminated + Entries[i].ResourceID = -1; + Entries[i].Method = METHOD_STORED; + Entries[i].FileName = NormalizeFileName(fileinfo[i].Name); + Lumps[i].Owner = this; Lumps[i].Position = Position; - Lumps[i].LumpSize = LittleLong(fileinfo[i].Size); + Lumps[i].LumpSize = Entries[i].Length; Position += fileinfo[i].Size; Lumps[i].Flags = 0; - fileinfo[i].NameWithZero[12] = '\0'; // Be sure filename is null-terminated Lumps[i].LumpNameSetup(fileinfo[i].NameWithZero, stringpool); } GenerateHash(); diff --git a/src/common/filesystem/source/file_lump.cpp b/src/common/filesystem/source/file_lump.cpp index 94da2af4f..07979ee12 100644 --- a/src/common/filesystem/source/file_lump.cpp +++ b/src/common/filesystem/source/file_lump.cpp @@ -68,6 +68,15 @@ FLumpFile::FLumpFile(const char *filename, FileReader &file, StringPool* sp) bool FLumpFile::Open(LumpFilterInfo*) { + AllocateEntries(1); + Entries[0].FileName = NormalizeFileName(ExtractBaseName(FileName, true).c_str()); + Entries[0].Namespace = ns_global; + Entries[0].ResourceID = -1; + Entries[0].Position = 0; + Entries[0].Length = Reader.GetLength(); + Entries[0].Method = METHOD_STORED; + Entries[0].Flags = 0; + Lumps.Resize(1); Lumps[0].LumpNameSetup(ExtractBaseName(FileName, true).c_str(), stringpool); Lumps[0].Owner = this; diff --git a/src/common/filesystem/source/file_pak.cpp b/src/common/filesystem/source/file_pak.cpp index 12ab2f4e2..4c5df1b3a 100644 --- a/src/common/filesystem/source/file_pak.cpp +++ b/src/common/filesystem/source/file_pak.cpp @@ -95,6 +95,7 @@ bool FPakFile::Open(LumpFilterInfo* filter) dpackheader_t header; Reader.Read(&header, sizeof(header)); + AllocateEntries(header.dirlen / sizeof(dpackfile_t)); NumLumps = LittleLong(header.dirlen) / sizeof(dpackfile_t); header.dirofs = LittleLong(header.dirofs); @@ -106,6 +107,14 @@ bool FPakFile::Open(LumpFilterInfo* filter) for(uint32_t i = 0; i < NumLumps; i++) { + Entries[i].Position = LittleLong(fileinfo[i].filepos); + Entries[i].Length = LittleLong(fileinfo[i].filelen); + Entries[i].Flags = RESFF_FULLPATH; + Entries[i].Namespace = ns_global; + Entries[i].ResourceID = -1; + Entries[i].Method = METHOD_STORED; + Entries[i].FileName = NormalizeFileName(fileinfo[i].name); + Lumps[i].LumpNameSetup(fileinfo[i].name, stringpool); Lumps[i].Flags = LUMPF_FULLPATH; Lumps[i].Owner = this; diff --git a/src/common/filesystem/source/file_rff.cpp b/src/common/filesystem/source/file_rff.cpp index dff7b69e9..a70a5547a 100644 --- a/src/common/filesystem/source/file_rff.cpp +++ b/src/common/filesystem/source/file_rff.cpp @@ -143,6 +143,7 @@ bool FRFFFile::Open(LumpFilterInfo*) Reader.Read(&header, sizeof(header)); + AllocateEntries(LittleLong(header.NumLumps)); NumLumps = LittleLong(header.NumLumps); header.DirOfs = LittleLong(header.DirOfs); lumps = new RFFLump[header.NumLumps]; @@ -154,6 +155,23 @@ bool FRFFFile::Open(LumpFilterInfo*) for (uint32_t i = 0; i < NumLumps; ++i) { + Entries[i].Position = LittleLong(lumps[i].FilePos); + Entries[i].Length = LittleLong(lumps[i].Size); + Entries[i].Flags = 0; + Entries[i].Method = METHOD_STORED; + if (lumps[i].Flags & 0x10) + { + Entries[i].Flags = RESFF_COMPRESSED; // flags the lump as not directly usable + Entries[i].Method = METHOD_INVALID; + } + else + { + Entries[i].Flags = 0; + Entries[i].Method = METHOD_STORED; + } + Entries[i].Namespace = ns_global; + Entries[i].ResourceID = LittleLong(lumps[i].IndexNum); + Lumps[i].Position = LittleLong(lumps[i].FilePos); Lumps[i].LumpSize = LittleLong(lumps[i].Size); Lumps[i].Owner = this; @@ -174,6 +192,7 @@ bool FRFFFile::Open(LumpFilterInfo*) name[len+3] = lumps[i].Extension[2]; name[len+4] = 0; Lumps[i].LumpNameSetup(name, stringpool); + Entries[i].FileName = NormalizeFileName(name); } delete[] lumps; GenerateHash(); diff --git a/src/common/filesystem/source/file_ssi.cpp b/src/common/filesystem/source/file_ssi.cpp index f23c8e5ea..3f7e155a4 100644 --- a/src/common/filesystem/source/file_ssi.cpp +++ b/src/common/filesystem/source/file_ssi.cpp @@ -70,6 +70,7 @@ FSSIFile::FSSIFile(const char *filename, FileReader &file, StringPool* sp) bool FSSIFile::Open(int version, int EntryCount, LumpFilterInfo*) { + AllocateEntries(EntryCount*2); NumLumps = EntryCount*2; Lumps.Resize(EntryCount*2); @@ -85,7 +86,30 @@ bool FSSIFile::Open(int version, int EntryCount, LumpFilterInfo*) fn[strlength] = 0; int flength = Reader.ReadInt32(); + Entries[i].Position = j; + Entries[i].Length = flength; + Entries[i].Flags = 0; + Entries[i].Namespace = ns_global; + Entries[i].Method = METHOD_STORED; + Entries[i].ResourceID = -1; + Entries[i].FileName = NormalizeFileName(fn); + if (strstr(fn, ".GRP")) Entries[i].Flags |= RESFF_EMBEDDED; + // SSI files can swap the order of the extension's characters - but there's no reliable detection for this and it can be mixed inside the same container, + // so we have no choice but to create another file record for the altered name. + std::swap(fn[strlength - 1], fn[strlength - 3]); + + Entries[i + 1].Position = j; + Entries[i + 1].Length = flength; + Entries[i + 1].Flags = 0; + Entries[i + 1].Namespace = ns_global; + Entries[i + 1].ResourceID = -1; + Entries[i + 1].FileName = NormalizeFileName(fn); + Entries[i + 1].Method = METHOD_STORED; + if (strstr(fn, ".GRP")) Entries[i + 1].Flags |= RESFF_EMBEDDED; + + // swap back... + std::swap(fn[strlength - 1], fn[strlength - 3]); Lumps[i].LumpNameSetup(fn, stringpool); Lumps[i].Position = j; Lumps[i].LumpSize = flength; diff --git a/src/common/filesystem/source/file_wad.cpp b/src/common/filesystem/source/file_wad.cpp index 3451b7da8..4ed28c668 100644 --- a/src/common/filesystem/source/file_wad.cpp +++ b/src/common/filesystem/source/file_wad.cpp @@ -37,6 +37,7 @@ #include "resourcefile.h" #include "fs_filesystem.h" #include "fs_swap.h" +#include "fs_stringpool.h" namespace FileSys { using namespace byteswap; @@ -192,8 +193,37 @@ bool FWadFile::Open(LumpFilterInfo*, FileSystemMessageFunc Printf) Reader.Seek (InfoTableOfs, FileReader::SeekSet); Reader.Read (fileinfo.Data(), NumLumps * sizeof(wadlump_t)); + AllocateEntries(NumLumps); Lumps.Resize(NumLumps); + for(uint32_t i = 0; i < NumLumps; i++) + { + // WAD only supports ASCII. It is also the only format which can use valid backslashes in its names. + char n[9]; + int ishigh = 0; + for (int j = 0; j < 8; j++) + { + if (fileinfo[i].Name[j] & 0x80) ishigh |= 1 << j; + n[j] = tolower(fileinfo[i].Name[j]); + } + n[8] = 0; + if (ishigh == 1) n[0] &= 0x7f; + else if (ishigh > 1) + { + // This may not end up printing something proper because we do not know what encoding might have been used. + Printf(FSMessageLevel::Warning, "%s: Lump name %s contains invalid characters\n", FileName, Lumps[i].getName()); + } + + Entries[i].FileName = nullptr; + Entries[i].Position = isBigEndian ? BigLong(fileinfo[i].FilePos) : LittleLong(fileinfo[i].FilePos); + Entries[i].Length = isBigEndian ? BigLong(fileinfo[i].Size) : LittleLong(fileinfo[i].Size); + Entries[i].Namespace = ns_global; + Entries[i].Flags = ishigh? RESFF_SHORTNAME | RESFF_COMPRESSED : RESFF_SHORTNAME; + Entries[i].Method = ishigh == 1? METHOD_LZSS : METHOD_STORED; + Entries[i].FileName = stringpool->Strdup(n); + // This doesn't set up the namespace yet. + } + for(uint32_t i = 0; i < NumLumps; i++) { char n[9]; @@ -238,6 +268,11 @@ bool FWadFile::Open(LumpFilterInfo*, FileSystemMessageFunc Printf) SetNamespace("HI_START", "HI_END", ns_hires, Printf); SetNamespace("VX_START", "VX_END", ns_voxels, Printf); SkinHack(Printf); + + for (uint32_t i = 0; i < NumLumps; i++) + { + Entries[i].Namespace = Lumps[i].Namespace; + } return true; } diff --git a/src/common/filesystem/source/file_whres.cpp b/src/common/filesystem/source/file_whres.cpp index ffa8dd8a5..3ed442489 100644 --- a/src/common/filesystem/source/file_whres.cpp +++ b/src/common/filesystem/source/file_whres.cpp @@ -84,24 +84,43 @@ bool FWHResFile::Open(LumpFilterInfo*) int nl =1024/3; Lumps.Resize(nl); + int k; + for (k = 0; k < nl; k++) + { + uint32_t offset = LittleLong(directory[k * 3]) * 4096; + uint32_t length = LittleLong(directory[k * 3 + 1]); + if (length == 0) + { + break; + } + } + AllocateEntries(k); + NumLumps = k; int i = 0; - for(int k = 0; k < nl; k++) + for(k = 0; k < NumLumps; k++) { uint32_t offset = LittleLong(directory[k*3]) * 4096; uint32_t length = LittleLong(directory[k*3+1]); - if (length == 0) break; char num[6]; snprintf(num, 6, "/%04d", k); std::string synthname = BaseName; synthname += num; + + Entries[i].Position = offset; + Entries[i].Length = length; + Entries[i].Flags = RESFF_FULLPATH; + Entries[i].Namespace = ns_global; + Entries[i].ResourceID = -1; + Entries[i].Method = METHOD_STORED; + Entries[i].FileName = NormalizeFileName(synthname.c_str()); + Lumps[i].LumpNameSetup(synthname.c_str(), stringpool); Lumps[i].Owner = this; Lumps[i].Position = offset; Lumps[i].LumpSize = length; i++; } - NumLumps = i; Lumps.Clamp(NumLumps); Lumps.ShrinkToFit(); return true; diff --git a/src/common/filesystem/source/file_zip.cpp b/src/common/filesystem/source/file_zip.cpp index dfeb41ea8..9847cdba4 100644 --- a/src/common/filesystem/source/file_zip.cpp +++ b/src/common/filesystem/source/file_zip.cpp @@ -40,6 +40,7 @@ #include "ancientzip.h" #include "fs_findfile.h" #include "fs_swap.h" +#include "fs_stringpool.h" namespace FileSys { using namespace byteswap; @@ -75,7 +76,21 @@ static bool UncompressZipLump(char *Cache, FileReader &Reader, int Method, ptrdi break; } - // Fixme: These should also use a stream + case METHOD_IMPLODE_0: + case METHOD_IMPLODE_2: + case METHOD_IMPLODE_4: + case METHOD_IMPLODE_6: + { + FZipExploder exploder; + if (exploder.Explode((unsigned char*)Cache, (unsigned)LumpSize, Reader, (unsigned)CompressedSize, Method - METHOD_IMPLODE_MIN) == -1) + { + // decompression failed so zero the cache. + memset(Cache, 0, LumpSize); + } + break; + } + + // This can go away once we are done with FResourceLump case METHOD_IMPLODE: { FZipExploder exploder; @@ -311,6 +326,8 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf) dirptr = (char*)directory; lump_p = Lumps; + AllocateEntries(NumLumps); + auto Entry = Entries; for (uint32_t i = 0; i < NumLumps; i++) { FZipCentralDirectoryInfo *zip_fh = (FZipCentralDirectoryInfo *)dirptr; @@ -399,6 +416,26 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf) } } + Entry->FileName = NormalizeFileName(name.c_str()); + Entry->Length = UncompressedSize; + // The start of the Reader will be determined the first time it is accessed. + Entry->Flags = RESFF_FULLPATH | RESFF_NEEDFILESTART; + Entry->Method = uint8_t(zip_fh->Method); + if (Entry->Method != METHOD_STORED) Entry->Flags |= RESFF_COMPRESSED; + if (Entry->Method == METHOD_IMPLODE) + { + // merge the flags into the compression method to tag less data around. + if ((zip_fh->Flags & 6) == 2) Entry->Method = METHOD_IMPLODE_2; + else if ((zip_fh->Flags & 6) == 4) Entry->Method = METHOD_IMPLODE_4; + else if ((zip_fh->Flags & 6) == 6) Entry->Method = METHOD_IMPLODE_6; + else Entry->Method = METHOD_IMPLODE_0; + } + Entry->CRC32 = zip_fh->CRC32; + Entry->CompressedSize = CompressedSize; + Entry->Position = LocalHeaderOffset; + + Entry++; + lump_p->LumpNameSetup(name.c_str(), stringpool); lump_p->LumpSize = UncompressedSize; lump_p->Owner = this; @@ -407,6 +444,13 @@ bool FZipFile::Open(LumpFilterInfo* filter, FileSystemMessageFunc Printf) lump_p->NeedFileStart = true; lump_p->Method = uint8_t(zip_fh->Method); if (lump_p->Method != METHOD_STORED) lump_p->Flags |= LUMPF_COMPRESSED; + /*if (lump_p->Method == METHOD_IMPLODE) + { + // merge the flags into the compression method to tag less data around. + if ((zip_fh->Flags & 6) == 2) lump_p->Method = METHOD_IMPLODE_2; + if ((zip_fh->Flags & 6) == 4) lump_p->Method = METHOD_IMPLODE_4; + if ((zip_fh->Flags & 6) == 6) lump_p->Method = METHOD_IMPLODE_6; + }*/ lump_p->GPFlags = zip_fh->Flags; lump_p->CRC32 = zip_fh->CRC32; lump_p->CompressedSize = CompressedSize; diff --git a/src/common/filesystem/source/file_zip.h b/src/common/filesystem/source/file_zip.h index 8313c9eb2..c7d73e6e9 100644 --- a/src/common/filesystem/source/file_zip.h +++ b/src/common/filesystem/source/file_zip.h @@ -28,7 +28,6 @@ private: FCompressedBuffer GetRawData(); }; - //========================================================================== // // Zip file diff --git a/src/common/filesystem/source/filesystem.cpp b/src/common/filesystem/source/filesystem.cpp index 3c4bf93c6..8f61e1ee1 100644 --- a/src/common/filesystem/source/filesystem.cpp +++ b/src/common/filesystem/source/filesystem.cpp @@ -101,6 +101,10 @@ struct FileSystem::LumpRecord void SetFromLump(FResourceFile* file, int fileindex, int filenum, StringPool* sp, const char* name = nullptr) { + if (fileindex == 649 && filenum == 0) + { + int a = 0; + } resfile = file; resindex = fileindex; rfnum = filenum; diff --git a/src/common/filesystem/source/fs_stringpool.cpp b/src/common/filesystem/source/fs_stringpool.cpp index 1e4231139..56e7521c6 100644 --- a/src/common/filesystem/source/fs_stringpool.cpp +++ b/src/common/filesystem/source/fs_stringpool.cpp @@ -104,10 +104,12 @@ StringPool::Block *StringPool::AddBlock(size_t size) return mem; } -void *StringPool::iAlloc(size_t size) +void *StringPool::Alloc(size_t size) { Block *block; + size = ((size)+8) & ~7; + for (block = TopBlock; block != nullptr; block = block->NextBlock) { void *res = block->Alloc(size); @@ -122,7 +124,7 @@ void *StringPool::iAlloc(size_t size) const char* StringPool::Strdup(const char* str) { - char* p = (char*)iAlloc((strlen(str) + 8) & ~7 ); + char* p = (char*)Alloc(strlen(str)); strcpy(p, str); return p; } diff --git a/src/common/filesystem/source/fs_stringpool.h b/src/common/filesystem/source/fs_stringpool.h index 3472cd792..199a3483b 100644 --- a/src/common/filesystem/source/fs_stringpool.h +++ b/src/common/filesystem/source/fs_stringpool.h @@ -12,12 +12,12 @@ private: public: ~StringPool(); const char* Strdup(const char*); + void* Alloc(size_t size); protected: struct Block; Block *AddBlock(size_t size); - void *iAlloc(size_t size); Block *TopBlock; Block *FreeBlocks; diff --git a/src/common/filesystem/source/resourcefile.cpp b/src/common/filesystem/source/resourcefile.cpp index f8be0f15a..ff348c991 100644 --- a/src/common/filesystem/source/resourcefile.cpp +++ b/src/common/filesystem/source/resourcefile.cpp @@ -39,6 +39,8 @@ #include "md5.hpp" #include "fs_stringpool.h" #include "files_internal.h" +#include "unicode.h" +#include "fs_findfile.h" namespace FileSys { @@ -347,6 +349,59 @@ FResourceFile::~FResourceFile() if (!stringpool->shared) delete stringpool; } +//========================================================================== +// +// normalize the visible file name in the system +// to lowercase canonical precomposed Unicode. +// +//========================================================================== + +const char* FResourceFile::NormalizeFileName(const char* fn, int fallbackcp) +{ + if (!fn || !*fn) return ""; + auto norm = tolower_normalize(fn); + if (!norm) + { + if (fallbackcp == 437) + { + std::vector buffer; + ibm437_to_utf8(fn, buffer); + norm = tolower_normalize(buffer.data()); + } + // maybe handle other codepages + else + { + // if the filename is not valid UTF-8, nuke all bytes larger than 0x80 so that we still got something semi-usable + std::string ffn = fn; + for (auto& c : ffn) + { + if (c & 0x80) c = '@'; + } + norm = tolower_normalize(&ffn.front()); + } + } + FixPathSeparator(norm); + auto pooled = stringpool->Strdup(norm); + free(norm); + return pooled; + +} + +//========================================================================== +// +// allocate the Entries array +// this also uses the string pool to reduce maintenance +// +//========================================================================== + +void FResourceFile::AllocateEntries(int count) +{ + NumLumps = count; + Entries = (FResourceEntry*)stringpool->Alloc(count * sizeof(FResourceEntry)); +} + + +//--------------------------------------------------- int lumpcmp(const void * a, const void * b) { FResourceLump * rec1 = (FResourceLump *)a; @@ -354,6 +409,14 @@ int lumpcmp(const void * a, const void * b) return stricmp(rec1->getName(), rec2->getName()); } +int entrycmp(const void* a, const void* b) +{ + FResourceEntry* rec1 = (FResourceEntry*)a; + FResourceEntry* rec2 = (FResourceEntry*)b; + // we are comparing lowercase UTF-8 here + return strcmp(rec1->FileName, rec2->FileName); +} + //========================================================================== // // FResourceFile :: GenerateHash @@ -402,8 +465,12 @@ void FResourceFile::GenerateHash() void FResourceFile::PostProcessArchive(void *lumps, size_t lumpsize, LumpFilterInfo *filter) { - // Entries in archives are sorted alphabetically - qsort(lumps, NumLumps, lumpsize, lumpcmp); + // only do this for archive types which contain full file names. All others are assumed to be pre-sorted. + //if (NumLumps < 2 || !(Entries[0].Flags & RESFF_FULLPATH)) return; + + // Entries in archives are sorted alphabetically (for now skip the sorting because both arrays might disagree due to file name postprocessing.) + //qsort(lumps, NumLumps, lumpsize, lumpcmp); + //qsort(Entries, NumLumps, sizeof(Entries[0]), entrycmp); if (!filter) return; // Filter out lumps using the same names as the Autoload.* sections @@ -460,8 +527,10 @@ int FResourceFile::FilterLumps(const std::string& filtername, void *lumps, size_ FResourceLump *lump = (FResourceLump *)lump_p; assert(strnicmp(lump->FullName, filter.c_str(), filter.length()) == 0); lump->LumpNameSetup(lump->FullName + filter.length(), nullptr); + Entries[i].FileName += filter.length(); } +#if 0 // disabled until everything is clean again // Move filtered lumps to the end of the lump list. size_t count = (end - start) * lumpsize; void *to = (uint8_t *)lumps + NumLumps * lumpsize - count; @@ -481,6 +550,7 @@ int FResourceFile::FilterLumps(const std::string& filtername, void *lumps, size_ delete[] filteredlumps; } +#endif } return end - start; }