From 39020f7f951f5f982d3bd651f6901096c4a238e0 Mon Sep 17 00:00:00 2001 From: Christoph Oelckers Date: Tue, 12 Dec 2023 20:17:59 +0100 Subject: [PATCH] preparations for getting rid of FZipLump * allow ancient compression algorithms to be handled by OpenDecompressor. * move FCompressedBuffer to fs_files.h * use a mutex lock for 7z access because it cannot be made thread save otherwise. --- src/common/filesystem/include/fs_files.h | 23 +++ src/common/filesystem/include/resourcefile.h | 22 --- src/common/filesystem/source/critsec.cpp | 150 ++++++++++++++++++ src/common/filesystem/source/critsec.h | 39 +++++ src/common/filesystem/source/file_7z.cpp | 5 +- src/common/filesystem/source/file_zip.cpp | 44 +---- .../filesystem/source/files_decompress.cpp | 91 +++++++++-- src/common/filesystem/source/files_internal.h | 2 +- .../filesystem/source/fs_stringpool.cpp | 5 +- 9 files changed, 299 insertions(+), 82 deletions(-) create mode 100644 src/common/filesystem/source/critsec.cpp create mode 100644 src/common/filesystem/source/critsec.h diff --git a/src/common/filesystem/include/fs_files.h b/src/common/filesystem/include/fs_files.h index 34536200d..5bc2d7702 100644 --- a/src/common/filesystem/include/fs_files.h +++ b/src/common/filesystem/include/fs_files.h @@ -175,6 +175,29 @@ public: }; +// This holds a compresed Zip entry with all needed info to decompress it. +struct FCompressedBuffer +{ + size_t mSize; + size_t mCompressedSize; + int mMethod; + unsigned mCRC32; + char* mBuffer; + const char* filename; + + bool Decompress(char* destbuffer); + void Clean() + { + mSize = mCompressedSize = 0; + if (mBuffer != nullptr) + { + delete[] mBuffer; + mBuffer = nullptr; + } + } +}; + + class FileReaderInterface { public: diff --git a/src/common/filesystem/include/resourcefile.h b/src/common/filesystem/include/resourcefile.h index d8aa1c776..485f88974 100644 --- a/src/common/filesystem/include/resourcefile.h +++ b/src/common/filesystem/include/resourcefile.h @@ -92,28 +92,6 @@ enum ELumpFlags }; -// This holds a compresed Zip entry with all needed info to decompress it. -struct FCompressedBuffer -{ - size_t mSize; - size_t mCompressedSize; - int mMethod; - unsigned mCRC32; - char *mBuffer; - const char* filename; - - bool Decompress(char *destbuffer); - void Clean() - { - mSize = mCompressedSize = 0; - if (mBuffer != nullptr) - { - delete[] mBuffer; - mBuffer = nullptr; - } - } -}; - struct FResourceLump { protected: diff --git a/src/common/filesystem/source/critsec.cpp b/src/common/filesystem/source/critsec.cpp new file mode 100644 index 000000000..874ade79f --- /dev/null +++ b/src/common/filesystem/source/critsec.cpp @@ -0,0 +1,150 @@ +/* +** +** +**--------------------------------------------------------------------------- +** Copyright 2005-2016 Randy Heit +** All rights reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions +** are met: +** +** 1. Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** 2. Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** 3. The name of the author may not be used to endorse or promote products +** derived from this software without specific prior written permission. +** +** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +** IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +** OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +** IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +** INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +** NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +** THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**--------------------------------------------------------------------------- +** +*/ + +namespace FileSys { + +#ifdef _WIN32 + +#ifndef _WINNT_ +#define WIN32_LEAN_AND_MEAN +#include +#endif + +class FInternalCriticalSection +{ +public: + void Enter() + { + AcquireSRWLockExclusive(&CritSec); + } + void Leave() + { + ReleaseSRWLockExclusive(&CritSec); + } +private: + SRWLOCK CritSec = SRWLOCK_INIT; +}; + + +FInternalCriticalSection *CreateCriticalSection() +{ + return new FInternalCriticalSection(); +} + +void DeleteCriticalSection(FInternalCriticalSection *c) +{ + delete c; +} + +void EnterCriticalSection(FInternalCriticalSection *c) +{ + c->Enter(); +} + +void LeaveCriticalSection(FInternalCriticalSection *c) +{ + c->Leave(); +} + +#else + +#include "critsec.h" + +#include + +class FInternalCriticalSection +{ +public: + FInternalCriticalSection(); + ~FInternalCriticalSection(); + + void Enter(); + void Leave(); + +private: + pthread_mutex_t m_mutex; + +}; + +// TODO: add error handling + +FInternalCriticalSection::FInternalCriticalSection() +{ + pthread_mutexattr_t attributes; + pthread_mutexattr_init(&attributes); + pthread_mutexattr_settype(&attributes, PTHREAD_MUTEX_RECURSIVE); + + pthread_mutex_init(&m_mutex, &attributes); + + pthread_mutexattr_destroy(&attributes); +} + +FInternalCriticalSection::~FInternalCriticalSection() +{ + pthread_mutex_destroy(&m_mutex); +} + +void FInternalCriticalSection::Enter() +{ + pthread_mutex_lock(&m_mutex); +} + +void FInternalCriticalSection::Leave() +{ + pthread_mutex_unlock(&m_mutex); +} + + +FInternalCriticalSection *CreateCriticalSection() +{ + return new FInternalCriticalSection(); +} + +void DeleteCriticalSection(FInternalCriticalSection *c) +{ + delete c; +} + +void EnterCriticalSection(FInternalCriticalSection *c) +{ + c->Enter(); +} + +void LeaveCriticalSection(FInternalCriticalSection *c) +{ + c->Leave(); +} + +#endif + +} \ No newline at end of file diff --git a/src/common/filesystem/source/critsec.h b/src/common/filesystem/source/critsec.h new file mode 100644 index 000000000..9ea56aab5 --- /dev/null +++ b/src/common/filesystem/source/critsec.h @@ -0,0 +1,39 @@ +#pragma once + +namespace FileSys { +// System independent critical sections without polluting the namespace with the operating system headers. +class FInternalCriticalSection; +FInternalCriticalSection *CreateCriticalSection(); +void DeleteCriticalSection(FInternalCriticalSection *c); +void EnterCriticalSection(FInternalCriticalSection *c); +void LeaveCriticalSection(FInternalCriticalSection *c); + +// This is just a convenience wrapper around the function interface adjusted to use std::lock_guard +class FCriticalSection +{ +public: + FCriticalSection() + { + c = CreateCriticalSection(); + } + + ~FCriticalSection() + { + DeleteCriticalSection(c); + } + + void lock() + { + EnterCriticalSection(c); + } + + void unlock() + { + LeaveCriticalSection(c); + } + +private: + FInternalCriticalSection *c; + +}; +} \ No newline at end of file diff --git a/src/common/filesystem/source/file_7z.cpp b/src/common/filesystem/source/file_7z.cpp index ae73af590..5f4ae722f 100644 --- a/src/common/filesystem/source/file_7z.cpp +++ b/src/common/filesystem/source/file_7z.cpp @@ -39,6 +39,8 @@ #include "resourcefile.h" #include "fs_findfile.h" #include "unicode.h" +#include "critsec.h" +#include namespace FileSys { @@ -171,6 +173,7 @@ class F7ZFile : public FResourceFile friend struct F7ZLump; C7zArchive *Archive; + FCriticalSection critsec; public: F7ZFile(const char * filename, FileReader &filer, StringPool* sp); @@ -318,7 +321,7 @@ FileData F7ZFile::Read(int entry) { auto p = buffer.allocate(Entries[entry].Length); // There is no realistic way to keep multiple references to a 7z file open without massive overhead so to make this thread-safe a mutex is the only option. - //std::lock_guard lock(critsec); // activate later + std::lock_guard lock(critsec); SRes code = Archive->Extract(Entries[entry].Position, (char*)p); if (code != SZ_OK) buffer.clear(); } diff --git a/src/common/filesystem/source/file_zip.cpp b/src/common/filesystem/source/file_zip.cpp index fc1ce42c2..d1e4f8d83 100644 --- a/src/common/filesystem/source/file_zip.cpp +++ b/src/common/filesystem/source/file_zip.cpp @@ -67,6 +67,11 @@ static bool UncompressZipLump(char *Cache, FileReader &Reader, int Method, ptrdi case METHOD_BZIP2: case METHOD_LZMA: case METHOD_XZ: + case METHOD_IMPLODE_0: + case METHOD_IMPLODE_2: + case METHOD_IMPLODE_4: + case METHOD_IMPLODE_6: + case METHOD_SHRINK: { FileReader frz; if (frz.OpenDecompressor(Reader, LumpSize, Method, false, exceptions)) @@ -76,38 +81,6 @@ static bool UncompressZipLump(char *Cache, FileReader &Reader, int Method, ptrdi break; } - case METHOD_IMPLODE_0: - case METHOD_IMPLODE_2: - case METHOD_IMPLODE_4: - case METHOD_IMPLODE_6: - { - FZipExploder exploder; - if (exploder.Explode((unsigned char*)Cache, (unsigned)LumpSize, Reader, (unsigned)CompressedSize, Method - METHOD_IMPLODE_MIN) == -1) - { - // decompression failed so zero the cache. - memset(Cache, 0, LumpSize); - } - break; - } - - // This can go away once we are done with FResourceLump - case METHOD_IMPLODE: - { - FZipExploder exploder; - if (exploder.Explode((unsigned char*)Cache, (unsigned)LumpSize, Reader, (unsigned)CompressedSize, GPFlags) == -1) - { - // decompression failed so zero the cache. - memset(Cache, 0, LumpSize); - } - break; - } - - case METHOD_SHRINK: - { - ShrinkLoop((unsigned char *)Cache, (unsigned)LumpSize, Reader, (unsigned)CompressedSize); - break; - } - default: assert(0); return false; @@ -115,13 +88,6 @@ static bool UncompressZipLump(char *Cache, FileReader &Reader, int Method, ptrdi return true; } -bool FCompressedBuffer::Decompress(char *destbuffer) -{ - FileReader mr; - mr.OpenMemory(mBuffer, mCompressedSize); - return UncompressZipLump(destbuffer, mr, mMethod, mSize, mCompressedSize, 0, false); -} - //----------------------------------------------------------------------- // // Finds the central directory end record in the end of the file. diff --git a/src/common/filesystem/source/files_decompress.cpp b/src/common/filesystem/source/files_decompress.cpp index c5552018f..7af2788bc 100644 --- a/src/common/filesystem/source/files_decompress.cpp +++ b/src/common/filesystem/source/files_decompress.cpp @@ -45,6 +45,8 @@ #include #include "fs_files.h" +#include "files_internal.h" +#include "ancientzip.h" namespace FileSys { using namespace byteswap; @@ -53,6 +55,7 @@ namespace FileSys { class DecompressorBase : public FileReaderInterface { bool exceptions = false; + public: // These do not work but need to be defined to satisfy the FileReaderInterface. // They will just error out when called. @@ -64,6 +67,10 @@ public: void EnableExceptions(bool on) { exceptions = on; } protected: + DecompressorBase() + { + //seekable = false; + } FileReader* File = nullptr; FileReader OwnedFile; }; @@ -843,6 +850,7 @@ public: bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, bool seekable, bool exceptions) { + FileReaderInterface* fr = nullptr; DecompressorBase* dec = nullptr; try { @@ -853,7 +861,7 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b case METHOD_ZLIB: { auto idec = new DecompressorZ; - dec = idec; + fr = dec = idec; idec->EnableExceptions(exceptions); if (!idec->Open(p, method == METHOD_DEFLATE)) { @@ -865,7 +873,7 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b case METHOD_BZIP2: { auto idec = new DecompressorBZ2; - dec = idec; + fr = dec = idec; idec->EnableExceptions(exceptions); if (!idec->Open(p)) { @@ -877,7 +885,7 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b case METHOD_LZMA: { auto idec = new DecompressorLZMA; - dec = idec; + fr = dec = idec; idec->EnableExceptions(exceptions); if (!idec->Open(p, length)) { @@ -889,7 +897,7 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b case METHOD_XZ: { auto idec = new DecompressorXZ; - dec = idec; + fr = dec = idec; idec->EnableExceptions(exceptions); if (!idec->Open(p, length)) { @@ -901,7 +909,7 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b case METHOD_LZSS: { auto idec = new DecompressorLZSS; - dec = idec; + fr = dec = idec; idec->EnableExceptions(exceptions); if (!idec->Open(p)) { @@ -911,34 +919,85 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b break; } - // todo: METHOD_IMPLODE, METHOD_SHRINK + // The decoders for these legacy formats can only handle the full data in one go so we have to perform the entire decompression here. + case METHOD_IMPLODE: + case METHOD_IMPLODE_2: + case METHOD_IMPLODE_4: + case METHOD_IMPLODE_6: + { + auto idec = new MemoryArrayReader; + fr = idec; + auto& buffer = idec->GetArray(); + auto bufr = (uint8_t*)buffer.allocate(length); + FZipExploder exploder; + if (exploder.Explode(bufr, length, *p, p->GetLength(), method) == -1) + { + if (exceptions) + { + throw FileSystemException("DecompressImplode failed"); + } + delete idec; + return false; + } + break; + } + + case METHOD_SHRINK: + { + auto idec = new MemoryArrayReader; + fr = idec; + auto& buffer = idec->GetArray(); + auto bufr = (uint8_t*)buffer.allocate(length); + ShrinkLoop(bufr, length, *p, p->GetLength()); // this never fails. + break; + } + default: return false; } - if (method & METHOD_TRANSFEROWNER) + if (dec) { - dec->SetOwnsReader(); + if (method & METHOD_TRANSFEROWNER) + { + dec->SetOwnsReader(); + } + dec->Length = length; } - - dec->Length = length; if (!seekable) { Close(); - mReader = dec; - return true; + mReader = fr; } else { - // todo: create a wrapper. for now this fails - delete dec; - return false; + // create a wrapper that can buffer the content so that seeking is possible + mReader = new BufferingReader(fr); } + return true; } catch (...) { - if (dec) delete dec; + if (fr) delete fr; throw; } } + +bool FCompressedBuffer::Decompress(char* destbuffer) +{ + FileReader mr; + mr.OpenMemory(mBuffer, mCompressedSize); + if (mMethod == METHOD_STORED) + { + return mr.Read(destbuffer, mSize) != mSize; + } + else + { + FileReader frz; + if (frz.OpenDecompressor(mr, mSize, mMethod, false, false)) + { + return frz.Read(destbuffer, mSize) != mSize; + } + } + return false; } diff --git a/src/common/filesystem/source/files_internal.h b/src/common/filesystem/source/files_internal.h index aa3595cbf..a59edd135 100644 --- a/src/common/filesystem/source/files_internal.h +++ b/src/common/filesystem/source/files_internal.h @@ -83,7 +83,7 @@ public: UpdateBuffer(); } - std::vector& GetArray() { return buf; } + T& GetArray() { return buf; } void UpdateBuffer() { diff --git a/src/common/filesystem/source/fs_stringpool.cpp b/src/common/filesystem/source/fs_stringpool.cpp index 56e7521c6..316078cae 100644 --- a/src/common/filesystem/source/fs_stringpool.cpp +++ b/src/common/filesystem/source/fs_stringpool.cpp @@ -108,8 +108,7 @@ void *StringPool::Alloc(size_t size) { Block *block; - size = ((size)+8) & ~7; - + size = (size + 7) & ~7; for (block = TopBlock; block != nullptr; block = block->NextBlock) { void *res = block->Alloc(size); @@ -124,7 +123,7 @@ void *StringPool::Alloc(size_t size) const char* StringPool::Strdup(const char* str) { - char* p = (char*)Alloc(strlen(str)); + char* p = (char*)Alloc(strlen(str) + 1); strcpy(p, str); return p; }