From 4fd5f00c4b21d4b15e13ab6731f02c50da1e6c45 Mon Sep 17 00:00:00 2001 From: Christoph Oelckers Date: Thu, 14 Sep 2023 21:45:38 +0200 Subject: [PATCH] - made all FileReader implementations 64 bit capable. --- src/common/filesystem/include/fs_files.h | 26 ++--- src/common/filesystem/source/files.cpp | 94 ++++++++++--------- .../filesystem/source/files_decompress.cpp | 76 +++++++++------ src/common/filesystem/source/files_internal.h | 12 +-- src/common/utility/filereadermusicinterface.h | 6 +- 5 files changed, 117 insertions(+), 97 deletions(-) diff --git a/src/common/filesystem/include/fs_files.h b/src/common/filesystem/include/fs_files.h index 878122e82..06160d258 100644 --- a/src/common/filesystem/include/fs_files.h +++ b/src/common/filesystem/include/fs_files.h @@ -92,14 +92,14 @@ class FileReader; class FileReaderInterface { public: - long Length = -1; + ptrdiff_t Length = -1; virtual ~FileReaderInterface() {} - virtual long Tell () const = 0; - virtual long Seek (long offset, int origin) = 0; - virtual long Read (void *buffer, long len) = 0; - virtual char *Gets(char *strbuf, int len) = 0; + virtual ptrdiff_t Tell () const = 0; + virtual ptrdiff_t Seek (ptrdiff_t offset, int origin) = 0; + virtual ptrdiff_t Read (void *buffer, ptrdiff_t len) = 0; + virtual char *Gets(char *strbuf, ptrdiff_t len) = 0; virtual const char *GetBuffer() const { return nullptr; } - long GetLength () const { return Length; } + ptrdiff_t GetLength () const { return Length; } }; struct FResourceLump; @@ -184,12 +184,12 @@ public: Size Seek(Size offset, ESeek origin) { - return mReader->Seek((long)offset, origin); + return mReader->Seek(offset, origin); } Size Read(void *buffer, Size len) const { - return mReader->Read(buffer, (long)len); + return mReader->Read(buffer, len); } std::vector Read(size_t len) @@ -197,7 +197,7 @@ public: std::vector buffer(len); if (len > 0) { - Size length = mReader->Read(&buffer[0], (long)len); + Size length = mReader->Read(&buffer[0], len); buffer.resize((size_t)length); } return buffer; @@ -214,7 +214,7 @@ public: std::vector buffer(len + padding); if (len > 0) { - Size length = mReader->Read(&buffer[0], (long)len); + Size length = mReader->Read(&buffer[0], len); if (length < len) buffer.clear(); else memset(buffer.data() + len, 0, padding); } @@ -224,7 +224,7 @@ public: char *Gets(char *strbuf, Size len) { - return mReader->Gets(strbuf, (int)len); + return mReader->Gets(strbuf, len); } const char *GetBuffer() @@ -331,8 +331,8 @@ public: static FileWriter *Open(const char *filename); virtual size_t Write(const void *buffer, size_t len); - virtual long Tell(); - virtual long Seek(long offset, int mode); + virtual ptrdiff_t Tell(); + virtual ptrdiff_t Seek(ptrdiff_t offset, int mode); size_t Printf(const char *fmt, ...); virtual void Close() diff --git a/src/common/filesystem/source/files.cpp b/src/common/filesystem/source/files.cpp index 8a0b5ed27..a17f7c1b8 100644 --- a/src/common/filesystem/source/files.cpp +++ b/src/common/filesystem/source/files.cpp @@ -53,6 +53,10 @@ FILE *myfopen(const char *filename, const char *flags) #endif } +#ifdef _WIN32 +#define fseek _fseeki64 +#define ftell _ftelli64 +#endif //========================================================================== // @@ -65,8 +69,8 @@ FILE *myfopen(const char *filename, const char *flags) class StdFileReader : public FileReaderInterface { FILE *File = nullptr; - long StartPos = 0; - long FilePos = 0; + ptrdiff_t StartPos = 0; + ptrdiff_t FilePos = 0; public: StdFileReader() @@ -81,7 +85,7 @@ public: File = nullptr; } - bool Open(const char *filename, long startpos = 0, long len = -1) + bool Open(const char *filename, ptrdiff_t startpos = 0, ptrdiff_t len = -1) { File = myfopen(filename, "rb"); if (File == nullptr) return false; @@ -93,12 +97,12 @@ public: return true; } - long Tell() const override + ptrdiff_t Tell() const override { return FilePos - StartPos; } - long Seek(long offset, int origin) override + ptrdiff_t Seek(ptrdiff_t offset, int origin) override { if (origin == SEEK_SET) { @@ -122,7 +126,7 @@ public: return -1; } - long Read(void *buffer, long len) override + ptrdiff_t Read(void *buffer, ptrdiff_t len) override { assert(len >= 0); if (len <= 0) return 0; @@ -130,16 +134,16 @@ public: { len = Length - FilePos + StartPos; } - len = (long)fread(buffer, 1, len, File); + len = fread(buffer, 1, len, File); FilePos += len; return len; } - char *Gets(char *strbuf, int len) override + char *Gets(char *strbuf, ptrdiff_t len) override { - if (len <= 0 || FilePos >= StartPos + Length) return NULL; + if (len <= 0 || len > 0x7fffffff || FilePos >= StartPos + Length) return nullptr; char *p = fgets(strbuf, len, File); - if (p != NULL) + if (p != nullptr) { int old = FilePos; FilePos = ftell(File); @@ -152,9 +156,9 @@ public: } private: - long CalcFileLen() const + ptrdiff_t CalcFileLen() const { - long endpos; + ptrdiff_t endpos; fseek(File, 0, SEEK_END); endpos = ftell(File); @@ -174,11 +178,11 @@ private: class FileReaderRedirect : public FileReaderInterface { FileReader *mReader = nullptr; - long StartPos = 0; - long FilePos = 0; + ptrdiff_t StartPos = 0; + ptrdiff_t FilePos = 0; public: - FileReaderRedirect(FileReader &parent, long start, long length) + FileReaderRedirect(FileReader &parent, ptrdiff_t start, ptrdiff_t length) { mReader = &parent; FilePos = start; @@ -187,12 +191,12 @@ public: Seek(0, SEEK_SET); } - virtual long Tell() const override + virtual ptrdiff_t Tell() const override { return FilePos - StartPos; } - virtual long Seek(long offset, int origin) override + virtual ptrdiff_t Seek(ptrdiff_t offset, int origin) override { switch (origin) { @@ -205,7 +209,7 @@ public: break; case SEEK_CUR: - offset += (long)mReader->Tell(); + offset += mReader->Tell(); break; } if (offset < StartPos || offset > StartPos + Length) return -1; // out of scope @@ -217,7 +221,7 @@ public: return -1; } - virtual long Read(void *buffer, long len) override + virtual ptrdiff_t Read(void *buffer, ptrdiff_t len) override { assert(len >= 0); if (len <= 0) return 0; @@ -225,19 +229,19 @@ public: { len = Length - FilePos + StartPos; } - len = (long)mReader->Read(buffer, len); + len = mReader->Read(buffer, len); FilePos += len; return len; } - virtual char *Gets(char *strbuf, int len) override + virtual char *Gets(char *strbuf, ptrdiff_t len) override { - if (len <= 0 || FilePos >= StartPos + Length) return NULL; + if (len <= 0 || FilePos >= StartPos + Length) return nullptr; char *p = mReader->Gets(strbuf, len); - if (p != NULL) + if (p != nullptr) { int old = FilePos; - FilePos = (long)mReader->Tell(); + FilePos = mReader->Tell(); if (FilePos - StartPos > Length) { strbuf[Length - old + StartPos] = 0; @@ -256,12 +260,12 @@ public: // //========================================================================== -long MemoryReader::Tell() const +ptrdiff_t MemoryReader::Tell() const { return FilePos; } -long MemoryReader::Seek(long offset, int origin) +ptrdiff_t MemoryReader::Seek(ptrdiff_t offset, int origin) { switch (origin) { @@ -275,11 +279,11 @@ long MemoryReader::Seek(long offset, int origin) } if (offset < 0 || offset > Length) return -1; - FilePos = std::clamp(offset, 0, Length); + FilePos = std::clamp(offset, 0, Length); return 0; } -long MemoryReader::Read(void *buffer, long len) +ptrdiff_t MemoryReader::Read(void *buffer, ptrdiff_t len) { if (len>Length - FilePos) len = Length - FilePos; if (len<0) len = 0; @@ -288,10 +292,10 @@ long MemoryReader::Read(void *buffer, long len) return len; } -char *MemoryReader::Gets(char *strbuf, int len) +char *MemoryReader::Gets(char *strbuf, ptrdiff_t len) { if (len>Length - FilePos) len = Length - FilePos; - if (len <= 0) return NULL; + if (len <= 0) return nullptr; char *p = strbuf; while (len > 1) @@ -313,7 +317,7 @@ char *MemoryReader::Gets(char *strbuf, int len) } FilePos++; } - if (p == strbuf) return NULL; + if (p == strbuf) return nullptr; *p++ = 0; return strbuf; } @@ -331,7 +335,7 @@ class MemoryArrayReader : public MemoryReader std::vector buf; public: - MemoryArrayReader(const char *buffer, long length) + MemoryArrayReader(const char *buffer, ptrdiff_t length) { if (length > 0) { @@ -347,7 +351,7 @@ public: { bufptr = (const char*)buf.data(); FilePos = 0; - Length = (long)buf.size(); + Length = buf.size(); } }; @@ -364,7 +368,7 @@ public: bool FileReader::OpenFile(const char *filename, FileReader::Size start, FileReader::Size length) { auto reader = new StdFileReader; - if (!reader->Open(filename, (long)start, (long)length)) + if (!reader->Open(filename, start, length)) { delete reader; return false; @@ -376,7 +380,7 @@ bool FileReader::OpenFile(const char *filename, FileReader::Size start, FileRead bool FileReader::OpenFilePart(FileReader &parent, FileReader::Size start, FileReader::Size length) { - auto reader = new FileReaderRedirect(parent, (long)start, (long)length); + auto reader = new FileReaderRedirect(parent, start, length); Close(); mReader = reader; return true; @@ -385,14 +389,14 @@ bool FileReader::OpenFilePart(FileReader &parent, FileReader::Size start, FileRe bool FileReader::OpenMemory(const void *mem, FileReader::Size length) { Close(); - mReader = new MemoryReader((const char *)mem, (long)length); + mReader = new MemoryReader((const char *)mem, length); return true; } bool FileReader::OpenMemoryArray(const void *mem, FileReader::Size length) { Close(); - mReader = new MemoryArrayReader((const char *)mem, (long)length); + mReader = new MemoryArrayReader((const char *)mem, length); return true; } @@ -424,7 +428,7 @@ bool FileReader::OpenMemoryArray(std::function&)> gett bool FileWriter::OpenDirect(const char *filename) { File = myfopen(filename, "wb"); - return (File != NULL); + return (File != nullptr); } FileWriter *FileWriter::Open(const char *filename) @@ -435,12 +439,12 @@ FileWriter *FileWriter::Open(const char *filename) return fwrit; } delete fwrit; - return NULL; + return nullptr; } size_t FileWriter::Write(const void *buffer, size_t len) { - if (File != NULL) + if (File != nullptr) { return fwrite(buffer, 1, len, File); } @@ -450,9 +454,9 @@ size_t FileWriter::Write(const void *buffer, size_t len) } } -long FileWriter::Tell() +ptrdiff_t FileWriter::Tell() { - if (File != NULL) + if (File != nullptr) { return ftell(File); } @@ -462,9 +466,9 @@ long FileWriter::Tell() } } -long FileWriter::Seek(long offset, int mode) +ptrdiff_t FileWriter::Seek(ptrdiff_t offset, int mode) { - if (File != NULL) + if (File != nullptr) { return fseek(File, offset, mode); } @@ -485,7 +489,7 @@ size_t FileWriter::Printf(const char *fmt, ...) va_start(arglist, fmt); vsnprintf(&buf.front(), n + 1, fmt, arglist); va_end(arglist); - return Write(buf.c_str(), strlen(buf.c_str())); // Make sure we write no null bytes. + return Write(buf.c_str(), strlen(buf.c_str())); // Make sure we write no nullptr bytes. } size_t BufferWriter::Write(const void *buffer, size_t len) diff --git a/src/common/filesystem/source/files_decompress.cpp b/src/common/filesystem/source/files_decompress.cpp index 86529716a..0be5abdaa 100644 --- a/src/common/filesystem/source/files_decompress.cpp +++ b/src/common/filesystem/source/files_decompress.cpp @@ -53,9 +53,9 @@ class DecompressorBase : public FileReaderInterface public: // These do not work but need to be defined to satisfy the FileReaderInterface. // They will just error out when called. - long Tell() const override; - long Seek(long offset, int origin) override; - char* Gets(char* strbuf, int len) override; + ptrdiff_t Tell() const override; + ptrdiff_t Seek(ptrdiff_t offset, int origin) override; + char* Gets(char* strbuf, ptrdiff_t len) override; void DecompressionError(const char* error, ...) const; void SetOwnsReader(); void EnableExceptions(bool on) { exceptions = on; } @@ -86,17 +86,17 @@ void DecompressorBase::DecompressionError(const char *error, ...) const } } -long DecompressorBase::Tell () const +ptrdiff_t DecompressorBase::Tell () const { DecompressionError("Cannot get position of decompressor stream"); return 0; } -long DecompressorBase::Seek (long offset, int origin) +ptrdiff_t DecompressorBase::Seek (ptrdiff_t offset, int origin) { DecompressionError("Cannot seek in decompressor stream"); return 0; } -char *DecompressorBase::Gets(char *strbuf, int len) +char *DecompressorBase::Gets(char *strbuf, ptrdiff_t len) { DecompressionError("Cannot use Gets on decompressor stream"); return nullptr; @@ -171,7 +171,7 @@ public: inflateEnd (&Stream); } - long Read (void *buffer, long len) override + ptrdiff_t Read (void *buffer, ptrdiff_t len) override { int err; @@ -180,18 +180,25 @@ public: DecompressionError("File not open"); return 0; } + if (len == 0) return 0; - Stream.next_out = (Bytef *)buffer; - Stream.avail_out = len; - - do + while (len > 0) { - err = inflate (&Stream, Z_SYNC_FLUSH); - if (Stream.avail_in == 0 && !SawEOF) + Stream.next_out = (Bytef*)buffer; + auto rlen = std::min(len, 0x40000000); + Stream.avail_out = rlen; + buffer = Stream.next_out + rlen; + len -= rlen; + + do { - FillBuffer (); - } - } while (err == Z_OK && Stream.avail_out != 0); + err = inflate(&Stream, Z_SYNC_FLUSH); + if (Stream.avail_in == 0 && !SawEOF) + { + FillBuffer(); + } + } while (err == Z_OK && Stream.avail_out != 0); + } if (err != Z_OK && err != Z_STREAM_END) { @@ -278,28 +285,36 @@ public: BZ2_bzDecompressEnd (&Stream); } - long Read (void *buffer, long len) override + ptrdiff_t Read (void *buffer, ptrdiff_t len) override { if (File == nullptr) { DecompressionError("File not open"); return 0; } + if (len == 0) return 0; - int err; + int err = BZ_OK; stupidGlobal = this; - Stream.next_out = (char *)buffer; - Stream.avail_out = len; - do + while (len > 0) { - err = BZ2_bzDecompress(&Stream); - if (Stream.avail_in == 0 && !SawEOF) + Stream.next_out = (char*)buffer; + auto rlen = std::min(len, 0x40000000); + Stream.avail_out = rlen; + buffer = Stream.next_out + rlen; + len -= rlen; + + do { - FillBuffer (); - } - } while (err == BZ_OK && Stream.avail_out != 0); + err = BZ2_bzDecompress(&Stream); + if (Stream.avail_in == 0 && !SawEOF) + { + FillBuffer(); + } + } while (err == BZ_OK && Stream.avail_out != 0); + } if (err != BZ_OK && err != BZ_STREAM_END) { @@ -419,7 +434,7 @@ public: LzmaDec_Free(&Stream, &g_Alloc); } - long Read (void *buffer, long len) override + ptrdiff_t Read (void *buffer, ptrdiff_t len) override { if (File == nullptr) { @@ -639,11 +654,12 @@ public: { } - long Read(void *buffer, long len) override + ptrdiff_t Read(void *buffer, ptrdiff_t len) override { + if (len > 0xffffffff) len = 0xffffffff; // this format cannot be larger than 4GB. uint8_t *Out = (uint8_t*)buffer; - long AvailOut = len; + unsigned AvailOut = len; do { @@ -754,7 +770,7 @@ bool FileReader::OpenDecompressor(FileReader &parent, Size length, int method, b dec->SetOwnsReader(); } - dec->Length = (long)length; + dec->Length = length; if (!seekable) { Close(); diff --git a/src/common/filesystem/source/files_internal.h b/src/common/filesystem/source/files_internal.h index 34e73a070..3317796ac 100644 --- a/src/common/filesystem/source/files_internal.h +++ b/src/common/filesystem/source/files_internal.h @@ -8,23 +8,23 @@ class MemoryReader : public FileReaderInterface { protected: const char * bufptr = nullptr; - long FilePos = 0; + ptrdiff_t FilePos = 0; MemoryReader() {} public: - MemoryReader(const char *buffer, long length) + MemoryReader(const char *buffer, ptrdiff_t length) { bufptr = buffer; Length = length; FilePos = 0; } - long Tell() const override; - long Seek(long offset, int origin) override; - long Read(void *buffer, long len) override; - char *Gets(char *strbuf, int len) override; + ptrdiff_t Tell() const override; + ptrdiff_t Seek(ptrdiff_t offset, int origin) override; + ptrdiff_t Read(void *buffer, ptrdiff_t len) override; + char *Gets(char *strbuf, ptrdiff_t len) override; virtual const char *GetBuffer() const override { return bufptr; } }; diff --git a/src/common/utility/filereadermusicinterface.h b/src/common/utility/filereadermusicinterface.h index 633599b63..042804b53 100644 --- a/src/common/utility/filereadermusicinterface.h +++ b/src/common/utility/filereadermusicinterface.h @@ -11,9 +11,9 @@ inline ZMusicCustomReader *GetMusicReader(FileReader& fr) zcr->handle = fr.GetInterface(); zcr->gets = [](ZMusicCustomReader* zr, char* buff, int n) { return reinterpret_cast(zr->handle)->Gets(buff, n); }; - zcr->read = [](ZMusicCustomReader* zr, void* buff, int32_t size) { return reinterpret_cast(zr->handle)->Read(buff, (long)size); }; - zcr->seek = [](ZMusicCustomReader* zr, long offset, int whence) { return reinterpret_cast(zr->handle)->Seek(offset, whence); }; - zcr->tell = [](ZMusicCustomReader* zr) { return reinterpret_cast(zr->handle)->Tell(); }; + zcr->read = [](ZMusicCustomReader* zr, void* buff, int32_t size) -> long { return reinterpret_cast(zr->handle)->Read(buff, size); }; + zcr->seek = [](ZMusicCustomReader* zr, long offset, int whence) -> long { return reinterpret_cast(zr->handle)->Seek(offset, whence); }; + zcr->tell = [](ZMusicCustomReader* zr) -> long { return reinterpret_cast(zr->handle)->Tell(); }; zcr->close = [](ZMusicCustomReader* zr) { delete reinterpret_cast(zr->handle);