- transition to new FileReader interface complete, but untested.

This commit is contained in:
Christoph Oelckers 2018-03-11 12:33:46 +01:00
commit b14ee50d0d
31 changed files with 1253 additions and 1708 deletions

View file

@ -33,150 +33,252 @@
**
*/
// This also pulls in windows.h
#include "LzmaDec.h"
#include "files.h"
#include "i_system.h"
#include "templates.h"
#include "m_misc.h"
//==========================================================================
//
// FileReader
// StdFileReader
//
// reads data from an uncompressed file or part of it
// reads data from an stdio FILE* or part of it.
//
//==========================================================================
FILE *FileReader::openfd(const char *filename)
class StdFileReader : public FileReaderInterface
{
return fopen(filename, "rb");
}
FILE *File = nullptr;
long Length = 0;
long StartPos = 0;
long FilePos = 0;
FileReader::FileReader ()
: File(NULL), Length(0), StartPos(0), FilePos(0), CloseOnDestruct(false)
{
}
public:
StdFileReader()
{}
FileReader::FileReader (const FileReader &other, long length)
: File(other.File), Length(length), CloseOnDestruct(false)
{
FilePos = StartPos = ftell (other.File);
}
FileReader::FileReader (const char *filename)
: File(NULL), Length(0), StartPos(0), FilePos(0), CloseOnDestruct(false)
{
if (!Open(filename))
~StdFileReader()
{
I_Error ("Could not open %s", filename);
if (File != nullptr)
{
fclose(File);
}
File = nullptr;
}
}
FileReader::FileReader (FILE *file)
: File(file), Length(0), StartPos(0), FilePos(0), CloseOnDestruct(false)
{
Length = CalcFileLen();
}
FileReader::FileReader (FILE *file, long length)
: File(file), Length(length), CloseOnDestruct(true)
{
FilePos = StartPos = ftell (file);
}
FileReader::~FileReader()
{
Close();
}
void FileReader::Close()
{
if (CloseOnDestruct && File != NULL)
bool Open(const char *filename, long startpos = 0, long len = -1)
{
fclose (File);
File = fopen(filename, "rb");
if (File == nullptr) return false;
FilePos = startpos;
StartPos = startpos;
Length = CalcFileLen();
if (len >= 0 && len < Length) Length = len;
if (startpos > 0) Seek(startpos, SEEK_SET);
return true;
}
File = NULL;
}
bool FileReader::Open (const char *filename)
{
File = openfd (filename);
if (File == NULL) return false;
FilePos = 0;
StartPos = 0;
CloseOnDestruct = true;
Length = CalcFileLen();
return true;
}
void FileReader::ResetFilePtr ()
{
FilePos = ftell (File);
}
long FileReader::Tell () const
{
return FilePos - StartPos;
}
long FileReader::Seek (long offset, int origin)
{
if (origin == SEEK_SET)
long Tell() const override
{
offset += StartPos;
return FilePos - StartPos;
}
else if (origin == SEEK_CUR)
long Seek(long offset, int origin) override
{
if (origin == SEEK_SET)
{
offset += StartPos;
}
else if (origin == SEEK_CUR)
{
offset += FilePos;
}
else if (origin == SEEK_END)
{
offset += StartPos + Length;
}
if (offset < StartPos || offset >= StartPos + Length) return -1; // out of scope
if (0 == fseek(File, offset, SEEK_SET))
{
FilePos = offset;
return 0;
}
return -1;
}
long Read(void *buffer, long len) override
{
assert(len >= 0);
if (len <= 0) return 0;
if (FilePos + len > StartPos + Length)
{
len = Length - FilePos + StartPos;
}
len = (long)fread(buffer, 1, len, File);
FilePos += len;
return len;
}
char *Gets(char *strbuf, int len) override
{
if (len <= 0 || FilePos >= StartPos + Length) return NULL;
char *p = fgets(strbuf, len, File);
if (p != NULL)
{
int old = FilePos;
FilePos = ftell(File);
if (FilePos - StartPos > Length)
{
strbuf[Length - old + StartPos] = 0;
}
}
return p;
}
private:
long CalcFileLen() const
{
long endpos;
fseek(File, 0, SEEK_END);
endpos = ftell(File);
fseek(File, 0, SEEK_SET);
return endpos;
}
};
//==========================================================================
//
// FileReaderRedirect
//
// like the above, but uses another File reader as its backing data
//
//==========================================================================
class FileReaderRedirect : public FileReaderInterface
{
FileRdr *mReader = nullptr;
long Length = 0;
long StartPos = 0;
long FilePos = 0;
public:
FileReaderRedirect(FileRdr &parent, long start, long length)
{
mReader = &parent;
FilePos = start;
StartPos = start;
Length = length;
Seek(0, SEEK_SET);
}
virtual long Tell() const override
{
return FilePos - StartPos;
}
virtual long Seek(long offset, int origin)
{
switch (origin)
{
case SEEK_SET:
offset += StartPos;
break;
case SEEK_END:
offset += StartPos + Length;
break;
case SEEK_CUR:
offset += (long)mReader->Tell();
break;
}
if (offset < StartPos || offset >= StartPos + Length) return -1; // out of scope
if (mReader->Seek(offset, FileRdr::SeekSet) == 0)
{
FilePos = offset;
return 0;
}
return -1;
}
virtual long Read(void *buffer, long len)
{
assert(len >= 0);
if (len <= 0) return 0;
if (FilePos + len > StartPos + Length)
{
len = Length - FilePos + StartPos;
}
len = (long)mReader->Read(buffer, len);
FilePos += len;
return len;
}
virtual char *Gets(char *strbuf, int len)
{
if (len <= 0 || FilePos >= StartPos + Length) return NULL;
char *p = mReader->Gets(strbuf, len);
if (p != NULL)
{
int old = FilePos;
FilePos = (long)mReader->Tell();
if (FilePos - StartPos > Length)
{
strbuf[Length - old + StartPos] = 0;
}
}
return p;
}
};
//==========================================================================
//
// MemoryReader
//
// reads data from a block of memory
//
//==========================================================================
long MemoryReader::Tell() const
{
return FilePos;
}
long MemoryReader::Seek(long offset, int origin)
{
switch (origin)
{
case SEEK_CUR:
offset += FilePos;
break;
case SEEK_END:
offset += Length;
break;
}
else if (origin == SEEK_END)
{
offset += StartPos + Length;
}
if (0 == fseek (File, offset, SEEK_SET))
{
FilePos = offset;
return 0;
}
return -1;
if (offset < 0 || offset >= Length) return -1;
FilePos = clamp<long>(offset, 0, Length);
return 0;
}
long FileReader::Read (void *buffer, long len)
long MemoryReader::Read(void *buffer, long len)
{
assert(len >= 0);
if (len <= 0) return 0;
if (FilePos + len > StartPos + Length)
{
len = Length - FilePos + StartPos;
}
len = (long)fread (buffer, 1, len, File);
if (len>Length - FilePos) len = Length - FilePos;
if (len<0) len = 0;
memcpy(buffer, bufptr + FilePos, len);
FilePos += len;
return len;
}
char *FileReader::Gets(char *strbuf, int len)
char *MemoryReader::Gets(char *strbuf, int len)
{
if (len <= 0 || FilePos >= StartPos + Length) return NULL;
char *p = fgets(strbuf, len, File);
if (p != NULL)
{
int old = FilePos;
FilePos = ftell(File);
if (FilePos - StartPos > Length)
{
strbuf[Length - old + StartPos] = 0;
}
}
return p;
}
char *FileReader::GetsFromBuffer(const char * bufptr, char *strbuf, int len)
{
if (len>Length-FilePos) len=Length-FilePos;
if (len>Length - FilePos) len = Length - FilePos;
if (len <= 0) return NULL;
char *p = strbuf;
@ -191,7 +293,7 @@ char *FileReader::GetsFromBuffer(const char * bufptr, char *strbuf, int len)
{
*p++ = bufptr[FilePos];
len--;
if (bufptr[FilePos] == '\n')
if (bufptr[FilePos] == '\n')
{
FilePos++;
break;
@ -199,360 +301,11 @@ char *FileReader::GetsFromBuffer(const char * bufptr, char *strbuf, int len)
}
FilePos++;
}
if (p==strbuf) return NULL;
*p++=0;
if (p == strbuf) return NULL;
*p++ = 0;
return strbuf;
}
long FileReader::CalcFileLen() const
{
long endpos;
fseek (File, 0, SEEK_END);
endpos = ftell (File);
fseek (File, 0, SEEK_SET);
return endpos;
}
//==========================================================================
//
// FileReaderZ
//
// The zlib wrapper
// reads data from a ZLib compressed stream
//
//==========================================================================
FileReaderZ::FileReaderZ (FileReader &file, bool zip)
: File(file), SawEOF(false)
{
int err;
FillBuffer ();
Stream.zalloc = Z_NULL;
Stream.zfree = Z_NULL;
if (!zip) err = inflateInit (&Stream);
else err = inflateInit2 (&Stream, -MAX_WBITS);
if (err != Z_OK)
{
I_Error ("FileReaderZ: inflateInit failed: %s\n", M_ZLibError(err).GetChars());
}
}
FileReaderZ::~FileReaderZ ()
{
inflateEnd (&Stream);
}
long FileReaderZ::Read (void *buffer, long len)
{
int err;
Stream.next_out = (Bytef *)buffer;
Stream.avail_out = len;
do
{
err = inflate (&Stream, Z_SYNC_FLUSH);
if (Stream.avail_in == 0 && !SawEOF)
{
FillBuffer ();
}
} while (err == Z_OK && Stream.avail_out != 0);
if (err != Z_OK && err != Z_STREAM_END)
{
I_Error ("Corrupt zlib stream");
}
if (Stream.avail_out != 0)
{
I_Error ("Ran out of data in zlib stream");
}
return len - Stream.avail_out;
}
void FileReaderZ::FillBuffer ()
{
long numread = File.Read (InBuff, BUFF_SIZE);
if (numread < BUFF_SIZE)
{
SawEOF = true;
}
Stream.next_in = InBuff;
Stream.avail_in = numread;
}
//==========================================================================
//
// FileReaderZ
//
// The bzip2 wrapper
// reads data from a libbzip2 compressed stream
//
//==========================================================================
FileReaderBZ2::FileReaderBZ2 (FileReader &file)
: File(file), SawEOF(false)
{
int err;
FillBuffer ();
Stream.bzalloc = NULL;
Stream.bzfree = NULL;
Stream.opaque = NULL;
err = BZ2_bzDecompressInit(&Stream, 0, 0);
if (err != BZ_OK)
{
I_Error ("FileReaderBZ2: bzDecompressInit failed: %d\n", err);
}
}
FileReaderBZ2::~FileReaderBZ2 ()
{
BZ2_bzDecompressEnd (&Stream);
}
long FileReaderBZ2::Read (void *buffer, long len)
{
int err;
Stream.next_out = (char *)buffer;
Stream.avail_out = len;
do
{
err = BZ2_bzDecompress(&Stream);
if (Stream.avail_in == 0 && !SawEOF)
{
FillBuffer ();
}
} while (err == BZ_OK && Stream.avail_out != 0);
if (err != BZ_OK && err != BZ_STREAM_END)
{
I_Error ("Corrupt bzip2 stream");
}
if (Stream.avail_out != 0)
{
I_Error ("Ran out of data in bzip2 stream");
}
return len - Stream.avail_out;
}
void FileReaderBZ2::FillBuffer ()
{
long numread = File.Read(InBuff, BUFF_SIZE);
if (numread < BUFF_SIZE)
{
SawEOF = true;
}
Stream.next_in = (char *)InBuff;
Stream.avail_in = numread;
}
//==========================================================================
//
// bz_internal_error
//
// libbzip2 wants this, since we build it with BZ_NO_STDIO set.
//
//==========================================================================
extern "C" void bz_internal_error (int errcode)
{
I_FatalError("libbzip2: internal error number %d\n", errcode);
}
//==========================================================================
//
// FileReaderLZMA
//
// The lzma wrapper
// reads data from a LZMA compressed stream
//
//==========================================================================
// This is retarded but necessary to work around the inclusion of windows.h in recent
// LZMA versions, meaning it's no longer possible to include the LZMA headers in files.h.
// As a result we cannot declare the CLzmaDec member in the header so we work around
// it my wrapping it into another struct that can be declared anonymously in the header.
struct FileReaderLZMA::StreamPointer
{
CLzmaDec Stream;
};
static void *SzAlloc(ISzAllocPtr, size_t size) { return malloc(size); }
static void SzFree(ISzAllocPtr, void *address) { free(address); }
ISzAlloc g_Alloc = { SzAlloc, SzFree };
FileReaderLZMA::FileReaderLZMA (FileReader &file, size_t uncompressed_size, bool zip)
: File(file), SawEOF(false)
{
uint8_t header[4 + LZMA_PROPS_SIZE];
int err;
assert(zip == true);
Size = uncompressed_size;
OutProcessed = 0;
// Read zip LZMA properties header
if (File.Read(header, sizeof(header)) < (long)sizeof(header))
{
I_Error("FileReaderLZMA: File too shart\n");
}
if (header[2] + header[3] * 256 != LZMA_PROPS_SIZE)
{
I_Error("FileReaderLZMA: LZMA props size is %d (expected %d)\n",
header[2] + header[3] * 256, LZMA_PROPS_SIZE);
}
FillBuffer();
Streamp = new StreamPointer;
LzmaDec_Construct(&Streamp->Stream);
err = LzmaDec_Allocate(&Streamp->Stream, header + 4, LZMA_PROPS_SIZE, &g_Alloc);
if (err != SZ_OK)
{
I_Error("FileReaderLZMA: LzmaDec_Allocate failed: %d\n", err);
}
LzmaDec_Init(&Streamp->Stream);
}
FileReaderLZMA::~FileReaderLZMA ()
{
LzmaDec_Free(&Streamp->Stream, &g_Alloc);
delete Streamp;
}
long FileReaderLZMA::Read (void *buffer, long len)
{
int err;
Byte *next_out = (Byte *)buffer;
do
{
ELzmaFinishMode finish_mode = LZMA_FINISH_ANY;
ELzmaStatus status;
size_t out_processed = len;
size_t in_processed = InSize;
err = LzmaDec_DecodeToBuf(&Streamp->Stream, next_out, &out_processed, InBuff + InPos, &in_processed, finish_mode, &status);
InPos += in_processed;
InSize -= in_processed;
next_out += out_processed;
len = (long)(len - out_processed);
if (err != SZ_OK)
{
I_Error ("Corrupt LZMA stream");
}
if (in_processed == 0 && out_processed == 0)
{
if (status != LZMA_STATUS_FINISHED_WITH_MARK)
{
I_Error ("Corrupt LZMA stream");
}
}
if (InSize == 0 && !SawEOF)
{
FillBuffer ();
}
} while (err == SZ_OK && len != 0);
if (err != Z_OK && err != Z_STREAM_END)
{
I_Error ("Corrupt LZMA stream");
}
if (len != 0)
{
I_Error ("Ran out of data in LZMA stream");
}
return (long)(next_out - (Byte *)buffer);
}
void FileReaderLZMA::FillBuffer ()
{
long numread = File.Read(InBuff, BUFF_SIZE);
if (numread < BUFF_SIZE)
{
SawEOF = true;
}
InPos = 0;
InSize = numread;
}
//==========================================================================
//
// MemoryReader
//
// reads data from a block of memory
//
//==========================================================================
MemoryReader::MemoryReader (const char *buffer, long length)
{
bufptr=buffer;
Length=length;
FilePos=0;
}
MemoryReader::~MemoryReader ()
{
}
long MemoryReader::Tell () const
{
return FilePos;
}
long MemoryReader::Seek (long offset, int origin)
{
switch (origin)
{
case SEEK_CUR:
offset+=FilePos;
break;
case SEEK_END:
offset+=Length;
break;
}
FilePos=clamp<long>(offset,0,Length);
return 0;
}
long MemoryReader::Read (void *buffer, long len)
{
if (len>Length-FilePos) len=Length-FilePos;
if (len<0) len=0;
memcpy(buffer,bufptr+FilePos,len);
FilePos+=len;
return len;
}
char *MemoryReader::Gets(char *strbuf, int len)
{
return GetsFromBuffer(bufptr, strbuf, len);
}
//==========================================================================
//
// MemoryArrayReader
@ -561,57 +314,88 @@ char *MemoryReader::Gets(char *strbuf, int len)
//
//==========================================================================
MemoryArrayReader::MemoryArrayReader (const char *buffer, long length)
class MemoryArrayReader : public MemoryReader
{
if (length > 0)
TArray<uint8_t> buf;
public:
MemoryArrayReader(const char *buffer, long length)
{
buf.Resize(length);
memcpy(&buf[0], buffer, length);
if (length > 0)
{
buf.Resize(length);
memcpy(&buf[0], buffer, length);
}
Length = length;
FilePos = 0;
bufptr = (const char *)&buf[0];
}
Length = length;
FilePos=0;
}
MemoryArrayReader::~MemoryArrayReader ()
TArray<uint8_t> &GetArray() { return buf; }
void UpdateLength() { Length = buf.Size(); }
};
//==========================================================================
//
// FileReader
//
// this wraps the different reader types in an object with value semantics.
//
//==========================================================================
bool FileRdr::OpenFile(const char *filename, FileRdr::Size start, FileRdr::Size length)
{
auto reader = new StdFileReader;
if (!reader->Open(filename, (long)start, (long)length)) return false;
Close();
mReader = reader;
return true;
}
long MemoryArrayReader::Tell () const
bool FileRdr::OpenFilePart(FileRdr &parent, FileRdr::Size start, FileRdr::Size length)
{
return FilePos;
auto reader = new FileReaderRedirect(parent, (long)start, (long)length);
Close();
mReader = reader;
return true;
}
long MemoryArrayReader::Seek (long offset, int origin)
bool FileRdr::OpenMemory(const void *mem, FileRdr::Size length)
{
switch (origin)
{
case SEEK_CUR:
offset+=FilePos;
break;
case SEEK_END:
offset+=Length;
break;
}
FilePos=clamp<long>(offset,0,Length);
return 0;
Close();
mReader = new MemoryReader((const char *)mem, (long)length);
return true;
}
long MemoryArrayReader::Read (void *buffer, long len)
bool FileRdr::OpenMemoryArray(const void *mem, FileRdr::Size length)
{
if (len>Length-FilePos) len=Length-FilePos;
if (len<0) len=0;
memcpy(buffer,&buf[FilePos],len);
FilePos+=len;
return len;
Close();
mReader = new MemoryArrayReader((const char *)mem, (long)length);
return true;
}
char *MemoryArrayReader::Gets(char *strbuf, int len)
bool FileRdr::OpenMemoryArray(std::function<bool(TArray<uint8_t>&)> getter)
{
return GetsFromBuffer((char*)&buf[0], strbuf, len);
auto reader = new MemoryArrayReader(nullptr, 0);
if (getter(reader->GetArray()))
{
Close();
reader->UpdateLength();
mReader = reader;
return true;
}
else
{
// This will keep the old buffer, if one existed
delete reader;
return false;
}
}
//==========================================================================
//
// FileWriter (the motivation here is to have a buffer writing subclass)
@ -671,7 +455,6 @@ long FileWriter::Seek(long offset, int mode)
}
}
size_t FileWriter::Printf(const char *fmt, ...)
{
va_list ap;
@ -689,52 +472,3 @@ size_t BufferWriter::Write(const void *buffer, size_t len)
memcpy(&mBuffer[ofs], buffer, len);
return len;
}
//////////////////////////////////////
//
// The new wrapper
//
//////////////////////////////////////
bool FileRdr::OpenFile(const char *filename)
{
auto reader = new FileReader;
if (!reader->Open(filename)) return false;
Close();
mReader = reader;
return true;
}
bool FileRdr::OpenMemory(const void *mem, FileRdr::Size length)
{
Close();
mReader = new MemoryReader((const char *)mem, (long)length);
return true;
}
bool FileRdr::OpenMemoryArray(const void *mem, FileRdr::Size length)
{
Close();
mReader = new MemoryArrayReader((const char *)mem, (long)length);
return true;
}
bool FileRdr::OpenMemoryArray(std::function<bool(TArray<uint8_t>&)> getter)
{
auto reader = new MemoryArrayReader(nullptr, 0);
if (getter(reader->GetArray()))
{
Close();
reader->UpdateLength();
mReader = reader;
return true;
}
else
{
// This will keep the old
delete reader;
return false;
}
}