From 66d9eb06299397bf2618486a28376bf0186069f7 Mon Sep 17 00:00:00 2001 From: Marisa Kirisame Date: Sat, 13 Apr 2019 20:07:45 +0200 Subject: [PATCH] Wow I forgot I have stuff here that wasn't added. --- dtexdupes.c | 313 ++++++++++++++++++++++++++++++++++ iwad64ex.c | 128 ++++++++++++++ ufontext.c | 477 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 918 insertions(+) create mode 100644 dtexdupes.c create mode 100644 iwad64ex.c create mode 100644 ufontext.c diff --git a/dtexdupes.c b/dtexdupes.c new file mode 100644 index 0000000..8e11d62 --- /dev/null +++ b/dtexdupes.c @@ -0,0 +1,313 @@ +/* + dtexdupes.c : Find textures with identical names in multiple doom mods. + Supports wad, pk3 and folders. Handles TEXTURE1/2 and TEXTURES. + (C)2018 Marisa Kirisame, UnSX Team. + Released under the GNU General Public License version 3 (or later). + Requires libarchive. Made for *nix platforms. +*/ +#define _XOPEN_SOURCE 700 +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// wad stuff +typedef struct +{ + char id[4]; + uint32_t nlumps; + uint32_t dirpos; +} wad_t; + +typedef struct +{ + uint32_t pos; + uint32_t size; + char name[8]; +} lump_t; + +// list of texture names +uint8_t **tex = 0; +size_t ntex = 0, nresv = 0; + +void freetex( void ) +{ + if ( !tex ) return; + for ( int i=0; i= nresv ) + { + nresv += CHUNK_SZ; + tex = realloc(tex,sizeof(uint8_t*)*nresv); + } + // append + tex[ntex] = malloc(len+1); + strncpy(tex[ntex],name,len); + tex[ntex][len] = '\0'; // guarantee null-termination + ntex++; +} + +int processtexture12( uint8_t type, uint8_t *data ) +{ + // parse TEXTURE1/2 lumps + int32_t nent = *(int32_t*)data; + int32_t *ofs = (int32_t*)(data+4); + for ( int i=0; iname,"F_END",8) ) break; + if ( cur->size != 4096 ) continue; + appendtex(cur->name,8); + cnt++; + } + printf(" F_START/END read (%d entries)\n",cnt); + return cnt; +} + +int processtxsection( lump_t *head ) +{ + // check everything between TX_START and TX_END + int cnt = 0; + lump_t *cur = head; + while ( cur ) + { + cur++; + if ( !strncmp(cur->name,"TX_END",8) ) break; + appendtex(cur->name,8); + cnt++; + } + printf(" TX_START/END read (%d entries)\n",cnt); + return cnt; +} + +int processwad( uint8_t *w ) +{ + lump_t *dir = (lump_t*)(w+((wad_t*)w)->dirpos); + for ( int i=0; i<((wad_t*)w)->nlumps; i++ ) + { + if ( !strncmp(dir->name,"TEXTURES",8) ) + processtextures(w+dir->pos); + else if ( !strncmp(dir->name,"TEXTURE1",8) + || !strncmp(dir->name,"TEXTURE2",8) ) + processtexture12(dir->name[7],w+dir->pos); + else if ( !strncmp(dir->name,"TX_START",8) ) + processtxsection(dir); + else if ( !strncmp(dir->name,"F_START",8) ) + processflats(dir); + dir++; + } +} + +int processtxfolder_arc( struct archive *ar ) +{ + // libarchive apparently has no concept of "folders" so we have to + // loop through everything here and check its full path + return 0; // TODO +} + +static int ncnt = 0, fcnt = 0; + +static int ftw_callback( const char *path, const struct stat *st, + const int type, struct FTW *info ) +{ + const char *bname = path+info->base; + if ( info->level == 1 ) + { + if ( (type == FTW_D) && strcasecmp(bname,"textures") && strcasecmp(bname,"flats") ) + return FTW_SKIP_SUBTREE; + if ( type != FTW_F ) return FTW_CONTINUE; + FILE *f; + if ( !(f = fopen(path,"rb")) ) + { + fprintf(stderr,"cannot open %s: %s\n",bname, + strerror(errno)); + return FTW_CONTINUE; + } + char magic[4]; + if ( fread(magic,1,4,f) == 4 ) + { + if ( !strncmp(magic,"IWAD",4) + || !strncmp(magic,"PWAD",4) ) + { + uint8_t *w = malloc(st->st_size); + if ( !w ) + { + fprintf(stderr,"cannot allocate memory" + " for %s: %s\n",bname, + strerror(errno)); + fclose(f); + return FTW_CONTINUE; + } + fseek(f,0,SEEK_SET); + fread(w,1,st->st_size,f); + fclose(f); + printf("Processing %.4s %s\n",magic,bname); + ncnt += processwad(w); + free(w); + return FTW_CONTINUE; + } + } + fseek(f,0,SEEK_SET); + if ( !strncasecmp(bname,"TEXTURES",8) ) + { + uint8_t *d; + if ( !(d = malloc(st->st_size)) ) + { + fprintf(stderr,"cannot allocate memory for %s:" + " %s\n",bname,strerror(errno)); + fclose(f); + return FTW_CONTINUE; + } + fread(d,1,st->st_size,f); + ncnt += processtextures(d); + free(d); + } + else if ( !strncasecmp(bname,"TEXTURE1",8) + || !strncasecmp(bname,"TEXTURE2",8) ) + { + uint8_t *d; + if ( !(d = malloc(st->st_size)) ) + { + fprintf(stderr,"cannot allocate memory for %s:" + " %s\n",bname,strerror(errno)); + fclose(f); + return FTW_CONTINUE; + } + fread(d,1,st->st_size,f); + ncnt += processtexture12(bname[7],d); + free(d); + } + fclose(f); + return FTW_CONTINUE; + } + if ( type == FTW_F ) + { + // strip extension + char *ext = strchr(bname,'.'); + if ( ext ) *ext = '\0'; + appendtex(bname,8); + ncnt++; + fcnt++; + } + return FTW_CONTINUE; +} + +int processtxfolder( const char *path ) +{ + ncnt = 0, fcnt = 0; + // recurse through specified directory and check any and all files + nftw(path,ftw_callback,15,FTW_ACTIONRETVAL|FTW_PHYS); + printf(" Directory read (%d entries)\n",fcnt); + return ncnt; +} + +int main( int argc, char **argv ) +{ + struct stat st; + FILE *f; + int tot = 0; + // sift through all parameters + for ( int i=1; i +#include +#include +#include +#include + +typedef struct +{ + char sig[4]; + uint32_t nlumps; + uint32_t dirpos; +} wad_t; + +typedef struct +{ + uint32_t filepos, size; + char name[8]; +} lump_t; + +char *data; +wad_t *wad; +lump_t *lumps; + +uint32_t btl( uint32_t b ) +{ + return ((b>>24)&0xff)|((b<<8)&0xff0000) + |((b>>8)&0xff00)|((b<<24)&0xff000000); +} + +int main( int argc, char **argv ) +{ + if ( argc < 3 ) + { + fprintf(stderr,"usage: iwad64ex \n"); + return 1; + } + FILE *romfile = fopen(argv[1],"rb"); + if ( !romfile ) + { + fprintf(stderr,"failed to open %s for reading: %s\n", + argv[1],strerror(errno)); + return 2; + } + ssize_t fpos = 0; + char buf[4096]; + ssize_t found = 0; + while ( !feof(romfile) ) + { + memset(buf,0,4096); + int bsiz = fread(buf,1,4096,romfile); + // to avoid GNU-exclusive memmem function I'll just roll out + // my own thing here (fuckin' GNU and their exclusives...) + for ( int i=0; inlumps = btl(wad->nlumps); + wad->dirpos = btl(wad->dirpos); + // reallocate to fit + size_t fullsiz = wad->dirpos+wad->nlumps*16; + data = realloc(data,fullsiz); + wad = (wad_t*)data; + if ( errno ) + { + fclose(romfile); + free(data); + fprintf(stderr,"cannot allocate %zd bytes for full wad: %s", + fullsiz,strerror(errno)); + return 32; + } + fread(data+12,1,fullsiz-12,romfile); + lumps = (lump_t*)(data+wad->dirpos); + // loop and fix endianness + for ( int i=0; inlumps; i++ ) + { + lumps[i].filepos = btl(lumps[i].filepos); + lumps[i].size = btl(lumps[i].size); + printf("%.8s %d %d\n",lumps[i].name,lumps[i].size, + lumps[i].filepos); + } + fclose(romfile); + FILE *wadfile = fopen(argv[2],"wb"); + if ( !wadfile ) + { + fprintf(stderr,"failed to open %s for writing: %s\n", + argv[2],strerror(errno)); + return 4; + } + fwrite(data,1,fullsiz,wadfile); + fclose(wadfile); + free(data); + printf("wrote %s, %d bytes\n",argv[2],fullsiz); + return 0; +} diff --git a/ufontext.c b/ufontext.c new file mode 100644 index 0000000..bfca85e --- /dev/null +++ b/ufontext.c @@ -0,0 +1,477 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define UPKG_MAGIC 0x9E2A83C1 + +// uncomment if you want full data dumps, helpful if you need to reverse engineer some unsupported format +//#define _DEBUG 1 + +typedef struct +{ + uint32_t magic; + uint16_t pkgver, license; + uint32_t flags, nnames, onames, nexports, oexports, nimports, oimports; +} upkg_header_t; + +uint8_t *pkgfile; +upkg_header_t *head; +size_t fpos = 0; + +uint8_t readbyte( void ) +{ + uint8_t val = pkgfile[fpos]; + fpos++; + return val; +} + +uint16_t readword( void ) +{ + uint16_t val = *(uint16_t*)(pkgfile+fpos); + fpos += 2; + return val; +} + +uint32_t readdword( void ) +{ + uint32_t val = *(uint32_t*)(pkgfile+fpos); + fpos += 4; + return val; +} + +float readfloat( void ) +{ + float val = *(float*)(pkgfile+fpos); + fpos += 4; + return val; +} + +#define READSTRUCT(x,y) {memcpy(&x,pkgfile+fpos,sizeof(y));fpos+=sizeof(y);} + +// reads a compact index value +int32_t readindex( void ) +{ + uint8_t byte[5] = {0}; + byte[0] = readbyte(); + if ( !byte[0] ) return 0; + if ( byte[0]&0x40 ) + { + for ( int i=1; i<5; i++ ) + { + byte[i] = readbyte(); + if ( !(byte[i]&0x80) ) break; + } + } + int32_t tf = byte[0]&0x3f; + tf |= (int32_t)(byte[1]&0x7f)<<6; + tf |= (int32_t)(byte[2]&0x7f)<<13; + tf |= (int32_t)(byte[3]&0x7f)<<20; + tf |= (int32_t)(byte[4]&0x7f)<<27; + if ( byte[0]&0x80 ) tf *= -1; + return tf; +} + +// reads a name table entry +size_t readname( int *olen ) +{ + size_t pos = fpos; + if ( head->pkgver >= 64 ) + { + int32_t len = readindex(); + pos = fpos; + if ( olen ) *olen = len; + if ( len <= 0 ) return pos; + fpos += len; + } + else + { + int c, p = 0; + while ( (c = readbyte()) ) p++; + if ( olen ) *olen = p; + } + fpos += 4; + return pos; +} + +size_t getname( int index, int *olen ) +{ + size_t prev = fpos; + fpos = head->onames; + size_t npos = 0; + for ( int i=0; i<=index; i++ ) + npos = readname(olen); + fpos = prev; + return npos; +} + +// checks if a name exists +int hasname( const char *needle ) +{ + if ( !needle ) return 0; + size_t prev = fpos; + fpos = head->onames; + int found = 0; + int nlen = strlen(needle); + for ( uint32_t i=0; innames; i++ ) + { + int32_t len = 0; + if ( head->pkgver >= 64 ) + { + len = readindex(); + if ( len <= 0 ) continue; + } + int c = 0, p = 0, match = 1; + while ( (c = readbyte()) ) + { + if ( (p >= nlen) || (needle[p] != c) ) match = 0; + p++; + if ( len && (p > len) ) break; + } + if ( match ) + { + found = 1; + break; + } + fpos += 4; + } + fpos = prev; + return found; +} + +// read import table entry and return index of its object name +int32_t readimport( void ) +{ + readindex(); + readindex(); + if ( head->pkgver >= 60 ) fpos += 4; + else readindex(); + return readindex(); +} + +int32_t getimport( int index ) +{ + size_t prev = fpos; + fpos = head->oimports; + int32_t iname = 0; + for ( int i=0; i<=index; i++ ) + iname = readimport(); + fpos = prev; + return iname; +} + +// fully read import table entry +void readimport2( int32_t *cpkg, int32_t *cname, int32_t *pkg, int32_t *name ) +{ + *cpkg = readindex(); + *cname = readindex(); + if ( head->pkgver >= 60 ) *pkg = readdword(); + else *pkg = readindex(); + *name = readindex(); +} + +void getimport2( int index, int32_t *cpkg, int32_t *cname, int32_t *pkg, + int32_t *name ) +{ + size_t prev = fpos; + fpos = head->oimports; + for ( int i=0; i<=index; i++ ) + readimport2(cpkg,cname,pkg,name); + fpos = prev; +} + +// read export table entry +void readexport( int32_t *class, int32_t *ofs, int32_t *siz, int32_t *name ) +{ + *class = readindex(); + readindex(); + if ( head->pkgver >= 60 ) fpos += 4; + *name = readindex(); + fpos += 4; + *siz = readindex(); + if ( *siz > 0 ) *ofs = readindex(); +} + +void getexport( int index, int32_t *class, int32_t *ofs, int32_t *siz, + int32_t *name ) +{ + size_t prev = fpos; + fpos = head->oexports; + for ( int i=0; i<=index; i++ ) + readexport(class,ofs,siz,name); + fpos = prev; +} + +// fully read export table entry +void readexport2( int32_t *class, int32_t *super, int32_t *pkg, int32_t *name, + uint32_t *flags, int32_t *siz, int32_t *ofs ) +{ + *class = readindex(); + *super = readindex(); + if ( head->pkgver >= 60 ) *pkg = readdword(); + else *pkg = readindex(); + *name = readindex(); + *flags = readdword(); + *siz = readindex(); + if ( *siz > 0 ) *ofs = readindex(); +} + +void getexport2( int index, int32_t *class, int32_t *super, int32_t *pkg, + int32_t *name, uint32_t *flags, int32_t *siz, int32_t *ofs ) +{ + size_t prev = fpos; + fpos = head->oexports; + for ( int i=0; i<=index; i++ ) + readexport2(class,super,pkg,name,flags,siz,ofs); + fpos = prev; +} + +// construct full name for object +// shamelessly recycled from my old upackage project +void imprefix( FILE *f, int32_t i ); +void exprefix( FILE *f, int32_t i ); +void imprefix( FILE *f, int32_t i ) +{ + int32_t cpkg, cnam, pkg, nam; + getimport2(i,&cpkg,&cnam,&pkg,&nam); + if ( pkg < 0 ) imprefix(f,-pkg-1); + else if ( pkg > 0 ) exprefix(f,pkg-1); + if ( pkg ) fprintf(f,"."); + int32_t l; + char *pname = (char*)(pkgfile+getname(nam,&l)); + fprintf(f,"%.*s",l,pname); +} +void exprefix( FILE *f, int32_t i ) +{ + int32_t cls, sup, pkg, nam, siz, ofs; + uint32_t fl; + getexport2(i,&cls,&sup,&pkg,&nam,&fl,&siz,&ofs); + if ( pkg > 0 ) + { + exprefix(f,pkg-1); + fprintf(f,"."); + } + int32_t l; + char *pname = (char*)(pkgfile+getname(nam,&l)); + fprintf(f,"%.*s",l,pname); +} +void construct_fullname( FILE *f, int32_t i ) +{ + if ( i > 0 ) exprefix(f,i-1); + else if ( i < 0 ) imprefix(f,-i-1); + else fprintf(f,"None"); +} + +typedef struct +{ + int32_t x, y, w, h; +} ufontchar_t; + +typedef struct +{ + int32_t texture; + uint32_t charcount; + ufontchar_t *chars; +} ufonttex_t; + +typedef struct +{ + uint8_t texcount; + ufonttex_t *tex; +} ufonthdr_t; + +void savefont( int32_t namelen, char *name ) +{ + char fname[256] = {0}; + FILE *f; + if ( head->license == 2 ) + { + printf(" FAIL: Postal 2 fonts not yet supported\n"); + return; + } + ufonthdr_t fhead; + memset(&fhead,0,sizeof(ufonthdr_t)); + fhead.texcount = readbyte(); + fhead.tex = calloc(fhead.texcount,sizeof(ufonttex_t)); + for ( int i=0; i\n"); + return 1; + } + int fd = open(argv[1],O_RDONLY); + if ( fd == -1 ) + { + printf("Failed to open file %s: %s\n",argv[1],strerror(errno)); + return 1; + } + struct stat st; + fstat(fd,&st); + pkgfile = malloc(st.st_size); + memset(pkgfile,0,st.st_size); + head = (upkg_header_t*)pkgfile; + int r = 0; + do + { + r = read(fd,pkgfile+fpos,131072); + if ( r == -1 ) + { + close(fd); + free(pkgfile); + printf("Read failed for file %s: %s\n",argv[1], + strerror(errno)); + return 4; + } + fpos += r; + } + while ( r > 0 ); + close(fd); + fpos = 0; + if ( head->magic != UPKG_MAGIC ) + { + printf("File %s is not a valid unreal package!\n",argv[1]); + free(pkgfile); + return 2; + } + if ( !hasname("Font") ) + { + printf("Package %s does not contain Fonts\n",argv[1]); + free(pkgfile); + return 4; + } + // loop through exports and search for fonts + fpos = head->oexports; + for ( uint32_t i=0; inexports; i++ ) + { + int32_t class, ofs, siz, name; + readexport(&class,&ofs,&siz,&name); + if ( (siz <= 0) || (class >= 0) ) continue; + // get the class name + class = -class-1; + if ( (uint32_t)class > head->nimports ) continue; + int32_t l = 0; + char *n = (char*)(pkgfile+getname(getimport(class),&l)); + if ( strncmp(n,"Font",l) ) continue; + char *fnt = (char*)(pkgfile+getname(name,&l)); + printf("Font found: %.*s\n",l,fnt); + int32_t fntl = l; +#ifdef _DEBUG + char fname[256] = {0}; + snprintf(fname,256,"%.*s.object",fntl,fnt); + printf(" Dumping full object data to %s\n",fname); + FILE *f = fopen(fname,"wb"); + fwrite(pkgfile+ofs,siz,1,f); + fclose(f); +#endif + // begin reading data + size_t prev = fpos; + fpos = ofs; + if ( head->pkgver < 40 ) fpos += 8; + if ( head->pkgver < 60 ) fpos += 16; + int32_t prop = readindex(); + if ( (uint32_t)prop >= head->nnames ) + { + printf("Unknown property %d, skipping\n",prop); + fpos = prev; + continue; + } + char *pname = (char*)(pkgfile+getname(prop,&l)); +retry: + if ( strncasecmp(pname,"none",l) ) + { + uint8_t info = readbyte(); + int psiz = (info>>4)&0x7; + switch ( psiz ) + { + case 0: + psiz = 1; + break; + case 1: + psiz = 2; + break; + case 2: + psiz = 4; + break; + case 3: + psiz = 12; + break; + case 4: + psiz = 16; + break; + case 5: + psiz = readbyte(); + break; + case 6: + psiz = readword(); + break; + case 7: + psiz = readdword(); + break; + } + fpos += psiz; + printf(" Skipping property %.*s\n",l,pname); + prop = readindex(); + pname = (char*)(pkgfile+getname(prop,&l)); + goto retry; + } +#ifdef _DEBUG + snprintf(fname,256,"%.*s.ufnt",fntl,fnt); + printf(" Dumping full font struct to %s\n",fname); + f = fopen(fname,"wb"); + fwrite(pkgfile+fpos,siz-(fpos-ofs),1,f); + fclose(f); +#endif + savefont(fntl,fnt); + fpos = prev; + } + free(pkgfile); + return 0; +} +