Wow I forgot I have stuff here that wasn't added.

This commit is contained in:
Marisa the Magician 2019-04-13 20:07:45 +02:00
commit 66d9eb0629
3 changed files with 918 additions and 0 deletions

313
dtexdupes.c Normal file
View file

@ -0,0 +1,313 @@
/*
dtexdupes.c : Find textures with identical names in multiple doom mods.
Supports wad, pk3 and folders. Handles TEXTURE1/2 and TEXTURES.
(C)2018 Marisa Kirisame, UnSX Team.
Released under the GNU General Public License version 3 (or later).
Requires libarchive. Made for *nix platforms.
*/
#define _XOPEN_SOURCE 700
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <strings.h>
#include <errno.h>
#include <archive.h>
#include <archive_entry.h>
#include <sys/stat.h>
#include <ftw.h>
// wad stuff
typedef struct
{
char id[4];
uint32_t nlumps;
uint32_t dirpos;
} wad_t;
typedef struct
{
uint32_t pos;
uint32_t size;
char name[8];
} lump_t;
// list of texture names
uint8_t **tex = 0;
size_t ntex = 0, nresv = 0;
void freetex( void )
{
if ( !tex ) return;
for ( int i=0; i<ntex; i++ )
if ( tex[i] ) free(tex[i]);
free(tex);
}
int findtex( const char *name, uint8_t len )
{
for ( int i=0; i<ntex; i++ )
if ( !strncasecmp(tex[i],name,len) )
return 1;
return 0;
}
// we don't need a very large chunk size, this isn't a time-critical program
#define CHUNK_SZ 64
int appendtex( const char *name, uint8_t len )
{
if ( findtex(name,len) )
{
printf(" Texture %.*s already defined!\n",len,name);
return 0;
}
if ( !tex )
{
tex = malloc(sizeof(uint8_t*)*CHUNK_SZ);
nresv = CHUNK_SZ;
}
else if ( ntex >= nresv )
{
nresv += CHUNK_SZ;
tex = realloc(tex,sizeof(uint8_t*)*nresv);
}
// append
tex[ntex] = malloc(len+1);
strncpy(tex[ntex],name,len);
tex[ntex][len] = '\0'; // guarantee null-termination
ntex++;
}
int processtexture12( uint8_t type, uint8_t *data )
{
// parse TEXTURE1/2 lumps
int32_t nent = *(int32_t*)data;
int32_t *ofs = (int32_t*)(data+4);
for ( int i=0; i<nent; i++ )
{
uint8_t *name = (uint8_t*)(data+ofs[i]);
appendtex(name,8);
}
printf(" TEXTURE%c read (%d entries)\n",type,nent);
return nent;
}
int processtextures( uint8_t *data )
{
// parse TEXTURES entries
printf(" TEXTURES not yet supported, skipping\n");
return 0; // TODO
}
int processflats( lump_t *head )
{
// check everything between F_START and F_END
int cnt = 0;
lump_t *cur = head;
while ( cur )
{
cur++;
if ( !strncmp(cur->name,"F_END",8) ) break;
if ( cur->size != 4096 ) continue;
appendtex(cur->name,8);
cnt++;
}
printf(" F_START/END read (%d entries)\n",cnt);
return cnt;
}
int processtxsection( lump_t *head )
{
// check everything between TX_START and TX_END
int cnt = 0;
lump_t *cur = head;
while ( cur )
{
cur++;
if ( !strncmp(cur->name,"TX_END",8) ) break;
appendtex(cur->name,8);
cnt++;
}
printf(" TX_START/END read (%d entries)\n",cnt);
return cnt;
}
int processwad( uint8_t *w )
{
lump_t *dir = (lump_t*)(w+((wad_t*)w)->dirpos);
for ( int i=0; i<((wad_t*)w)->nlumps; i++ )
{
if ( !strncmp(dir->name,"TEXTURES",8) )
processtextures(w+dir->pos);
else if ( !strncmp(dir->name,"TEXTURE1",8)
|| !strncmp(dir->name,"TEXTURE2",8) )
processtexture12(dir->name[7],w+dir->pos);
else if ( !strncmp(dir->name,"TX_START",8) )
processtxsection(dir);
else if ( !strncmp(dir->name,"F_START",8) )
processflats(dir);
dir++;
}
}
int processtxfolder_arc( struct archive *ar )
{
// libarchive apparently has no concept of "folders" so we have to
// loop through everything here and check its full path
return 0; // TODO
}
static int ncnt = 0, fcnt = 0;
static int ftw_callback( const char *path, const struct stat *st,
const int type, struct FTW *info )
{
const char *bname = path+info->base;
if ( info->level == 1 )
{
if ( (type == FTW_D) && strcasecmp(bname,"textures") && strcasecmp(bname,"flats") )
return FTW_SKIP_SUBTREE;
if ( type != FTW_F ) return FTW_CONTINUE;
FILE *f;
if ( !(f = fopen(path,"rb")) )
{
fprintf(stderr,"cannot open %s: %s\n",bname,
strerror(errno));
return FTW_CONTINUE;
}
char magic[4];
if ( fread(magic,1,4,f) == 4 )
{
if ( !strncmp(magic,"IWAD",4)
|| !strncmp(magic,"PWAD",4) )
{
uint8_t *w = malloc(st->st_size);
if ( !w )
{
fprintf(stderr,"cannot allocate memory"
" for %s: %s\n",bname,
strerror(errno));
fclose(f);
return FTW_CONTINUE;
}
fseek(f,0,SEEK_SET);
fread(w,1,st->st_size,f);
fclose(f);
printf("Processing %.4s %s\n",magic,bname);
ncnt += processwad(w);
free(w);
return FTW_CONTINUE;
}
}
fseek(f,0,SEEK_SET);
if ( !strncasecmp(bname,"TEXTURES",8) )
{
uint8_t *d;
if ( !(d = malloc(st->st_size)) )
{
fprintf(stderr,"cannot allocate memory for %s:"
" %s\n",bname,strerror(errno));
fclose(f);
return FTW_CONTINUE;
}
fread(d,1,st->st_size,f);
ncnt += processtextures(d);
free(d);
}
else if ( !strncasecmp(bname,"TEXTURE1",8)
|| !strncasecmp(bname,"TEXTURE2",8) )
{
uint8_t *d;
if ( !(d = malloc(st->st_size)) )
{
fprintf(stderr,"cannot allocate memory for %s:"
" %s\n",bname,strerror(errno));
fclose(f);
return FTW_CONTINUE;
}
fread(d,1,st->st_size,f);
ncnt += processtexture12(bname[7],d);
free(d);
}
fclose(f);
return FTW_CONTINUE;
}
if ( type == FTW_F )
{
// strip extension
char *ext = strchr(bname,'.');
if ( ext ) *ext = '\0';
appendtex(bname,8);
ncnt++;
fcnt++;
}
return FTW_CONTINUE;
}
int processtxfolder( const char *path )
{
ncnt = 0, fcnt = 0;
// recurse through specified directory and check any and all files
nftw(path,ftw_callback,15,FTW_ACTIONRETVAL|FTW_PHYS);
printf(" Directory read (%d entries)\n",fcnt);
return ncnt;
}
int main( int argc, char **argv )
{
struct stat st;
FILE *f;
int tot = 0;
// sift through all parameters
for ( int i=1; i<argc; i++ )
{
if ( stat(argv[i],&st) )
{
fprintf(stderr,"cannot stat %s: %s\n",argv[i],
strerror(errno));
continue;
}
if ( S_ISDIR(st.st_mode) )
{
printf("Processing directory %s\n",argv[i]);
tot += processtxfolder(argv[i]);
continue;
}
else if ( !(f = fopen(argv[i],"rb")) )
{
fprintf(stderr,"cannot open file %s: %s\n",argv[i],
strerror(errno));
continue;
}
/* check for wad */
char magic[4];
if ( fread(magic,1,4,f) == 4 )
{
if ( !strncmp(magic,"IWAD",4)
|| !strncmp(magic,"PWAD",4) )
{
uint8_t *w = malloc(st.st_size);
if ( !w )
{
fprintf(stderr,"cannot allocate memory"
" for %s: %s\n",argv[i],
strerror(errno));
fclose(f);
continue;
}
fseek(f,0,SEEK_SET);
fread(w,1,st.st_size,f);
fclose(f);
printf("Processing %.4s %s\n",magic,argv[i]);
tot = processwad(w);
free(w);
continue;
}
}
fclose(f);
}
freetex();
return 0;
}

128
iwad64ex.c Normal file
View file

@ -0,0 +1,128 @@
/*
iwad64ex.c : Extracts the embedded IWAD in Hexen 64.
Currently does not decompress the data (or fix the endianness in data).
(C)2019 Marisa Kirisame, UnSX Team.
Released under the GNU General Public License version 3.
*/
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <stdint.h>
#include <errno.h>
typedef struct
{
char sig[4];
uint32_t nlumps;
uint32_t dirpos;
} wad_t;
typedef struct
{
uint32_t filepos, size;
char name[8];
} lump_t;
char *data;
wad_t *wad;
lump_t *lumps;
uint32_t btl( uint32_t b )
{
return ((b>>24)&0xff)|((b<<8)&0xff0000)
|((b>>8)&0xff00)|((b<<24)&0xff000000);
}
int main( int argc, char **argv )
{
if ( argc < 3 )
{
fprintf(stderr,"usage: iwad64ex <romfile> <wadfile>\n");
return 1;
}
FILE *romfile = fopen(argv[1],"rb");
if ( !romfile )
{
fprintf(stderr,"failed to open %s for reading: %s\n",
argv[1],strerror(errno));
return 2;
}
ssize_t fpos = 0;
char buf[4096];
ssize_t found = 0;
while ( !feof(romfile) )
{
memset(buf,0,4096);
int bsiz = fread(buf,1,4096,romfile);
// to avoid GNU-exclusive memmem function I'll just roll out
// my own thing here (fuckin' GNU and their exclusives...)
for ( int i=0; i<bsiz-5; i++ )
{
if ( !memcmp(buf+i,"IWAD",4)
|| !memcmp(buf+i,"PWAD",4) )
{
fpos += i;
found = 1;
break;
}
}
if ( found ) break;
fpos += bsiz;
}
if ( !found )
{
fclose(romfile);
fprintf(stderr,"WAD signature not found in ROM\n");
return 8;
}
printf("Embedded wad found at %#zx\n",fpos);
data = malloc(12);
if ( !data )
{
fclose(romfile);
fprintf(stderr,"cannot allocate 12 bytes for wad header: %s",
strerror(errno));
return 16;
}
fseek(romfile,fpos,SEEK_SET);
fread(data,1,12,romfile);
wad = (wad_t*)data;
// fix endianness
wad->nlumps = btl(wad->nlumps);
wad->dirpos = btl(wad->dirpos);
// reallocate to fit
size_t fullsiz = wad->dirpos+wad->nlumps*16;
data = realloc(data,fullsiz);
wad = (wad_t*)data;
if ( errno )
{
fclose(romfile);
free(data);
fprintf(stderr,"cannot allocate %zd bytes for full wad: %s",
fullsiz,strerror(errno));
return 32;
}
fread(data+12,1,fullsiz-12,romfile);
lumps = (lump_t*)(data+wad->dirpos);
// loop and fix endianness
for ( int i=0; i<wad->nlumps; i++ )
{
lumps[i].filepos = btl(lumps[i].filepos);
lumps[i].size = btl(lumps[i].size);
printf("%.8s %d %d\n",lumps[i].name,lumps[i].size,
lumps[i].filepos);
}
fclose(romfile);
FILE *wadfile = fopen(argv[2],"wb");
if ( !wadfile )
{
fprintf(stderr,"failed to open %s for writing: %s\n",
argv[2],strerror(errno));
return 4;
}
fwrite(data,1,fullsiz,wadfile);
fclose(wadfile);
free(data);
printf("wrote %s, %d bytes\n",argv[2],fullsiz);
return 0;
}

477
ufontext.c Normal file
View file

@ -0,0 +1,477 @@
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <unistd.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>
#define UPKG_MAGIC 0x9E2A83C1
// uncomment if you want full data dumps, helpful if you need to reverse engineer some unsupported format
//#define _DEBUG 1
typedef struct
{
uint32_t magic;
uint16_t pkgver, license;
uint32_t flags, nnames, onames, nexports, oexports, nimports, oimports;
} upkg_header_t;
uint8_t *pkgfile;
upkg_header_t *head;
size_t fpos = 0;
uint8_t readbyte( void )
{
uint8_t val = pkgfile[fpos];
fpos++;
return val;
}
uint16_t readword( void )
{
uint16_t val = *(uint16_t*)(pkgfile+fpos);
fpos += 2;
return val;
}
uint32_t readdword( void )
{
uint32_t val = *(uint32_t*)(pkgfile+fpos);
fpos += 4;
return val;
}
float readfloat( void )
{
float val = *(float*)(pkgfile+fpos);
fpos += 4;
return val;
}
#define READSTRUCT(x,y) {memcpy(&x,pkgfile+fpos,sizeof(y));fpos+=sizeof(y);}
// reads a compact index value
int32_t readindex( void )
{
uint8_t byte[5] = {0};
byte[0] = readbyte();
if ( !byte[0] ) return 0;
if ( byte[0]&0x40 )
{
for ( int i=1; i<5; i++ )
{
byte[i] = readbyte();
if ( !(byte[i]&0x80) ) break;
}
}
int32_t tf = byte[0]&0x3f;
tf |= (int32_t)(byte[1]&0x7f)<<6;
tf |= (int32_t)(byte[2]&0x7f)<<13;
tf |= (int32_t)(byte[3]&0x7f)<<20;
tf |= (int32_t)(byte[4]&0x7f)<<27;
if ( byte[0]&0x80 ) tf *= -1;
return tf;
}
// reads a name table entry
size_t readname( int *olen )
{
size_t pos = fpos;
if ( head->pkgver >= 64 )
{
int32_t len = readindex();
pos = fpos;
if ( olen ) *olen = len;
if ( len <= 0 ) return pos;
fpos += len;
}
else
{
int c, p = 0;
while ( (c = readbyte()) ) p++;
if ( olen ) *olen = p;
}
fpos += 4;
return pos;
}
size_t getname( int index, int *olen )
{
size_t prev = fpos;
fpos = head->onames;
size_t npos = 0;
for ( int i=0; i<=index; i++ )
npos = readname(olen);
fpos = prev;
return npos;
}
// checks if a name exists
int hasname( const char *needle )
{
if ( !needle ) return 0;
size_t prev = fpos;
fpos = head->onames;
int found = 0;
int nlen = strlen(needle);
for ( uint32_t i=0; i<head->nnames; i++ )
{
int32_t len = 0;
if ( head->pkgver >= 64 )
{
len = readindex();
if ( len <= 0 ) continue;
}
int c = 0, p = 0, match = 1;
while ( (c = readbyte()) )
{
if ( (p >= nlen) || (needle[p] != c) ) match = 0;
p++;
if ( len && (p > len) ) break;
}
if ( match )
{
found = 1;
break;
}
fpos += 4;
}
fpos = prev;
return found;
}
// read import table entry and return index of its object name
int32_t readimport( void )
{
readindex();
readindex();
if ( head->pkgver >= 60 ) fpos += 4;
else readindex();
return readindex();
}
int32_t getimport( int index )
{
size_t prev = fpos;
fpos = head->oimports;
int32_t iname = 0;
for ( int i=0; i<=index; i++ )
iname = readimport();
fpos = prev;
return iname;
}
// fully read import table entry
void readimport2( int32_t *cpkg, int32_t *cname, int32_t *pkg, int32_t *name )
{
*cpkg = readindex();
*cname = readindex();
if ( head->pkgver >= 60 ) *pkg = readdword();
else *pkg = readindex();
*name = readindex();
}
void getimport2( int index, int32_t *cpkg, int32_t *cname, int32_t *pkg,
int32_t *name )
{
size_t prev = fpos;
fpos = head->oimports;
for ( int i=0; i<=index; i++ )
readimport2(cpkg,cname,pkg,name);
fpos = prev;
}
// read export table entry
void readexport( int32_t *class, int32_t *ofs, int32_t *siz, int32_t *name )
{
*class = readindex();
readindex();
if ( head->pkgver >= 60 ) fpos += 4;
*name = readindex();
fpos += 4;
*siz = readindex();
if ( *siz > 0 ) *ofs = readindex();
}
void getexport( int index, int32_t *class, int32_t *ofs, int32_t *siz,
int32_t *name )
{
size_t prev = fpos;
fpos = head->oexports;
for ( int i=0; i<=index; i++ )
readexport(class,ofs,siz,name);
fpos = prev;
}
// fully read export table entry
void readexport2( int32_t *class, int32_t *super, int32_t *pkg, int32_t *name,
uint32_t *flags, int32_t *siz, int32_t *ofs )
{
*class = readindex();
*super = readindex();
if ( head->pkgver >= 60 ) *pkg = readdword();
else *pkg = readindex();
*name = readindex();
*flags = readdword();
*siz = readindex();
if ( *siz > 0 ) *ofs = readindex();
}
void getexport2( int index, int32_t *class, int32_t *super, int32_t *pkg,
int32_t *name, uint32_t *flags, int32_t *siz, int32_t *ofs )
{
size_t prev = fpos;
fpos = head->oexports;
for ( int i=0; i<=index; i++ )
readexport2(class,super,pkg,name,flags,siz,ofs);
fpos = prev;
}
// construct full name for object
// shamelessly recycled from my old upackage project
void imprefix( FILE *f, int32_t i );
void exprefix( FILE *f, int32_t i );
void imprefix( FILE *f, int32_t i )
{
int32_t cpkg, cnam, pkg, nam;
getimport2(i,&cpkg,&cnam,&pkg,&nam);
if ( pkg < 0 ) imprefix(f,-pkg-1);
else if ( pkg > 0 ) exprefix(f,pkg-1);
if ( pkg ) fprintf(f,".");
int32_t l;
char *pname = (char*)(pkgfile+getname(nam,&l));
fprintf(f,"%.*s",l,pname);
}
void exprefix( FILE *f, int32_t i )
{
int32_t cls, sup, pkg, nam, siz, ofs;
uint32_t fl;
getexport2(i,&cls,&sup,&pkg,&nam,&fl,&siz,&ofs);
if ( pkg > 0 )
{
exprefix(f,pkg-1);
fprintf(f,".");
}
int32_t l;
char *pname = (char*)(pkgfile+getname(nam,&l));
fprintf(f,"%.*s",l,pname);
}
void construct_fullname( FILE *f, int32_t i )
{
if ( i > 0 ) exprefix(f,i-1);
else if ( i < 0 ) imprefix(f,-i-1);
else fprintf(f,"None");
}
typedef struct
{
int32_t x, y, w, h;
} ufontchar_t;
typedef struct
{
int32_t texture;
uint32_t charcount;
ufontchar_t *chars;
} ufonttex_t;
typedef struct
{
uint8_t texcount;
ufonttex_t *tex;
} ufonthdr_t;
void savefont( int32_t namelen, char *name )
{
char fname[256] = {0};
FILE *f;
if ( head->license == 2 )
{
printf(" FAIL: Postal 2 fonts not yet supported\n");
return;
}
ufonthdr_t fhead;
memset(&fhead,0,sizeof(ufonthdr_t));
fhead.texcount = readbyte();
fhead.tex = calloc(fhead.texcount,sizeof(ufonttex_t));
for ( int i=0; i<fhead.texcount; i++ )
{
fhead.tex[i].texture = readindex();
fhead.tex[i].charcount = readindex();
fhead.tex[i].chars = calloc(fhead.tex[i].charcount,
sizeof(ufontchar_t));
for ( int j=0; j<fhead.tex[i].charcount; j++ )
{
fhead.tex[i].chars[j].x = readdword();
fhead.tex[i].chars[j].y = readdword();
fhead.tex[i].chars[j].w = readdword();
fhead.tex[i].chars[j].h = readdword();
}
}
// save to text
snprintf(fname,256,"%.*s.txt",namelen,name);
f = fopen(fname,"w");
printf(" Dumping Font to %s in UTPT format\n",fname);
fprintf(f,"Character: TextureName (X,Y)-(Width,Height)\n"
"-------------------------------------------\n");
int cc = 0;
for ( int i=0; i<fhead.texcount; i++ )
for ( int j=0; j<fhead.tex[i].charcount; j++ )
{
fprintf(f,"0x%02x: ",cc);
construct_fullname(f,fhead.tex[i].texture);
fprintf(f," (%d,%d)-(%d,%d)\n",fhead.tex[i].chars[j].x,
fhead.tex[i].chars[j].y,fhead.tex[i].chars[j].w,
fhead.tex[i].chars[j].h);
cc++;
}
fclose(f);
/*
TODO also fetch the textures referenced and extract the characters
from them into a folder (removing the need to use mkfont)
*/
// cleanup
for ( int i=0; i<fhead.texcount; i++ ) free(fhead.tex[i].chars);
free(fhead.tex);
}
int main( int argc, char **argv )
{
if ( argc < 2 )
{
printf("Usage: ufontext <archive>\n");
return 1;
}
int fd = open(argv[1],O_RDONLY);
if ( fd == -1 )
{
printf("Failed to open file %s: %s\n",argv[1],strerror(errno));
return 1;
}
struct stat st;
fstat(fd,&st);
pkgfile = malloc(st.st_size);
memset(pkgfile,0,st.st_size);
head = (upkg_header_t*)pkgfile;
int r = 0;
do
{
r = read(fd,pkgfile+fpos,131072);
if ( r == -1 )
{
close(fd);
free(pkgfile);
printf("Read failed for file %s: %s\n",argv[1],
strerror(errno));
return 4;
}
fpos += r;
}
while ( r > 0 );
close(fd);
fpos = 0;
if ( head->magic != UPKG_MAGIC )
{
printf("File %s is not a valid unreal package!\n",argv[1]);
free(pkgfile);
return 2;
}
if ( !hasname("Font") )
{
printf("Package %s does not contain Fonts\n",argv[1]);
free(pkgfile);
return 4;
}
// loop through exports and search for fonts
fpos = head->oexports;
for ( uint32_t i=0; i<head->nexports; i++ )
{
int32_t class, ofs, siz, name;
readexport(&class,&ofs,&siz,&name);
if ( (siz <= 0) || (class >= 0) ) continue;
// get the class name
class = -class-1;
if ( (uint32_t)class > head->nimports ) continue;
int32_t l = 0;
char *n = (char*)(pkgfile+getname(getimport(class),&l));
if ( strncmp(n,"Font",l) ) continue;
char *fnt = (char*)(pkgfile+getname(name,&l));
printf("Font found: %.*s\n",l,fnt);
int32_t fntl = l;
#ifdef _DEBUG
char fname[256] = {0};
snprintf(fname,256,"%.*s.object",fntl,fnt);
printf(" Dumping full object data to %s\n",fname);
FILE *f = fopen(fname,"wb");
fwrite(pkgfile+ofs,siz,1,f);
fclose(f);
#endif
// begin reading data
size_t prev = fpos;
fpos = ofs;
if ( head->pkgver < 40 ) fpos += 8;
if ( head->pkgver < 60 ) fpos += 16;
int32_t prop = readindex();
if ( (uint32_t)prop >= head->nnames )
{
printf("Unknown property %d, skipping\n",prop);
fpos = prev;
continue;
}
char *pname = (char*)(pkgfile+getname(prop,&l));
retry:
if ( strncasecmp(pname,"none",l) )
{
uint8_t info = readbyte();
int psiz = (info>>4)&0x7;
switch ( psiz )
{
case 0:
psiz = 1;
break;
case 1:
psiz = 2;
break;
case 2:
psiz = 4;
break;
case 3:
psiz = 12;
break;
case 4:
psiz = 16;
break;
case 5:
psiz = readbyte();
break;
case 6:
psiz = readword();
break;
case 7:
psiz = readdword();
break;
}
fpos += psiz;
printf(" Skipping property %.*s\n",l,pname);
prop = readindex();
pname = (char*)(pkgfile+getname(prop,&l));
goto retry;
}
#ifdef _DEBUG
snprintf(fname,256,"%.*s.ufnt",fntl,fnt);
printf(" Dumping full font struct to %s\n",fname);
f = fopen(fname,"wb");
fwrite(pkgfile+fpos,siz-(fpos-ofs),1,f);
fclose(f);
#endif
savefont(fntl,fnt);
fpos = prev;
}
free(pkgfile);
return 0;
}