OrdinaryMagician/vkdoom_m
OrdinaryMagician/vkdoom_m
1
Fork 0
Code Activity

Merge remote-tracking branch 'gzdoom/staging'

Browse source
This commit is contained in:
Magnus Norddahl 2024-01-02 17:10:47 +01:00
commit 73ada7f84b
214 changed files with 23523 additions and 1206 deletions
Download patch file Download diff file Expand all files Collapse all files

2
CMakeLists.txt
View file

@ -301,7 +301,7 @@ else()
if ( UNIX )
include(CheckSymbolExists)
check_symbol_exists( "fts_set" "fts.h" HAVE_FTS )
check_symbol_exists( "fts_set" "sys/types.h;sys/stat.h;fts.h" HAVE_FTS )
if ( NOT HAVE_FTS )
include ( FindPkgConfig )
pkg_check_modules( MUSL_FTS musl-fts )

40
specs/udmf_zdoom.txt
View file

@ -238,6 +238,16 @@ Note: All <bool> fields default to false unless mentioned otherwise.
skew_middle = <int>; // Vertical texture alignment defines the position at the leftmost point of the wall.
skew_bottom = <int>; // Possible values: 0 = no skewing, 1 = align to front floor, 2 = align to front ceiling, 3 = align to back floor, 4 = align to back ceiling.
// Front/Back are relative to the sidedef, not the owning linedef. Default = 0.
xscroll = <float>; // wall scrolling X speed in map units per tic.
yscroll = <float>; // wall scrolling Y speed in map units per tic.
xscrolltop = <float>; // upper wall scrolling X speed in map units per tic.
yscrolltop = <float>; // upper wall scrolling Y speed in map units per tic.
xscrollmid = <float>; // mid wall scrolling X speed in map units per tic.
yscrollmid = <float>; // mid wall scrolling Y speed in map units per tic.
xscrollbottom = <float>; // lower wall scrolling X speed in map units per tic.
yscrollbottom = <float>; // lower wall scrolling Y speed in map units per tic.
}
@ -337,6 +347,36 @@ Note: All <bool> fields default to false unless mentioned otherwise.
healthfloorgroup = <int>; // ID of destructible object to synchronize hitpoints (optional, default is 0)
healthceiling = <int>; // Amount of hitpoints for this sector (includes ceiling and top-outside linedef sides)
healthceilinggroup = <int>; // ID of destructible object to synchronize hitpoints (optional, default is 0)
xscrollfloor = <float>; // X map units per frame to scroll the floor.
yscrollfloor = <float>; // Y map units per frame to scroll the floor.
scrollfloormode = <int>; // Floor scroll mode bit mask (1 = scroll textures, 2 = carry static objects, 4 = carry players, 8 = carry monsters.
xscrollceiling = <float>; // X map units per frame to scroll the ceiling.
yscrollceiling = <float>; // Y map units per frame to scroll the ceiling.
scrollceilingmode = <int>; // ceiling scroll mode bit mask (1 = scroll textures, 2 = carry static objects, 4 = carry players, 8 = carry monsters.
scroll_ceil_x = <float>; // deprecated Eternity based alternatives for the above.
scroll_ceil_y = <float>; // Due to using unintuitive units of measurement and a more limited feature set they should not be used anymore.
scroll_ceil_type = <string>;
scroll_floor_x = <float>;
scroll_floor_y = <float>;
scroll_floor_type = <string>;
friction = <float>; // sets the sector's friction factor. Must be between 0 and 1.
movefactor = <float> // sets the sector's movement acceleration factor. Must be > 0.
skyfloor = <string> // defines lower sky for this sector.
skyceiling = <string> // defines upper sky for this sector.
skyfloor2 = <string> // defines secondary lower sky for this sector. (for lightning or transparent layers)
skyceiling2 = <string> // defines secondary upper sky for this sector.
colormap = <string>; // only provided for backwards compatibility. Do not use in GZDoom projects.
xthrust = <float>; // applies thrust to actors - x-magnitude
ythrust = <float>; // applies thrust to actors - y-magnitude
thrustgroup = <int>; // specifies which actors get thrusted. Bitfield with (1 = static objects, 2 = player, 4 = monsters, 8 = projectiles, 16 = actors with +WINDTHRUST)
thrustlocation = <int>; // specifies where in the sector actors get thrusted: (1 = on the ground, 2 = in the air, 4 = on the ceiling)
* Note about dropactors

6
src/CMakeLists.txt
View file

@ -597,6 +597,7 @@ file( GLOB HEADER_FILES
common/thirdparty/rapidjson/*.h
common/thirdparty/math/*h
common/thirdparty/libsmackerdec/include/*.h
common/thirdparty/utf8proc/*.h
common/rendering/*.h
common/rendering/hwrenderer/data/*.h
common/rendering/vulkan/*.h
@ -857,6 +858,7 @@ set (PCH_SOURCES
playsim/mapthinkers/a_pillar.cpp
playsim/mapthinkers/a_plats.cpp
playsim/mapthinkers/a_pusher.cpp
playsim/mapthinkers/a_thruster.cpp
playsim/mapthinkers/a_scroll.cpp
playsim/mapthinkers/dsectoreffect.cpp
playsim/a_pickups.cpp
@ -1162,6 +1164,7 @@ set( GAME_SOURCES
${PCH_SOURCES}
common/utility/x86.cpp
common/thirdparty/strnatcmp.c
common/thirdparty/utf8proc/utf8proc.c
common/thirdparty/stb/stb_sprintf.c
common/utility/zstring.cpp
common/utility/findfile.cpp
@ -1202,6 +1205,7 @@ set( GAME_SOURCES
common/filesystem/source/files_decompress.cpp
common/filesystem/source/fs_findfile.cpp
common/filesystem/source/fs_stringpool.cpp
common/filesystem/source/unicode.cpp
)
@ -1243,6 +1247,7 @@ include_directories(
common/thirdparty/libsmackerdec/include
common/thirdparty
common/thirdparty/stb
common/thirdparty/utf8proc
common/textures
common/textures/formats
common/textures/hires
@ -1488,6 +1493,7 @@ source_group("Common\\Third Party\\Math" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SO
source_group("Common\\Third Party\\RapidJSON" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/common/thirdparty/rapidjson/.+")
source_group("Common\\Third Party\\SFMT" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/common/thirdparty/sfmt/.+")
source_group("Common\\Third Party\\stb" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/common/thirdparty/stb/.+")
source_group("Common\\Third Party\\utf8proc" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/common/thirdparty/utf8proc/.+")
source_group("Utility" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/utility/.+")
source_group("Utility\\Node Builder" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/utility/nodebuilder/.+")
source_group("Utility\\Smackerdec" REGULAR_EXPRESSION "^${CMAKE_CURRENT_SOURCE_DIR}/smackerdec/.+")

8
src/am_map.cpp
View file

@ -996,7 +996,7 @@ class DAutomap :public DAutomapBase
void calcMinMaxMtoF();
void DrawMarker(FGameTexture *tex, double x, double y, int yadjust,
INTBOOL flip, double xscale, double yscale, int translation, double alpha, uint32_t fillcolor, FRenderStyle renderstyle);
INTBOOL flip, double xscale, double yscale, FTranslationID translation, double alpha, uint32_t fillcolor, FRenderStyle renderstyle);
void rotatePoint(double *x, double *y);
void rotate(double *x, double *y, DAngle an);
@ -3080,7 +3080,7 @@ void DAutomap::drawThings ()
//=============================================================================
void DAutomap::DrawMarker (FGameTexture *tex, double x, double y, int yadjust,
INTBOOL flip, double xscale, double yscale, int translation, double alpha, uint32_t fillcolor, FRenderStyle renderstyle)
INTBOOL flip, double xscale, double yscale, FTranslationID translation, double alpha, uint32_t fillcolor, FRenderStyle renderstyle)
{
if (tex == nullptr || !tex->isValid())
{
@ -3103,7 +3103,7 @@ void DAutomap::DrawMarker (FGameTexture *tex, double x, double y, int yadjust,
DTA_ClipLeft, f_x,
DTA_ClipRight, f_x + f_w,
DTA_FlipX, flip,
DTA_TranslationIndex, translation,
DTA_TranslationIndex, translation.index(),
DTA_Alpha, alpha,
DTA_FillColor, fillcolor,
DTA_RenderStyle, renderstyle.AsDWORD,
@ -3134,7 +3134,7 @@ void DAutomap::drawMarks ()
if (font == nullptr)
{
DrawMarker(TexMan.GetGameTexture(marknums[i], true), markpoints[i].x, markpoints[i].y, -3, 0,
1, 1, 0, 1, 0, LegacyRenderStyles[STYLE_Normal]);
1, 1, NO_TRANSLATION, 1, 0, LegacyRenderStyles[STYLE_Normal]);
}
else
{

10
src/common/2d/v_2ddrawer.cpp
View file

@ -443,7 +443,7 @@ void F2DDrawer::AddTexture(FGameTexture* img, DrawParms& parms)
if (img->isWarped()) dg.mFlags |= DTF_Wrap;
if (parms.indexed) dg.mFlags |= DTF_Indexed;
dg.mTranslationId = 0;
dg.mTranslationId = NO_TRANSLATION;
SetStyle(img, parms, vertexcolor, dg);
if (parms.indexed)
{
@ -451,7 +451,7 @@ void F2DDrawer::AddTexture(FGameTexture* img, DrawParms& parms)
vertexcolor = 0xffffffff;
}
if (!img->isHardwareCanvas() && parms.TranslationId != -1)
if (!img->isHardwareCanvas() && parms.TranslationId != INVALID_TRANSLATION)
{
dg.mTranslationId = parms.TranslationId;
}
@ -607,7 +607,7 @@ void F2DDrawer::AddShape(FGameTexture* img, DShape2D* shape, DrawParms& parms)
dg.mFlags |= DTF_Wrap;
dg.mTexture = img;
dg.mTranslationId = 0;
dg.mTranslationId = NO_TRANSLATION;
SetStyle(img, parms, vertexcolor, dg);
if (shape->lastParms == nullptr) {
@ -624,7 +624,7 @@ void F2DDrawer::AddShape(FGameTexture* img, DShape2D* shape, DrawParms& parms)
shape->lastParms = new DrawParms(parms);
}
if (!(img != nullptr && img->isHardwareCanvas()) && parms.TranslationId != -1)
if (!(img != nullptr && img->isHardwareCanvas()) && parms.TranslationId != INVALID_TRANSLATION)
dg.mTranslationId = parms.TranslationId;
auto osave = offset;
@ -796,7 +796,7 @@ void F2DDrawer::AddPoly(FGameTexture *texture, FVector2 *points, int npoints,
//
//==========================================================================
void F2DDrawer::AddPoly(FGameTexture* img, FVector4* vt, size_t vtcount, const unsigned int* ind, size_t idxcount, int translation, PalEntry color, FRenderStyle style, const IntRect* clip)
void F2DDrawer::AddPoly(FGameTexture* img, FVector4* vt, size_t vtcount, const unsigned int* ind, size_t idxcount, FTranslationID translation, PalEntry color, FRenderStyle style, const IntRect* clip)
{
RenderCommand dg;

4
src/common/2d/v_2ddrawer.h
View file

@ -133,7 +133,7 @@ public:
int mIndexCount;
FGameTexture *mTexture;
int mTranslationId;
FTranslationID mTranslationId;
PalEntry mSpecialColormap[2];
int mScissor[4];
int mDesaturate;
@ -218,7 +218,7 @@ public:
void AddPoly(FGameTexture *texture, FVector2 *points, int npoints,
double originx, double originy, double scalex, double scaley,
DAngle rotation, const FColormap &colormap, PalEntry flatcolor, double lightlevel, uint32_t *indices, size_t indexcount);
void AddPoly(FGameTexture* img, FVector4 *vt, size_t vtcount, const unsigned int *ind, size_t idxcount, int translation, PalEntry color, FRenderStyle style, const IntRect* clip);
void AddPoly(FGameTexture* img, FVector4 *vt, size_t vtcount, const unsigned int *ind, size_t idxcount, FTranslationID translation, PalEntry color, FRenderStyle style, const IntRect* clip);
void FillPolygon(int* rx1, int* ry1, int* xb1, int32_t npoints, int pic, int palette, int shade, int props, const FVector2& xtex, const FVector2& ytex, const FVector2& otex,
int clipx1, int clipy1, int clipx2, int clipy2);
void AddFlatFill(int left, int top, int right, int bottom, FGameTexture *src, int local_origin = false, double flatscale = 1.0, PalEntry color = 0xffffffff, ERenderStyle rs = STYLE_Normal);

4
src/common/2d/v_draw.cpp
View file

@ -866,7 +866,7 @@ bool ParseDrawTextureTags(F2DDrawer *drawer, FGameTexture *img, double x, double
parms->destheight = INT_MAX;
parms->Alpha = type == DrawTexture_Fill ? (float)fillalpha : 1.f;
parms->fillcolor = type == DrawTexture_Fill ? fill : PalEntry(~0u);
parms->TranslationId = -1;
parms->TranslationId = INVALID_TRANSLATION;
parms->colorOverlay = 0;
parms->alphaChannel = false;
parms->flipX = false;
@ -1090,7 +1090,7 @@ bool ParseDrawTextureTags(F2DDrawer *drawer, FGameTexture *img, double x, double
break;
case DTA_TranslationIndex:
parms->TranslationId = ListGetInt(tags);
parms->TranslationId = FTranslationID::fromInt(ListGetInt(tags));
break;
case DTA_ColorOverlay:

2
src/common/2d/v_draw.h
View file

@ -176,7 +176,7 @@ struct DrawParms
double left;
float Alpha;
PalEntry fillcolor;
int TranslationId;
FTranslationID TranslationId;
PalEntry colorOverlay;
PalEntry color;
int alphaChannel;

10
src/common/2d/v_drawtext.cpp
View file

@ -185,7 +185,7 @@ void DrawChar(F2DDrawer *drawer, FFont* font, int normalcolor, double x, double
{
return;
}
bool palettetrans = (normalcolor == CR_NATIVEPAL && parms.TranslationId != 0);
bool palettetrans = (normalcolor == CR_NATIVEPAL && parms.TranslationId != NO_TRANSLATION);
PalEntry color = 0xffffffff;
if (!palettetrans) parms.TranslationId = font->GetColorTranslation((EColorRange)normalcolor, &color);
parms.color = PalEntry((color.a * parms.color.a) / 255, (color.r * parms.color.r) / 255, (color.g * parms.color.g) / 255, (color.b * parms.color.b) / 255);
@ -210,7 +210,7 @@ void DrawChar(F2DDrawer *drawer, FFont *font, int normalcolor, double x, double
uint32_t tag = ListGetInt(args);
bool res = ParseDrawTextureTags(drawer, pic, x, y, tag, args, &parms, DrawTexture_Normal);
if (!res) return;
bool palettetrans = (normalcolor == CR_NATIVEPAL && parms.TranslationId != 0);
bool palettetrans = (normalcolor == CR_NATIVEPAL && parms.TranslationId != NO_TRANSLATION);
PalEntry color = 0xffffffff;
if (!palettetrans) parms.TranslationId = font->GetColorTranslation((EColorRange)normalcolor, &color);
parms.color = PalEntry((color.a * parms.color.a) / 255, (color.r * parms.color.r) / 255, (color.g * parms.color.g) / 255, (color.b * parms.color.b) / 255);
@ -272,7 +272,7 @@ void DrawTextCommon(F2DDrawer *drawer, FFont *font, int normalcolor, double x, d
double cx;
double cy;
int boldcolor;
int trans = -1;
FTranslationID trans = INVALID_TRANSLATION;
int kerning;
FGameTexture *pic;
@ -282,7 +282,7 @@ void DrawTextCommon(F2DDrawer *drawer, FFont *font, int normalcolor, double x, d
if (parms.celly == 0) parms.celly = font->GetHeight() + 1;
parms.celly = int (parms.celly * scaley);
bool palettetrans = (normalcolor == CR_NATIVEPAL && parms.TranslationId != 0);
bool palettetrans = (normalcolor == CR_NATIVEPAL && parms.TranslationId != NO_TRANSLATION);
if (normalcolor >= NumTextColors)
normalcolor = CR_UNTRANSLATED;
@ -290,7 +290,7 @@ void DrawTextCommon(F2DDrawer *drawer, FFont *font, int normalcolor, double x, d
PalEntry colorparm = parms.color;
PalEntry color = 0xffffffff;
trans = palettetrans? -1 : font->GetColorTranslation((EColorRange)normalcolor, &color);
trans = palettetrans? INVALID_TRANSLATION : font->GetColorTranslation((EColorRange)normalcolor, &color);
parms.color = PalEntry(colorparm.a, (color.r * colorparm.r) / 255, (color.g * colorparm.g) / 255, (color.b * colorparm.b) / 255);
kerning = font->GetDefaultKerning();

2
src/common/audio/music/i_music.cpp
View file

@ -337,7 +337,7 @@ static ZMusic_MidiSource GetMIDISource(const char *fn)
}
auto data = wlump.Read();
auto source = ZMusic_CreateMIDISource(data.data(), data.size(), type);
auto source = ZMusic_CreateMIDISource(data.bytes(), data.size(), type);
if (source == nullptr)
{

12
src/common/audio/music/i_soundfont.cpp
View file

@ -215,10 +215,10 @@ FileReader FZipPatReader::OpenFile(const char *name)
FileReader fr;
if (resf != nullptr)
{
auto lump = resf->FindLump(name);
if (lump != nullptr)
auto lump = resf->FindEntry(name);
if (lump >= 0)
{
return lump->NewReader();
return resf->GetEntryReader(lump);
}
}
fr.OpenFile(name);
@ -367,10 +367,10 @@ void FSoundFontManager::ProcessOneFile(const char* fn)
auto zip = FResourceFile::OpenResourceFile(fn, true);
if (zip != nullptr)
{
if (zip->LumpCount() > 1) // Anything with just one lump cannot possibly be a packed GUS patch set so skip it right away and simplify the lookup code
if (zip->EntryCount() > 1) // Anything with just one lump cannot possibly be a packed GUS patch set so skip it right away and simplify the lookup code
{
auto zipl = zip->FindLump("timidity.cfg");
if (zipl != nullptr)
auto zipl = zip->FindEntry("timidity.cfg");
if (zipl >= 0)
{
// It seems like this is what we are looking for
FSoundFontInfo sft = { fb, fbe, fn, SF_GUS };

2
src/common/audio/sound/oalsound.cpp
View file

@ -1141,7 +1141,7 @@ SoundHandle OpenALSoundRenderer::LoadSound(uint8_t *sfxdata, int length, int def
return retval;
}
std::vector<uint8_t> data;
TArray<uint8_t> data;
unsigned total = 0;
unsigned got;

2
src/common/audio/sound/s_soundinternal.h
View file

@ -216,7 +216,7 @@ private:
// Checks if a copy of this sound is already playing.
bool CheckSingular(FSoundID sound_id);
virtual std::vector<uint8_t> ReadSound(int lumpnum) = 0;
virtual TArray<uint8_t> ReadSound(int lumpnum) = 0;
protected:
virtual bool CheckSoundLimit(sfxinfo_t* sfx, const FVector3& pos, int near_limit, float limit_range, int sourcetype, const void* actor, int channel, float attenuation);

57
src/common/console/c_cvars.cpp
View file

@ -44,6 +44,7 @@
#include "printf.h"
#include "palutil.h"
#include "i_interface.h"
#include "gstrings.h"
#include "dobject.h"
#include "dobjtype.h"
@ -1695,16 +1696,37 @@ CCMD (toggle)
}
}
void FBaseCVar::ListVars (const char *filter, bool plain)
void FBaseCVar::ListVars (const char *filter, int listtype)
{
int count = 0;
bool plain = listtype == LCT_Plain;
bool includedesc = listtype == LCT_FullSearch;
decltype(cvarMap)::Iterator it(cvarMap);
decltype(cvarMap)::Pair *pair;
while (it.NextPair(pair))
{
auto var = pair->Value;
if (CheckWildcards (filter, var->GetName()))
bool ismatch;
if (filter && includedesc)
{
// search always allow partial matches
// also allow matching to cvar name, localised description, and description language-id
FString SearchString = FString("*") + filter + "*";
ismatch = CheckWildcards (SearchString.GetChars(), var->GetName()) ||
CheckWildcards (SearchString.GetChars(), var->GetDescription().GetChars()) ||
CheckWildcards (SearchString.GetChars(), GStrings.localize(var->GetDescription().GetChars()));
}
else
{
ismatch = CheckWildcards (filter, var->GetName());
}
if (ismatch)
{
uint32_t flags = var->GetFlags();
if (plain)
@ -1718,7 +1740,8 @@ void FBaseCVar::ListVars (const char *filter, bool plain)
else
{
++count;
Printf ("%c%c%c%c%c %s = %s\n",
Printf ("%c%c%c%c%c %s = %s",
flags & CVAR_ARCHIVE ? 'A' : ' ',
flags & CVAR_USERINFO ? 'U' :
flags & CVAR_SERVERINFO ? 'S' :
@ -1730,6 +1753,16 @@ void FBaseCVar::ListVars (const char *filter, bool plain)
flags & CVAR_IGNORE ? 'X' : ' ',
var->GetName(),
var->GetHumanString());
if (includedesc)
if (var->GetDescription().Len())
Printf(" // \"%s\"\n", GStrings.localize(var->GetDescription().GetChars()));
else
Printf("\n");
else
Printf("\n");
}
}
}
@ -1740,17 +1773,29 @@ CCMD (cvarlist)
{
if (argv.argc() == 1)
{
FBaseCVar::ListVars (NULL, false);
FBaseCVar::ListVars (NULL, LCT_Default);
}
else
{
FBaseCVar::ListVars (argv[1], false);
FBaseCVar::ListVars (argv[1], LCT_Default);
}
}
CCMD (cvarlistplain)
{
FBaseCVar::ListVars (NULL, true);
FBaseCVar::ListVars (NULL, LCT_Plain);
}
CCMD (cvarsearch)
{
if (argv.argc() == 1)
{
FBaseCVar::ListVars (NULL, LCT_FullSearch);
}
else
{
FBaseCVar::ListVars (argv[1], LCT_FullSearch);
}
}
CCMD (archivecvar)

12
src/common/console/c_cvars.h
View file

@ -89,6 +89,13 @@ enum ECVarType
CVAR_Dummy, // Unknown
};
enum ListCCMDType
{
LCT_Default,
LCT_Plain,
LCT_FullSearch,
};
class FIntCVarRef;
union UCVarValue
@ -201,7 +208,7 @@ public:
static void MarkZSCallbacks ();
static void ResetColors (); // recalc color cvars' indices after screen change
static void ListVars (const char *filter, bool plain);
static void ListVars (const char *filter, int listtype);
const FString &GetDescription() const { return Description; };
const FString& GetToggleMessage(int which) { return ToggleMessages[which]; }
@ -218,6 +225,9 @@ public:
void* GetExtraDataPointer();
int pnum = -1;
FName userinfoName;
protected:
virtual void DoSet (UCVarValue value, ECVarType type) = 0;
virtual void InstantiateZSCVar()

5
src/common/cutscenes/movieplayer.cpp
View file

@ -153,7 +153,7 @@ class AnmPlayer : public MoviePlayer
{
// This doesn't need its own class type
anim_t anim;
std::vector<uint8_t> buffer;
FileSys::ResourceData buffer;
int numframes = 0;
int curframe = 1;
int frametime = 0;
@ -171,9 +171,10 @@ public:
memcpy(frameTicks, frameticks, 3 * sizeof(int));
flags = flags_;
buffer = fr.ReadPadded(1);
if (buffer.size() < 4) return;
fr.Close();
if (ANIM_LoadAnim(&anim, buffer.data(), buffer.size() - 1) < 0)
if (ANIM_LoadAnim(&anim, buffer.bytes(), buffer.size() - 1) < 0)
{
return;
}

2
src/common/engine/i_interface.h
View file

@ -10,6 +10,7 @@ class FGameTexture;
class FTextureID;
enum EUpscaleFlags : int;
class FConfigFile;
struct FTranslationID;
struct SystemCallbacks
{
@ -47,6 +48,7 @@ struct SystemCallbacks
bool (*OkForLocalization)(FTextureID, const char*);
FConfigFile* (*GetConfig)();
bool (*WantEscape)();
FTranslationID(*RemapTranslation)(FTranslationID trans);
};
extern SystemCallbacks sysCallbacks;

7
src/common/engine/namedef.h
View file

@ -10,6 +10,7 @@ xx(Object)
xx(Actor)
xx(Class)
xx(Thinker)
xx(VisualThinker)
xx(Crosshairs)
xx(Untranslated)
@ -134,6 +135,10 @@ xx(FVector3)
xx(FVector4)
xx(FQuat)
xx(let)
xx(BlockThingsIterator)
xx(BlockLinesIterator)
xx(ActorIterator)
xx(ThinkerIterator)
xx(Min)
xx(Max)
@ -182,6 +187,7 @@ xx(Voidptr)
xx(StateLabel)
xx(SpriteID)
xx(TextureID)
xx(TranslationID)
xx(Overlay)
xx(IsValid)
xx(IsNull)
@ -272,6 +278,7 @@ xx(BuiltinFRandom)
xx(BuiltinNameToClass)
xx(BuiltinClassCast)
xx(BuiltinFunctionPtrCast)
xx(BuiltinFindTranslation)
xx(ScreenJobRunner)
xx(Action)

15
src/common/engine/palettecontainer.cpp
View file

@ -220,7 +220,7 @@ FRemapTable* PaletteContainer::AddRemap(FRemapTable* remap)
//
//----------------------------------------------------------------------------
void PaletteContainer::UpdateTranslation(int trans, FRemapTable* remap)
void PaletteContainer::UpdateTranslation(FTranslationID trans, FRemapTable* remap)
{
auto newremap = AddRemap(remap);
TranslationTables[GetTranslationType(trans)].SetVal(GetTranslationIndex(trans), newremap);
@ -232,7 +232,7 @@ void PaletteContainer::UpdateTranslation(int trans, FRemapTable* remap)
//
//----------------------------------------------------------------------------
int PaletteContainer::AddTranslation(int slot, FRemapTable* remap, int count)
FTranslationID PaletteContainer::AddTranslation(int slot, FRemapTable* remap, int count)
{
uint32_t id = 0;
for (int i = 0; i < count; i++)
@ -249,9 +249,9 @@ int PaletteContainer::AddTranslation(int slot, FRemapTable* remap, int count)
//
//----------------------------------------------------------------------------
void PaletteContainer::CopyTranslation(int dest, int src)
void PaletteContainer::CopyTranslation(FTranslationID dest, FTranslationID src)
{
TranslationTables[GetTranslationType(dest)].SetVal(GetTranslationIndex(dest), TranslationToTable(src));
TranslationTables[GetTranslationType(dest)].SetVal(GetTranslationIndex(dest), TranslationToTable(src.index()));
}
//----------------------------------------------------------------------------
@ -260,8 +260,9 @@ void PaletteContainer::CopyTranslation(int dest, int src)
//
//----------------------------------------------------------------------------
FRemapTable *PaletteContainer::TranslationToTable(int translation)
FRemapTable *PaletteContainer::TranslationToTable(int translation) const
{
if (IsLuminosityTranslation(translation)) return nullptr;
unsigned int type = GetTranslationType(translation);
unsigned int index = GetTranslationIndex(translation);
@ -278,14 +279,14 @@ FRemapTable *PaletteContainer::TranslationToTable(int translation)
//
//----------------------------------------------------------------------------
int PaletteContainer::StoreTranslation(int slot, FRemapTable *remap)
FTranslationID PaletteContainer::StoreTranslation(int slot, FRemapTable *remap)
{
unsigned int i;
auto size = NumTranslations(slot);
for (i = 0; i < size; i++)
{
if (*remap == *TranslationToTable(TRANSLATION(slot, i)))
if (*remap == *TranslationToTable(TRANSLATION(slot, i).index()))
{
// A duplicate of this translation already exists
return TRANSLATION(slot, i);

111
src/common/engine/palettecontainer.h
View file

@ -42,6 +42,49 @@ struct FRemapTable
private:
};
// outside facing translation ID
struct FTranslationID
{
public:
FTranslationID() = default;
private:
constexpr FTranslationID(int id) : ID(id)
{
}
public:
static constexpr FTranslationID fromInt(int i)
{
return FTranslationID(i);
}
FTranslationID(const FTranslationID& other) = default;
FTranslationID& operator=(const FTranslationID& other) = default;
bool operator !=(FTranslationID other) const
{
return ID != other.ID;
}
bool operator ==(FTranslationID other) const
{
return ID == other.ID;
}
constexpr int index() const
{
return ID;
}
constexpr bool isvalid() const
{
return ID >= 0;
}
private:
int ID;
};
constexpr FTranslationID NO_TRANSLATION = FTranslationID::fromInt(0);
constexpr FTranslationID INVALID_TRANSLATION = FTranslationID::fromInt(-1);
// A class that initializes unusued pointers to NULL. This is used so that when
// the TAutoGrowArray below is expanded, the new elements will be NULLed.
class FRemapTablePtr
@ -67,14 +110,19 @@ enum
TRANSLATIONTYPE_MASK = (255 << TRANSLATION_SHIFT)
};
inline constexpr uint32_t TRANSLATION(uint8_t a, uint32_t b)
inline constexpr FTranslationID TRANSLATION(uint8_t a, uint32_t b)
{
return (a << TRANSLATION_SHIFT) | b;
return FTranslationID::fromInt((a << TRANSLATION_SHIFT) | b);
}
inline constexpr uint32_t LuminosityTranslation(int range, uint8_t min, uint8_t max)
inline constexpr FTranslationID MakeLuminosityTranslation(int range, uint8_t min, uint8_t max)
{
// ensure that the value remains positive.
return ( (1 << 30) | ((range&0x3fff) << 16) | (min << 8) | max );
return FTranslationID::fromInt( (1 << 30) | ((range&0x3fff) << 16) | (min << 8) | max );
}
inline constexpr bool IsLuminosityTranslation(FTranslationID trans)
{
return trans.index() > 0 && (trans.index() & (1 << 30));
}
inline constexpr bool IsLuminosityTranslation(int trans)
@ -82,13 +130,25 @@ inline constexpr bool IsLuminosityTranslation(int trans)
return trans > 0 && (trans & (1 << 30));
}
inline constexpr int GetTranslationType(uint32_t trans)
inline constexpr int GetTranslationType(FTranslationID trans)
{
assert(!IsLuminosityTranslation(trans));
return (trans.index() & TRANSLATIONTYPE_MASK) >> TRANSLATION_SHIFT;
}
inline constexpr int GetTranslationIndex(FTranslationID trans)
{
assert(!IsLuminosityTranslation(trans));
return (trans.index() & TRANSLATION_MASK);
}
inline constexpr int GetTranslationType(int trans)
{
assert(!IsLuminosityTranslation(trans));
return (trans & TRANSLATIONTYPE_MASK) >> TRANSLATION_SHIFT;
}
inline constexpr int GetTranslationIndex(uint32_t trans)
inline constexpr int GetTranslationIndex(int trans)
{
assert(!IsLuminosityTranslation(trans));
return (trans & TRANSLATION_MASK);
@ -122,11 +182,16 @@ public:
void Clear();
int DetermineTranslucency(FileReader& file);
FRemapTable* AddRemap(FRemapTable* remap);
void UpdateTranslation(int trans, FRemapTable* remap);
int AddTranslation(int slot, FRemapTable* remap, int count = 1);
void CopyTranslation(int dest, int src);
int StoreTranslation(int slot, FRemapTable* remap);
FRemapTable* TranslationToTable(int translation);
void UpdateTranslation(FTranslationID trans, FRemapTable* remap);
FTranslationID AddTranslation(int slot, FRemapTable* remap, int count = 1);
void CopyTranslation(FTranslationID dest, FTranslationID src);
FTranslationID StoreTranslation(int slot, FRemapTable* remap);
FRemapTable* TranslationToTable(int translation) const;
FRemapTable* TranslationToTable(FTranslationID translation) const
{
return TranslationToTable(translation.index());
}
void GenerateGlobalBrightmapFromColormap(const uint8_t* cmapdata, int numlevels);
void PushIdentityTable(int slot)
@ -134,7 +199,7 @@ public:
AddTranslation(slot, nullptr);
}
FRemapTable* GetTranslation(int slot, int index)
FRemapTable* GetTranslation(int slot, int index) const
{
if (TranslationTables.Size() <= (unsigned)slot) return nullptr;
return TranslationTables[slot].GetVal(index);
@ -152,6 +217,28 @@ public:
return TranslationTables[slot].Size();
}
};
struct LuminosityTranslationDesc
{
int colorrange;
int lum_min;
int lum_max;
static LuminosityTranslationDesc fromInt(int translation)
{
LuminosityTranslationDesc t;
t.colorrange = (translation >> 16) & 0x3fff;
t.lum_min = (translation >> 8) & 0xff;
t.lum_max = translation & 0xff;
return t;
}
static LuminosityTranslationDesc fromID(FTranslationID translation)
{
return fromInt(translation.index());
}
};
extern PaletteContainer GPalette;

2
src/common/engine/sc_man.cpp
View file

@ -150,7 +150,7 @@ bool FScanner::OpenFile (const char *name)
auto filebuff = fr.Read();
if (filebuff.size() == 0 && filesize > 0) return false;
ScriptBuffer = FString((const char *)filebuff.data(), filesize);
ScriptBuffer = FString(filebuff.string(), filesize);
ScriptName = name; // This is used for error messages so the full file name is preferable
LumpNum = -1;
PrepareScript ();

8
src/common/engine/sc_man.h
View file

@ -71,12 +71,10 @@ public:
void Open(const char *lumpname);
bool OpenFile(const char *filename);
void OpenMem(const char *name, const char *buffer, int size);
void OpenMem(const char *name, const TArray<uint8_t> &buffer)
{
OpenMem(name, (const char*)buffer.Data(), buffer.Size());
}
void OpenMem(const char* name, const std::vector<uint8_t>& buffer)
template<class T>
void OpenMem(const char* name, const T& buffer)
{
static_assert(sizeof(typename T::value_type) == 1);
OpenMem(name, (const char*)buffer.data(), (int)buffer.size());
}
void OpenString(const char *name, FString buffer);

2
src/common/engine/sc_man_scanner.re
View file

@ -167,7 +167,7 @@ std2:
'vector2' { RET(TK_Vector2); }
'vector3' { RET(TK_Vector3); }
'map' { RET(TK_Map); }
'mapiterator' { RET(TK_MapIterator); }
'mapiterator' { RET(ParseVersion >= MakeVersion(4, 10, 0)? TK_MapIterator : TK_Identifier); }
'array' { RET(TK_Array); }
'function' { RET(ParseVersion >= MakeVersion(4, 12, 0)? TK_FunctionType : TK_Identifier); }
'in' { RET(TK_In); }

37
src/common/engine/serializer.cpp
View file

@ -1,5 +1,4 @@
/*
/*
** serializer.cpp
** Savegame wrapper around RapidJSON
**
@ -57,6 +56,7 @@
#include "texturemanager.h"
#include "base64.h"
#include "vm.h"
#include "i_interface.h"
using namespace FileSys;
@ -1197,6 +1197,28 @@ FSerializer &Serialize(FSerializer &arc, const char *key, FTextureID &value, FTe
return arc;
}
//==========================================================================
//
//
//
//==========================================================================
FSerializer& Serialize(FSerializer& arc, const char* key, FTranslationID& value, FTranslationID* defval)
{
int v = value.index();
int* defv = (int*)defval;
Serialize(arc, key, v, defv);
if (arc.isReading())
{
// allow games to alter the loaded value to handle dynamic lists.
if (sysCallbacks.RemapTranslation) value = sysCallbacks.RemapTranslation(FTranslationID::fromInt(v));
else value = FTranslationID::fromInt(v);
}
return arc;
}
//==========================================================================
//
// This never uses defval and instead uses 'null' as default
@ -1735,6 +1757,19 @@ void SerializeFunctionPointer(FSerializer &arc, const char *key, FunctionPointer
}
}
bool FSerializer::ReadOptionalInt(const char * key, int &into)
{
if(!isReading()) return false;
auto val = r->FindKey(key);
if(val && val->IsInt())
{
into = val->GetInt();
return true;
}
return false;
}
#include "renderstyle.h"
FSerializer& Serialize(FSerializer& arc, const char* key, FRenderStyle& style, FRenderStyle* def)
{

9
src/common/engine/serializer.h
View file

@ -20,6 +20,7 @@ class FSoundID;
union FRenderStyle;
class DObject;
class FTextureID;
struct FTranslationID;
inline bool nullcmp(const void *buffer, size_t length)
{
@ -107,6 +108,10 @@ public:
FSerializer &AddString(const char *key, const char *charptr);
const char *GetString(const char *key);
FSerializer &ScriptNum(const char *key, int &num);
bool ReadOptionalInt(const char * key, int &into);
bool isReading() const
{
return r != nullptr;
@ -239,7 +244,9 @@ FSerializer &Serialize(FSerializer &arc, const char *key, FName &value, FName *d
FSerializer &Serialize(FSerializer &arc, const char *key, FSoundID &sid, FSoundID *def);
FSerializer &Serialize(FSerializer &arc, const char *key, FString &sid, FString *def);
FSerializer &Serialize(FSerializer &arc, const char *key, NumericValue &sid, NumericValue *def);
FSerializer &Serialize(FSerializer &arc, const char *key, struct ModelOverride &sid, struct ModelOverride *def);
FSerializer &Serialize(FSerializer &arc, const char *key, struct ModelOverride &mo, struct ModelOverride *def);
FSerializer &Serialize(FSerializer &arc, const char *key, struct AnimOverride &ao, struct AnimOverride *def);
FSerializer& Serialize(FSerializer& arc, const char* key, FTranslationID& value, FTranslationID* defval);
void SerializeFunctionPointer(FSerializer &arc, const char *key, FunctionPointerValue *&p);

111
src/common/filesystem/include/fs_files.h
View file

@ -4,7 +4,7 @@
**
**---------------------------------------------------------------------------
** Copyright 1998-2008 Randy Heit
** Copyright 2005-2008 Christoph Oelckers
** Copyright 2005-2023 Christoph Oelckers
** All rights reserved.
**
** Redistribution and use in source and binary forms, with or without
@ -90,6 +90,84 @@ enum
class FileReader;
// an opaque memory buffer to the file's content. Can either own the memory or just point to an external buffer.
class ResourceData
{
void* memory;
size_t length;
bool owned;
public:
using value_type = uint8_t;
ResourceData() { memory = nullptr; length = 0; owned = true; }
const void* data() const { return memory; }
size_t size() const { return length; }
const char* string() const { return (const char*)memory; }
const uint8_t* bytes() const { return (const uint8_t*)memory; }
ResourceData& operator = (const ResourceData& copy)
{
if (owned && memory) free(memory);
length = copy.length;
owned = copy.owned;
if (owned)
{
memory = malloc(length);
memcpy(memory, copy.memory, length);
}
else memory = copy.memory;
return *this;
}
ResourceData& operator = (ResourceData&& copy) noexcept
{
if (owned && memory) free(memory);
length = copy.length;
owned = copy.owned;
memory = copy.memory;
copy.memory = nullptr;
copy.length = 0;
copy.owned = true;
return *this;
}
ResourceData(const ResourceData& copy)
{
memory = nullptr;
*this = copy;
}
~ResourceData()
{
if (owned && memory) free(memory);
}
void* allocate(size_t len)
{
if (!owned) memory = nullptr;
length = len;
owned = true;
memory = realloc(memory, length);
return memory;
}
void set(const void* mem, size_t len)
{
memory = (void*)mem;
length = len;
owned = false;
}
void clear()
{
if (owned && memory) free(memory);
memory = nullptr;
length = 0;
owned = true;
}
};
class FileReaderInterface
{
public:
@ -171,10 +249,12 @@ public:
mReader = nullptr;
}
bool OpenFile(const char *filename, Size start = 0, Size length = -1);
bool OpenFile(const char *filename, Size start = 0, Size length = -1, bool buffered = false);
bool OpenFilePart(FileReader &parent, Size start, Size length);
bool OpenMemory(const void *mem, Size length); // read directly from the buffer
bool OpenMemoryArray(const void *mem, Size length); // read from a copy of the buffer.
bool OpenMemoryArray(std::vector<uint8_t>& data); // take the given array
bool OpenMemoryArray(ResourceData& data); // take the given array
bool OpenMemoryArray(std::function<bool(std::vector<uint8_t>&)> getter); // read contents to a buffer and return a reader to it
bool OpenDecompressor(FileReader &parent, Size length, int method, bool seekable, bool exceptions = false); // creates a decompressor stream. 'seekable' uses a buffered version so that the Seek and Tell methods can be used.
@ -193,33 +273,34 @@ public:
return mReader->Read(buffer, len);
}
std::vector<uint8_t> Read(size_t len)
ResourceData Read(size_t len)
{
std::vector<uint8_t> buffer(len);
ResourceData buffer;
if (len > 0)
{
Size length = mReader->Read(&buffer[0], len);
buffer.resize((size_t)length);
Size length = mReader->Read(buffer.allocate(len), len);
if ((size_t)length < len) buffer.allocate(length);
}
return buffer;
}
std::vector<uint8_t> Read()
ResourceData Read()
{
return Read(GetLength());
}
std::vector<uint8_t> ReadPadded(size_t padding)
ResourceData ReadPadded(size_t padding)
{
auto len = GetLength();
std::vector<uint8_t> buffer(len + padding);
ResourceData buffer;
if (len > 0)
{
Size length = mReader->Read(&buffer[0], len);
auto p = (char*)buffer.allocate(len + padding);
Size length = mReader->Read(p, len);
if (length < len) buffer.clear();
else memset(buffer.data() + len, 0, padding);
else memset(p + len, 0, padding);
}
else buffer[0] = 0;
return buffer;
}
@ -353,13 +434,13 @@ protected:
class BufferWriter : public FileWriter
{
protected:
TArray<unsigned char> mBuffer;
std::vector<unsigned char> mBuffer;
public:
BufferWriter() {}
virtual size_t Write(const void *buffer, size_t len) override;
TArray<unsigned char> *GetBuffer() { return &mBuffer; }
TArray<unsigned char>&& TakeBuffer() { return std::move(mBuffer); }
std::vector<unsigned char> *GetBuffer() { return &mBuffer; }
std::vector<unsigned char>&& TakeBuffer() { return std::move(mBuffer); }
};
}

1
src/common/filesystem/include/fs_filesystem.h
View file

@ -174,7 +174,6 @@ public:
int AddFromBuffer(const char* name, const char* type, char* data, int size, int id, int flags);
FileReader* GetFileReader(int wadnum); // Gets a FileReader object to the entire WAD
void InitHashChains();
FResourceLump* GetFileAt(int no);
protected:

85
src/common/filesystem/include/resourcefile.h
View file

@ -88,8 +88,8 @@ enum ELumpFlags
// This holds a compresed Zip entry with all needed info to decompress it.
struct FCompressedBuffer
{
unsigned mSize;
unsigned mCompressedSize;
size_t mSize;
size_t mCompressedSize;
int mMethod;
int mZipFlags;
unsigned mCRC32;
@ -110,18 +110,34 @@ struct FCompressedBuffer
struct FResourceLump
{
protected:
friend class FResourceFile;
friend class FWadFile; // this still needs direct access.
friend class FileData;
friend class FileSystem;
friend class FLumpFile;
friend class FLumpReader;
friend class FGrpFile;
friend class F7ZFile;
friend class FSSIFile;
friend class FWHResFile;
friend class FZipFile;
friend class FPakFile;
friend class FRFFFile;
friend class FDirectory;
friend int lumpcmp(const void* a, const void* b);
int LumpSize;
int RefCount;
protected:
//protected:
const char* FullName;
public:
//public:
uint8_t Flags;
char * Cache;
FResourceFile * Owner;
public:
FResourceLump()
{
Cache = NULL;
@ -129,9 +145,12 @@ public:
Flags = 0;
RefCount = 0;
FullName = "";
LumpSize = 0;
}
virtual ~FResourceLump();
protected:
virtual FileReader *GetReader();
virtual FileReader NewReader();
virtual int GetFileOffset() { return -1; }
@ -157,9 +176,9 @@ protected:
class FResourceFile
{
public:
protected:
FileReader Reader;
const char* FileName;
protected:
uint32_t NumLumps;
char Hash[48];
StringPool* stringpool;
@ -186,15 +205,61 @@ public:
static FResourceFile *OpenDirectory(const char *filename, LumpFilterInfo* filter = nullptr, FileSystemMessageFunc Printf = nullptr, StringPool* sp = nullptr);
virtual ~FResourceFile();
// If this FResourceFile represents a directory, the Reader object is not usable so don't return it.
FileReader *GetReader() { return Reader.isOpen()? &Reader : nullptr; }
uint32_t LumpCount() const { return NumLumps; }
FileReader *GetContainerReader() { return Reader.isOpen()? &Reader : nullptr; }
const char* GetFileName() const { return FileName; }
uint32_t GetFirstEntry() const { return FirstLump; }
void SetFirstLump(uint32_t f) { FirstLump = f; }
const char* GetHash() const { return Hash; }
virtual FResourceLump *GetLump(int no) = 0;
FResourceLump *FindLump(const char *name);
int EntryCount() const { return NumLumps; }
int FindEntry(const char* name);
size_t Length(int entry)
{
auto l = GetLump(entry);
return l ? l->LumpSize : -1;
}
FileReader GetEntryReader(int entry, bool newreader = true)
{
auto l = GetLump(entry);
return l ? l->NewReader() : FileReader();
}
int GetEntryFlags(int entry)
{
auto l = GetLump(entry);
return l ? l->Flags : 0;
}
ResourceData Read(int entry)
{
auto fr = GetEntryReader(entry, false);
return fr.Read();
}
const char* getName(int entry)
{
auto l = GetLump(entry);
return l ? l->FullName : nullptr;
}
FCompressedBuffer GetRawData(int entry)
{
auto l = GetLump(entry);
if (!l) return {};
return l->GetRawData();
}
FileReader Destroy()
{
auto fr = std::move(Reader);
delete this;
return fr;
}
};

3
src/common/filesystem/source/file_7z.cpp
View file

@ -367,8 +367,7 @@ FResourceFile *Check7Z(const char *filename, FileReader &file, LumpFilterInfo* f
auto rf = new F7ZFile(filename, file, sp);
if (rf->Open(filter, Printf)) return rf;
file = std::move(rf->Reader); // to avoid destruction of reader
delete rf;
file = rf->Destroy();
}
}
return NULL;

4
src/common/filesystem/source/file_grp.cpp
View file

@ -145,9 +145,7 @@ FResourceFile *CheckGRP(const char *filename, FileReader &file, LumpFilterInfo*
{
auto rf = new FGrpFile(filename, file, sp);
if (rf->Open(filter)) return rf;
file = std::move(rf->Reader); // to avoid destruction of reader
delete rf;
file = rf->Destroy();
}
}
return NULL;

3
src/common/filesystem/source/file_lump.cpp
View file

@ -89,8 +89,7 @@ FResourceFile *CheckLump(const char *filename, FileReader &file, LumpFilterInfo*
// always succeeds
auto rf = new FLumpFile(filename, file, sp);
if (rf->Open(filter)) return rf;
file = std::move(rf->Reader); // to avoid destruction of reader
delete rf;
file = rf->Destroy();
return NULL;
}

4
src/common/filesystem/source/file_pak.cpp
View file

@ -138,9 +138,7 @@ FResourceFile *CheckPak(const char *filename, FileReader &file, LumpFilterInfo*
{
auto rf = new FPakFile(filename, file, sp);
if (rf->Open(filter)) return rf;
file = std::move(rf->Reader); // to avoid destruction of reader
delete rf;
file = rf->Destroy();
}
}
return NULL;

4
src/common/filesystem/source/file_rff.cpp
View file

@ -252,9 +252,7 @@ FResourceFile *CheckRFF(const char *filename, FileReader &file, LumpFilterInfo*
{
auto rf = new FRFFFile(filename, file, sp);
if (rf->Open(filter)) return rf;
file = std::move(rf->Reader); // to avoid destruction of reader
delete rf;
file = rf->Destroy();
}
}
return NULL;

13
src/common/filesystem/source/file_ssi.cpp
View file

@ -46,7 +46,7 @@ class FSSIFile : public FUncompressedFile
{
public:
FSSIFile(const char * filename, FileReader &file, StringPool* sp);
bool Open(int version, int lumpcount, LumpFilterInfo* filter);
bool Open(int version, int EntryCount, LumpFilterInfo* filter);
};
@ -68,13 +68,13 @@ FSSIFile::FSSIFile(const char *filename, FileReader &file, StringPool* sp)
//
//==========================================================================
bool FSSIFile::Open(int version, int lumpcount, LumpFilterInfo*)
bool FSSIFile::Open(int version, int EntryCount, LumpFilterInfo*)
{
NumLumps = lumpcount*2;
Lumps.Resize(lumpcount*2);
NumLumps = EntryCount*2;
Lumps.Resize(EntryCount*2);
int32_t j = (version == 2 ? 267 : 254) + (lumpcount * 121);
int32_t j = (version == 2 ? 267 : 254) + (EntryCount * 121);
for (uint32_t i = 0; i < NumLumps; i+=2)
{
char fn[13];
@ -147,8 +147,7 @@ FResourceFile* CheckSSI(const char* filename, FileReader& file, LumpFilterInfo*
}
auto ssi = new FSSIFile(filename, file, sp);
if (ssi->Open(version, numfiles, filter)) return ssi;
file = std::move(ssi->Reader); // to avoid destruction of reader
delete ssi;
file = ssi->Destroy();
}
}
return nullptr;

15
src/common/filesystem/source/file_wad.cpp
View file

@ -76,8 +76,8 @@ public:
{
if(!Compressed)
{
Owner->Reader.Seek(Position, FileReader::SeekSet);
return &Owner->Reader;
Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet);
return Owner->GetContainerReader();
}
return NULL;
}
@ -85,7 +85,7 @@ public:
{
if(!Compressed)
{
const char * buffer = Owner->Reader.GetBuffer();
const char * buffer = Owner->GetContainerReader()->GetBuffer();
if (buffer != NULL)
{
@ -96,20 +96,20 @@ public:
}
}
Owner->Reader.Seek(Position, FileReader::SeekSet);
Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet);
Cache = new char[LumpSize];
if(Compressed)
{
FileReader lzss;
if (lzss.OpenDecompressor(Owner->Reader, LumpSize, METHOD_LZSS, false, true))
if (lzss.OpenDecompressor(*Owner->GetContainerReader(), LumpSize, METHOD_LZSS, false, true))
{
lzss.Read(Cache, LumpSize);
}
}
else
{
auto read = Owner->Reader.Read(Cache, LumpSize);
auto read = Owner->GetContainerReader()->Read(Cache, LumpSize);
if (read != LumpSize)
{
throw FileSystemException("only read %d of %d bytes", (int)read, (int)LumpSize);
@ -478,8 +478,7 @@ FResourceFile *CheckWad(const char *filename, FileReader &file, LumpFilterInfo*
auto rf = new FWadFile(filename, file, sp);
if (rf->Open(filter, Printf)) return rf;
file = std::move(rf->Reader); // to avoid destruction of reader
delete rf;
file = rf->Destroy();
}
}
return NULL;

3
src/common/filesystem/source/file_whres.cpp
View file

@ -136,8 +136,7 @@ FResourceFile *CheckWHRes(const char *filename, FileReader &file, LumpFilterInfo
}
auto rf = new FWHResFile(filename, file, sp);
if (rf->Open(filter)) return rf;
file = std::move(rf->Reader); // to avoid destruction of reader
delete rf;
file = rf->Destroy();
}
return NULL;
}

36
src/common/filesystem/source/file_zip.cpp
View file

@ -52,7 +52,7 @@ namespace FileSys {
//
//==========================================================================
static bool UncompressZipLump(char *Cache, FileReader &Reader, int Method, int LumpSize, int CompressedSize, int GPFlags, bool exceptions)
static bool UncompressZipLump(char *Cache, FileReader &Reader, int Method, ptrdiff_t LumpSize, ptrdiff_t CompressedSize, int GPFlags, bool exceptions)
{
switch (Method)
{
@ -79,7 +79,7 @@ static bool UncompressZipLump(char *Cache, FileReader &Reader, int Method, int L
case METHOD_IMPLODE:
{
FZipExploder exploder;
if (exploder.Explode((unsigned char*)Cache, LumpSize, Reader, CompressedSize, GPFlags) == -1)
if (exploder.Explode((unsigned char*)Cache, (unsigned)LumpSize, Reader, (unsigned)CompressedSize, GPFlags) == -1)
{
// decompression failed so zero the cache.
memset(Cache, 0, LumpSize);
@ -89,7 +89,7 @@ static bool UncompressZipLump(char *Cache, FileReader &Reader, int Method, int L
case METHOD_SHRINK:
{
ShrinkLoop((unsigned char *)Cache, LumpSize, Reader, CompressedSize);
ShrinkLoop((unsigned char *)Cache, (unsigned)LumpSize, Reader, (unsigned)CompressedSize);
break;
}
@ -445,8 +445,8 @@ FCompressedBuffer FZipLump::GetRawData()
{
FCompressedBuffer cbuf = { (unsigned)LumpSize, (unsigned)CompressedSize, Method, GPFlags, CRC32, new char[CompressedSize] };
if (NeedFileStart) SetLumpAddress();
Owner->Reader.Seek(Position, FileReader::SeekSet);
Owner->Reader.Read(cbuf.mBuffer, CompressedSize);
Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet);
Owner->GetContainerReader()->Read(cbuf.mBuffer, CompressedSize);
return cbuf;
}
@ -464,8 +464,8 @@ void FZipLump::SetLumpAddress()
FZipLocalFileHeader localHeader;
int skiplen;
Owner->Reader.Seek(Position, FileReader::SeekSet);
Owner->Reader.Read(&localHeader, sizeof(localHeader));
Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet);
Owner->GetContainerReader()->Read(&localHeader, sizeof(localHeader));
skiplen = LittleShort(localHeader.NameLength) + LittleShort(localHeader.ExtraLength);
Position += sizeof(localHeader) + skiplen;
NeedFileStart = false;
@ -484,8 +484,8 @@ FileReader *FZipLump::GetReader()
if (Method == METHOD_STORED)
{
if (NeedFileStart) SetLumpAddress();
Owner->Reader.Seek(Position, FileReader::SeekSet);
return &Owner->Reader;
Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet);
return Owner->GetContainerReader();
}
else return NULL;
}
@ -501,7 +501,7 @@ int FZipLump::FillCache()
if (NeedFileStart) SetLumpAddress();
const char *buffer;
if (Method == METHOD_STORED && (buffer = Owner->Reader.GetBuffer()) != NULL)
if (Method == METHOD_STORED && (buffer = Owner->GetContainerReader()->GetBuffer()) != NULL)
{
// This is an in-memory file so the cache can point directly to the file's data.
Cache = const_cast<char*>(buffer) + Position;
@ -509,9 +509,9 @@ int FZipLump::FillCache()
return -1;
}
Owner->Reader.Seek(Position, FileReader::SeekSet);
Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet);
Cache = new char[LumpSize];
UncompressZipLump(Cache, Owner->Reader, Method, LumpSize, CompressedSize, GPFlags, true);
UncompressZipLump(Cache, *Owner->GetContainerReader(), Method, LumpSize, CompressedSize, GPFlags, true);
RefCount = 1;
return 1;
}
@ -548,9 +548,7 @@ FResourceFile *CheckZip(const char *filename, FileReader &file, LumpFilterInfo*
{
auto rf = new FZipFile(filename, file, sp);
if (rf->Open(filter, Printf)) return rf;
file = std::move(rf->Reader); // to avoid destruction of reader
delete rf;
file = rf->Destroy();
}
}
return NULL;
@ -606,8 +604,8 @@ static int AppendToZip(FileWriter *zip_file, const FCompressedBuffer &content, s
local.ModDate = LittleShort(dostime.first);
local.ModTime = LittleShort(dostime.second);
local.CRC32 = content.mCRC32;
local.UncompressedSize = LittleLong(content.mSize);
local.CompressedSize = LittleLong(content.mCompressedSize);
local.UncompressedSize = LittleLong((unsigned)content.mSize);
local.CompressedSize = LittleLong((unsigned)content.mCompressedSize);
local.NameLength = LittleShort((unsigned short)strlen(content.filename));
local.ExtraLength = 0;
@ -648,8 +646,8 @@ int AppendCentralDirectory(FileWriter *zip_file, const FCompressedBuffer &conten
dir.ModTime = LittleShort(dostime.first);
dir.ModDate = LittleShort(dostime.second);
dir.CRC32 = content.mCRC32;
dir.CompressedSize32 = LittleLong(content.mCompressedSize);
dir.UncompressedSize32 = LittleLong(content.mSize);
dir.CompressedSize32 = LittleLong((unsigned)content.mCompressedSize);
dir.UncompressedSize32 = LittleLong((unsigned)content.mSize);
dir.NameLength = LittleShort((unsigned short)strlen(content.filename));
dir.ExtraLength = 0;
dir.CommentLength = 0;

82
src/common/filesystem/source/files.cpp
View file

@ -34,6 +34,7 @@
*/
#include <string>
#include <memory>
#include "files_internal.h"
namespace FileSys {
@ -285,7 +286,7 @@ ptrdiff_t MemoryReader::Seek(ptrdiff_t offset, int origin)
ptrdiff_t MemoryReader::Read(void *buffer, ptrdiff_t len)
{
if (len>Length - FilePos) len = Length - FilePos;
if (len > Length - FilePos) len = Length - FilePos;
if (len<0) len = 0;
memcpy(buffer, bufptr + FilePos, len);
FilePos += len;
@ -322,40 +323,37 @@ char *MemoryReader::Gets(char *strbuf, ptrdiff_t len)
return strbuf;
}
//==========================================================================
//
// MemoryArrayReader
//
// reads data from an array of memory
//
//==========================================================================
class MemoryArrayReader : public MemoryReader
int BufferingReader::FillBuffer(ptrdiff_t newpos)
{
std::vector<uint8_t> buf;
public:
MemoryArrayReader(const char *buffer, ptrdiff_t length)
if (newpos > Length) newpos = Length;
if (newpos < bufferpos) return 0;
auto read = baseReader->Read(&buf[bufferpos], newpos - bufferpos);
bufferpos += read;
if (bufferpos == Length)
{
if (length > 0)
{
buf.resize(length);
memcpy(&buf[0], buffer, length);
}
UpdateBuffer();
// we have read the entire file, so delete our data provider.
baseReader.reset();
}
return read == newpos - bufferpos ? 0 : -1;
}
std::vector<uint8_t> &GetArray() { return buf; }
void UpdateBuffer()
{
bufptr = (const char*)buf.data();
FilePos = 0;
Length = buf.size();
}
};
ptrdiff_t BufferingReader::Seek(ptrdiff_t offset, int origin)
{
if (-1 == MemoryReader::Seek(offset, origin)) return -1;
return FillBuffer(FilePos);
}
ptrdiff_t BufferingReader::Read(void* buffer, ptrdiff_t len)
{
if (FillBuffer(FilePos + len) < 0) return 0;
return MemoryReader::Read(buffer, len);
}
char* BufferingReader::Gets(char* strbuf, ptrdiff_t len)
{
if (FillBuffer(FilePos + len) < 0) return nullptr;
return MemoryReader::Gets(strbuf, len);
}
//==========================================================================
//
@ -365,7 +363,7 @@ public:
//
//==========================================================================
bool FileReader::OpenFile(const char *filename, FileReader::Size start, FileReader::Size length)
bool FileReader::OpenFile(const char *filename, FileReader::Size start, FileReader::Size length, bool buffered)
{
auto reader = new StdFileReader;
if (!reader->Open(filename, start, length))
@ -374,7 +372,8 @@ bool FileReader::OpenFile(const char *filename, FileReader::Size start, FileRead
return false;
}
Close();
mReader = reader;
if (buffered) mReader = new BufferingReader(reader);
else mReader = reader;
return true;
}
@ -396,13 +395,27 @@ bool FileReader::OpenMemory(const void *mem, FileReader::Size length)
bool FileReader::OpenMemoryArray(const void *mem, FileReader::Size length)
{
Close();
mReader = new MemoryArrayReader((const char *)mem, length);
mReader = new MemoryArrayReader<std::vector<uint8_t>>((const char *)mem, length);
return true;
}
bool FileReader::OpenMemoryArray(std::vector<uint8_t>& data)
{
Close();
if (data.size() > 0) mReader = new MemoryArrayReader<std::vector<uint8_t>>(data);
return true;
}
bool FileReader::OpenMemoryArray(ResourceData& data)
{
Close();
if (data.size() > 0) mReader = new MemoryArrayReader<ResourceData>(data);
return true;
}
bool FileReader::OpenMemoryArray(std::function<bool(std::vector<uint8_t>&)> getter)
{
auto reader = new MemoryArrayReader(nullptr, 0);
auto reader = new MemoryArrayReader<std::vector<uint8_t>>(nullptr, 0);
if (getter(reader->GetArray()))
{
Close();
@ -494,7 +507,8 @@ size_t FileWriter::Printf(const char *fmt, ...)
size_t BufferWriter::Write(const void *buffer, size_t len)
{
unsigned int ofs = mBuffer.Reserve((unsigned)len);
size_t ofs = mBuffer.size();
mBuffer.resize(ofs + len);
memcpy(&mBuffer[ofs], buffer, len);
return len;
}

68
src/common/filesystem/source/files_internal.h
View file

@ -1,5 +1,6 @@
#pragma once
#include <memory>
#include "fs_files.h"
namespace FileSys {
@ -28,4 +29,71 @@ public:
virtual const char *GetBuffer() const override { return bufptr; }
};
class BufferingReader : public MemoryReader
{
std::vector<uint8_t> buf;
std::unique_ptr<FileReaderInterface> baseReader;
ptrdiff_t bufferpos = 0;
int FillBuffer(ptrdiff_t newpos);
public:
BufferingReader(FileReaderInterface* base)
: baseReader(base)
{
}
ptrdiff_t Seek(ptrdiff_t offset, int origin) override;
ptrdiff_t Read(void* buffer, ptrdiff_t len) override;
char* Gets(char* strbuf, ptrdiff_t len) override;
};
//==========================================================================
//
// MemoryArrayReader
//
// reads data from an array of memory
//
//==========================================================================
template<class T>
class MemoryArrayReader : public MemoryReader
{
T buf;
public:
MemoryArrayReader()
{
FilePos = 0;
Length = 0;
}
MemoryArrayReader(const char* buffer, ptrdiff_t length)
{
if (length > 0)
{
buf.resize(length);
memcpy(&buf[0], buffer, length);
}
UpdateBuffer();
}
MemoryArrayReader(T& buffer)
{
buf = std::move(buffer);
UpdateBuffer();
}
std::vector<uint8_t>& GetArray() { return buf; }
void UpdateBuffer()
{
bufptr = (const char*)buf.data();
FilePos = 0;
Length = buf.size();
}
};
bool OpenMemoryArray(std::vector<uint8_t>& data); // take the given array
}

47
src/common/filesystem/source/filesystem.cpp
View file

@ -404,14 +404,14 @@ void FileSystem::AddFile (const char *filename, FileReader *filer, LumpFilterInf
if (resfile != NULL)
{
if (Printf)
Printf(FSMessageLevel::Message, "adding %s, %d lumps\n", filename, resfile->LumpCount());
Printf(FSMessageLevel::Message, "adding %s, %d lumps\n", filename, resfile->EntryCount());
uint32_t lumpstart = (uint32_t)FileInfo.size();
resfile->SetFirstLump(lumpstart);
for (uint32_t i=0; i < resfile->LumpCount(); i++)
for (int i = 0; i < resfile->EntryCount(); i++)
{
FResourceLump *lump = resfile->GetLump(i);
FResourceLump* lump = resfile->GetLump(i);
FileInfo.resize(FileInfo.size() + 1);
FileSystem::LumpRecord* lump_p = &FileInfo.back();
lump_p->SetFromLump((int)Files.size(), lump, stringpool);
@ -419,15 +419,15 @@ void FileSystem::AddFile (const char *filename, FileReader *filer, LumpFilterInf
Files.push_back(resfile);
for (uint32_t i=0; i < resfile->LumpCount(); i++)
for (int i = 0; i < resfile->EntryCount(); i++)
{
FResourceLump *lump = resfile->GetLump(i);
if (lump->Flags & LUMPF_EMBEDDED)
int flags = resfile->GetEntryFlags(i);
if (flags & LUMPF_EMBEDDED)
{
std::string path = filename;
path += ':';
path += lump->getName();
auto embedded = lump->NewReader();
path += resfile->getName(i);
auto embedded = resfile->GetEntryReader(i, true);
AddFile(path.c_str(), &embedded, filter, Printf, hashfile);
}
}
@ -454,13 +454,12 @@ void FileSystem::AddFile (const char *filename, FileReader *filer, LumpFilterInf
else
fprintf(hashfile, "file: %s, Directory structure\n", filename);
for (uint32_t i = 0; i < resfile->LumpCount(); i++)
for (int i = 0; i < resfile->EntryCount(); i++)
{
FResourceLump *lump = resfile->GetLump(i);
if (!(lump->Flags & LUMPF_EMBEDDED))
int flags = resfile->GetEntryFlags(i);
if (!(flags & LUMPF_EMBEDDED))
{
auto reader = lump->NewReader();
auto reader = resfile->GetEntryReader(i, true);
md5Hash(filereader, cksum);
for (size_t j = 0; j < sizeof(cksum); ++j)
@ -468,7 +467,7 @@ void FileSystem::AddFile (const char *filename, FileReader *filer, LumpFilterInf
snprintf(cksumout + (j * 2), 3, "%02X", cksum[j]);
}
fprintf(hashfile, "file: %s, lump: %s, hash: %s, size: %d\n", filename, lump->getName(), cksumout, lump->LumpSize);
fprintf(hashfile, "file: %s, lump: %s, hash: %s, size: %llu\n", filename, resfile->getName(i), cksumout, (uint64_t)resfile->Length(i));
}
}
}
@ -1421,7 +1420,7 @@ FileReader *FileSystem::GetFileReader(int rfnum)
return NULL;
}
return Files[rfnum]->GetReader();
return Files[rfnum]->GetContainerReader();
}
//==========================================================================
@ -1441,7 +1440,7 @@ const char *FileSystem::GetResourceFileName (int rfnum) const noexcept
return NULL;
}
name = Files[rfnum]->FileName;
name = Files[rfnum]->GetFileName();
slash = strrchr (name, '/');
return (slash != nullptr && slash[1] != 0) ? slash+1 : name;
}
@ -1473,7 +1472,7 @@ int FileSystem::GetLastEntry (int rfnum) const noexcept
return 0;
}
return Files[rfnum]->GetFirstEntry() + Files[rfnum]->LumpCount() - 1;
return Files[rfnum]->GetFirstEntry() + Files[rfnum]->EntryCount() - 1;
}
//==========================================================================
@ -1488,7 +1487,7 @@ int FileSystem::GetEntryCount (int rfnum) const noexcept
return 0;
}
return Files[rfnum]->LumpCount();
return Files[rfnum]->EntryCount();
}
@ -1507,7 +1506,7 @@ const char *FileSystem::GetResourceFileFullName (int rfnum) const noexcept
return nullptr;
}
return Files[rfnum]->FileName;
return Files[rfnum]->GetFileName();
}
@ -1592,15 +1591,5 @@ static void PrintLastError (FileSystemMessageFunc Printf)
}
#endif
//==========================================================================
//
// NBlood style lookup functions
//
//==========================================================================
FResourceLump* FileSystem::GetFileAt(int no)
{
return FileInfo[no].lump;
}
}

34
src/common/filesystem/source/resourcefile.cpp
View file

@ -157,7 +157,7 @@ static bool IsWadInFolder(const FResourceFile* const archive, const char* const
return false;
}
const auto dirName = ExtractBaseName(archive->FileName);
const auto dirName = ExtractBaseName(archive->GetFileName());
const auto fileName = ExtractBaseName(resPath, true);
const std::string filePath = dirName + '/' + fileName;
@ -242,7 +242,7 @@ void *FResourceLump::Lock()
catch (const FileSystemException& err)
{
// enrich the message with info about this lump.
throw FileSystemException("%s, file '%s': %s", getName(), Owner->FileName, err.what());
throw FileSystemException("%s, file '%s': %s", getName(), Owner->GetFileName(), err.what());
}
}
return Cache;
@ -360,13 +360,13 @@ int lumpcmp(const void * a, const void * b)
//
// Generates a hash identifier for use in file identification.
// Potential uses are mod-wide compatibility settings or localization add-ons.
// This only hashes the lump directory but not the actual content
// This only hashes the directory but not the actual content
//
//==========================================================================
void FResourceFile::GenerateHash()
{
// hash the lump directory after sorting
// hash the directory after sorting
using namespace FileSys::md5;
auto n = snprintf(Hash, 48, "%08X-%04X-", (unsigned)Reader.GetLength(), NumLumps);
@ -377,9 +377,10 @@ void FResourceFile::GenerateHash()
uint8_t digest[16];
for(uint32_t i = 0; i < NumLumps; i++)
{
auto lump = GetLump(i);
md5_append(&state, (const uint8_t*)lump->FullName, (unsigned)strlen(lump->FullName) + 1);
md5_append(&state, (const uint8_t*)&lump->LumpSize, 4);
auto name = getName(i);
auto size = Length(i);
md5_append(&state, (const uint8_t*)name, (unsigned)strlen(name) + 1);
md5_append(&state, (const uint8_t*)&size, sizeof(size));
}
md5_finish(&state, digest);
for (auto c : digest)
@ -618,17 +619,16 @@ bool FResourceFile::FindPrefixRange(const char* filter, void *lumps, size_t lump
//
//==========================================================================
FResourceLump *FResourceFile::FindLump(const char *name)
int FResourceFile::FindEntry(const char *name)
{
for (unsigned i = 0; i < NumLumps; i++)
{
FResourceLump *lump = GetLump(i);
if (!stricmp(name, lump->FullName))
if (!stricmp(name, getName(i)))
{
return lump;
return i;
}
}
return nullptr;
return -1;
}
//==========================================================================
@ -639,8 +639,8 @@ FResourceLump *FResourceFile::FindLump(const char *name)
FileReader *FUncompressedLump::GetReader()
{
Owner->Reader.Seek(Position, FileReader::SeekSet);
return &Owner->Reader;
Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet);
return Owner->GetContainerReader();
}
//==========================================================================
@ -651,7 +651,7 @@ FileReader *FUncompressedLump::GetReader()
int FUncompressedLump::FillCache()
{
const char * buffer = Owner->Reader.GetBuffer();
const char * buffer = Owner->GetContainerReader()->GetBuffer();
if (buffer != NULL)
{
@ -661,10 +661,10 @@ int FUncompressedLump::FillCache()
return -1;
}
Owner->Reader.Seek(Position, FileReader::SeekSet);
Owner->GetContainerReader()->Seek(Position, FileReader::SeekSet);
Cache = new char[LumpSize];
auto read = Owner->Reader.Read(Cache, LumpSize);
auto read = Owner->GetContainerReader()->Read(Cache, LumpSize);
if (read != LumpSize)
{
throw FileSystemException("only read %d of %d bytes", (int)read, (int)LumpSize);

161
src/common/filesystem/source/unicode.cpp Normal file
View file

@ -0,0 +1,161 @@
/*
** unicode.cpp
** handling for conversion / comparison of filenames
**
**---------------------------------------------------------------------------
** Copyright 2019 Christoph Oelckers
** All rights reserved.
**
** Redistribution and use in source and binary forms, with or without
** modification, are permitted provided that the following conditions
** are met:
**
** 1. Redistributions of source code must retain the above copyright
** notice, this list of conditions and the following disclaimer.
** 2. Redistributions in binary form must reproduce the above copyright
** notice, this list of conditions and the following disclaimer in the
** documentation and/or other materials provided with the distribution.
** 3. The name of the author may not be used to endorse or promote products
** derived from this software without specific prior written permission.
**
** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
** IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
** OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
** IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
** INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
** NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
** THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
**---------------------------------------------------------------------------
**
*/
#include "unicode.h"
#include "utf8proc.h"
namespace FileSys {
//==========================================================================
//
//
//
//==========================================================================
static void utf8_encode(int32_t codepoint, std::vector<char>& buffer)
{
if (codepoint < 0 || codepoint > 0x10FFFF || (codepoint >= 0xD800 && codepoint < 0xDFFF))
{
codepoint = 0xFFFD;
}
uint8_t buf[4];
auto size = utf8proc_encode_char(codepoint, buf);
for(int i = 0; i < size; i++)
buffer.push_back(buf[i]);
}
//==========================================================================
//
// convert UTF16 to UTF8 (needed for 7z)
//
//==========================================================================
void utf16_to_utf8(const unsigned short* in, std::vector<char>& buffer)
{
buffer.clear();
if (!*in) return;
while (int char1 = *in++)
{
if (char1 >= 0xD800 && char1 <= 0xDBFF)
{
int char2 = *in;
if (char2 >= 0xDC00 && char2 <= 0xDFFF)
{
in++;
char1 -= 0xD800;
char2 -= 0xDC00;
char1 <<= 10;
char1 += char2;
char1 += 0x010000;
}
else
{
// invalid code point - replace with placeholder
char1 = 0xFFFD;
}
}
else if (char1 >= 0xDC00 && char1 <= 0xDFFF)
{
// invalid code point - replace with placeholder
char1 = 0xFFFD;
}
utf8_encode(char1, buffer);
}
buffer.push_back(0);
}
//==========================================================================
//
// convert UTF16 to UTF8 (needed for Zip)
//
//==========================================================================
void ibm437_to_utf8(const char* in, std::vector<char>& buffer)
{
static const uint16_t ibm437map[] = {
0x00c7, 0x00fc, 0x00e9, 0x00e2, 0x00e4, 0x00e0, 0x00e5, 0x00e7, 0x00ea, 0x00eb, 0x00e8, 0x00ef, 0x00ee, 0x00ec, 0x00c4, 0x00c5,
0x00c9, 0x00e6, 0x00c6, 0x00f4, 0x00f6, 0x00f2, 0x00fb, 0x00f9, 0x00ff, 0x00d6, 0x00dc, 0x00a2, 0x00a3, 0x00a5, 0x20a7, 0x0192,
0x00e1, 0x00ed, 0x00f3, 0x00fa, 0x00f1, 0x00d1, 0x00aa, 0x00ba, 0x00bf, 0x2310, 0x00ac, 0x00bd, 0x00bc, 0x00a1, 0x00ab, 0x00bb,
0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561, 0x2562, 0x2556, 0x2555, 0x2563, 0x2551, 0x2557, 0x255d, 0x255c, 0x255b, 0x2510,
0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x255e, 0x255f, 0x255a, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256c, 0x2567,
0x2568, 0x2564, 0x2565, 0x2559, 0x2558, 0x2552, 0x2553, 0x256b, 0x256a, 0x2518, 0x250c, 0x2588, 0x2584, 0x258c, 0x2590, 0x2580,
0x03b1, 0x00df, 0x0393, 0x03c0, 0x03a3, 0x03c3, 0x00b5, 0x03c4, 0x03a6, 0x0398, 0x03a9, 0x03b4, 0x221e, 0x03c6, 0x03b5, 0x2229,
0x2261, 0x00b1, 0x2265, 0x2264, 0x2320, 0x2321, 0x00f7, 0x2248, 0x00b0, 0x2219, 0x00b7, 0x221a, 0x207f, 0x00b2, 0x25a0, 0x00a0,
};
buffer.clear();
if (!*in) return;
while (int char1 = (uint8_t)*in++)
{
if (char1 >= 0x80) char1 = ibm437map[char1 - 0x80];
utf8_encode(char1, buffer);
}
buffer.push_back(0);
}
//==========================================================================
//
// create a normalized lowercase version of a string.
//
//==========================================================================
char *tolower_normalize(const char *str)
{
utf8proc_uint8_t *retval;
utf8proc_map((const uint8_t*)str, 0, &retval, (utf8proc_option_t)(UTF8PROC_NULLTERM | UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_CASEFOLD));
return (char*)retval;
}
//==========================================================================
//
// validates the string for proper UTF-8
//
//==========================================================================
bool unicode_validate(const char* str)
{
while (*str != 0)
{
int cp;
int result = utf8proc_iterate((const uint8_t*)str, -1, &cp);
if (result < 0) return false;
}
return true;
}
}

12
src/common/filesystem/source/unicode.h Normal file
View file

@ -0,0 +1,12 @@
#pragma once
#include <stdint.h>
#include <vector>
namespace FileSys {
void utf16_to_utf8(const unsigned short* in, std::vector<char>& buffer);
void ibm437_to_utf8(const char* in, std::vector<char>& buffer);
char *tolower_normalize(const char *str);
bool unicode_validate(const char* str);
}

10
src/common/fonts/font.cpp
View file

@ -711,10 +711,10 @@ int FFont::GetLuminosity (uint32_t *colorsused, TArray<double> &Luminosity, int*
//
//==========================================================================
int FFont::GetColorTranslation (EColorRange range, PalEntry *color) const
FTranslationID FFont::GetColorTranslation (EColorRange range, PalEntry *color) const
{
// Single pic fonts do not set up their translation table and must always return 0.
if (Translations.Size() == 0) return 0;
if (Translations.Size() == 0) return NO_TRANSLATION;
assert(Translations.Size() == (unsigned)NumTextColors);
if (noTranslate)
@ -728,7 +728,7 @@ int FFont::GetColorTranslation (EColorRange range, PalEntry *color) const
if (color != nullptr) *color = retcolor;
}
if (range == CR_UNDEFINED)
return -1;
return INVALID_TRANSLATION;
else if (range >= NumTextColors)
range = CR_UNTRANSLATED;
return Translations[range];
@ -1059,8 +1059,8 @@ void FFont::LoadTranslations()
Translations.Resize(NumTextColors);
for (int i = 0; i < NumTextColors; i++)
{
if (i == CR_UNTRANSLATED) Translations[i] = 0;
else Translations[i] = LuminosityTranslation(i*2 + TranslationType, minlum, maxlum);
if (i == CR_UNTRANSLATED) Translations[i] = NO_TRANSLATION;
else Translations[i] = MakeLuminosityTranslation(i*2 + TranslationType, minlum, maxlum);
}
}

21
src/common/fonts/hexfont.cpp
View file

@ -58,12 +58,12 @@ struct HexDataSource
//
//==========================================================================
void ParseDefinition(FileSys::FResourceLump* font)
void ParseDefinition(FResourceFile* resf, int index)
{
FScanner sc;
auto data = font->Lock();
sc.OpenMem("newconsolefont.hex", (const char*)data, font->Size());
auto data = resf->Read(index);
sc.OpenMem("newconsolefont.hex", data.string(), data.size());
sc.SetCMode(true);
glyphdata.Push(0); // ensure that index 0 can be used as 'not present'.
while (sc.GetString())
@ -97,7 +97,6 @@ struct HexDataSource
lumb = i * 255 / 17;
SmallPal[i] = PalEntry(255, lumb, lumb, lumb);
}
font->Unlock();
}
};
@ -323,8 +322,8 @@ public:
Translations.Resize(NumTextColors);
for (int i = 0; i < NumTextColors; i++)
{
if (i == CR_UNTRANSLATED) Translations[i] = 0;
else Translations[i] = LuminosityTranslation(i * 2 + 1, minlum, maxlum);
if (i == CR_UNTRANSLATED) Translations[i] = NO_TRANSLATION;
else Translations[i] = MakeLuminosityTranslation(i * 2 + 1, minlum, maxlum);
}
}
@ -387,8 +386,8 @@ public:
Translations.Resize(NumTextColors);
for (int i = 0; i < NumTextColors; i++)
{
if (i == CR_UNTRANSLATED) Translations[i] = 0;
else Translations[i] = LuminosityTranslation(i * 2, minlum, maxlum);
if (i == CR_UNTRANSLATED) Translations[i] = NO_TRANSLATION;
else Translations[i] = MakeLuminosityTranslation(i * 2, minlum, maxlum);
}
}
};
@ -440,8 +439,8 @@ void LoadHexFont(const char* filename)
{
auto resf = FResourceFile::OpenResourceFile(filename);
if (resf == nullptr) I_FatalError("Unable to open %s", filename);
auto hexfont = resf->FindLump("newconsolefont.hex");
if (hexfont == nullptr) I_FatalError("Unable to find newconsolefont.hex in %s", filename);
hexdata.ParseDefinition(hexfont);
auto hexfont = resf->FindEntry("newconsolefont.hex");
if (hexfont < 0) I_FatalError("Unable to find newconsolefont.hex in %s", filename);
hexdata.ParseDefinition(resf, hexfont);
delete resf;
}

4
src/common/fonts/singlelumpfont.cpp
View file

@ -194,8 +194,8 @@ void FSingleLumpFont::LoadTranslations()
Translations.Resize(NumTextColors);
for (int i = 0; i < NumTextColors; i++)
{
if (i == CR_UNTRANSLATED) Translations[i] = 0;
else Translations[i] = LuminosityTranslation(i * 2 + (FontType == FONT1 ? 1 : 0), minlum, maxlum);
if (i == CR_UNTRANSLATED) Translations[i] = NO_TRANSLATION;
else Translations[i] = MakeLuminosityTranslation(i * 2 + (FontType == FONT1 ? 1 : 0), minlum, maxlum);
}
}

2
src/common/fonts/specialfont.cpp
View file

@ -182,7 +182,7 @@ void FSpecialFont::LoadTranslations()
workpal[i * 4 + 2] = GPalette.BaseColors[i].r;
workpal[i * 4 + 3] = GPalette.BaseColors[i].a;
}
V_ApplyLuminosityTranslation(trans, workpal, 256);
V_ApplyLuminosityTranslation(LuminosityTranslationDesc::fromID(trans), workpal, 256);
for (int i = 0; i < 256; i++)
{
if (!notranslate[i])

4
src/common/fonts/ttffont.cpp
View file

@ -127,8 +127,8 @@ public:
Translations.Resize(NumTextColors);
for (int i = 0; i < NumTextColors; i++)
{
if (i == CR_UNTRANSLATED) Translations[i] = 0;
else Translations[i] = LuminosityTranslation(i * 2, minlum, maxlum);
if (i == CR_UNTRANSLATED) Translations[i] = NO_TRANSLATION;
else Translations[i] = MakeLuminosityTranslation(i * 2, minlum, maxlum);
}
}

23
src/common/fonts/v_font.cpp
View file

@ -667,12 +667,12 @@ static void CreateLuminosityTranslationRanges()
//
//==========================================================================
void V_ApplyLuminosityTranslation(int translation, uint8_t* pixel, int size)
void V_ApplyLuminosityTranslation(const LuminosityTranslationDesc& lum, uint8_t* pixel, int size)
{
int colorrange = (translation >> 16) & 0x3fff;
int colorrange = lum.colorrange;
if (colorrange >= NumTextColors * 2) return;
int lum_min = (translation >> 8) & 0xff;
int lum_max = translation & 0xff;
int lum_min = lum.lum_min;
int lum_max = lum.lum_max;
int lum_range = (lum_max - lum_min + 1);
PalEntry* remap = paletteptr + colorrange * 256;
@ -898,10 +898,11 @@ void V_LoadTranslations()
CalcDefaultTranslation(BigFont, CR_UNTRANSLATED * 2 + 1);
if (OriginalBigFont != nullptr && OriginalBigFont != BigFont)
{
int sometrans = OriginalBigFont->Translations[0];
assert(IsLuminosityTranslation(OriginalBigFont->Translations[0]));
int sometrans = OriginalBigFont->Translations[0].index();
sometrans &= ~(0x3fff << 16);
sometrans |= (CR_UNTRANSLATED * 2 + 1) << 16;
OriginalBigFont->Translations[CR_UNTRANSLATED] = sometrans;
OriginalBigFont->Translations[CR_UNTRANSLATED] = FTranslationID::fromInt(sometrans);
OriginalBigFont->forceremap = true;
}
}
@ -910,18 +911,20 @@ void V_LoadTranslations()
CalcDefaultTranslation(SmallFont, CR_UNTRANSLATED * 2);
if (OriginalSmallFont != nullptr && OriginalSmallFont != SmallFont)
{
int sometrans = OriginalSmallFont->Translations[0];
assert(IsLuminosityTranslation(OriginalSmallFont->Translations[0]));
int sometrans = OriginalSmallFont->Translations[0].index();
sometrans &= ~(0x3fff << 16);
sometrans |= (CR_UNTRANSLATED * 2) << 16;
OriginalSmallFont->Translations[CR_UNTRANSLATED] = sometrans;
OriginalSmallFont->Translations[CR_UNTRANSLATED] = FTranslationID::fromInt(sometrans);
OriginalSmallFont->forceremap = true;
}
if (NewSmallFont != nullptr)
{
int sometrans = NewSmallFont->Translations[0];
assert(IsLuminosityTranslation(NewSmallFont->Translations[0]));
int sometrans = NewSmallFont->Translations[0].index();
sometrans &= ~(0x3fff << 16);
sometrans |= (CR_UNTRANSLATED * 2) << 16;
NewSmallFont->Translations[CR_UNTRANSLATED] = sometrans;
NewSmallFont->Translations[CR_UNTRANSLATED] = FTranslationID::fromInt(sometrans);
NewSmallFont->forceremap = true;
}
}

7
src/common/fonts/v_font.h
View file

@ -37,6 +37,7 @@
#include "vectors.h"
#include "palentry.h"
#include "name.h"
#include "palettecontainer.h"
class FGameTexture;
struct FRemapTable;
@ -104,7 +105,7 @@ public:
virtual FGameTexture *GetChar (int code, int translation, int *const width) const;
virtual int GetCharWidth (int code) const;
int GetColorTranslation (EColorRange range, PalEntry *color = nullptr) const;
FTranslationID GetColorTranslation (EColorRange range, PalEntry *color = nullptr) const;
int GetLump() const { return Lump; }
int GetSpaceWidth () const { return SpaceWidth; }
int GetHeight () const { return FontHeight; }
@ -197,7 +198,7 @@ protected:
int XMove = INT_MIN;
};
TArray<CharData> Chars;
TArray<int> Translations;
TArray<FTranslationID> Translations;
int Lump;
FName FontName = NAME_None;
@ -219,7 +220,7 @@ PalEntry V_LogColorFromColorRange (EColorRange range);
EColorRange V_ParseFontColor (const uint8_t *&color_value, int normalcolor, int boldcolor);
void V_InitFontColors();
char* CleanseString(char* str);
void V_ApplyLuminosityTranslation(int translation, uint8_t* pixel, int size);
void V_ApplyLuminosityTranslation(const LuminosityTranslationDesc& lum, uint8_t* pixel, int size);
void V_LoadTranslations();
class FBitmap;

29
src/common/menu/savegamemanager.cpp
View file

@ -297,37 +297,34 @@ unsigned FSavegameManagerBase::ExtractSaveData(int index)
!node->bOldVersion &&
(resf = FResourceFile::OpenResourceFile(node->Filename.GetChars(), true)) != nullptr)
{
auto info = resf->FindLump("info.json");
if (info == nullptr)
auto info = resf->FindEntry("info.json");
if (info < 0)
{
// this should not happen because the file has already been verified.
return index;
}
void* data = info->Lock();
auto data = resf->Read(info);
FSerializer arc;
if (!arc.OpenReader((const char*)data, info->LumpSize))
if (!arc.OpenReader(data.string(), data.size()))
{
info->Unlock();
return index;
}
info->Unlock();
SaveCommentString = ExtractSaveComment(arc);
auto pic = resf->FindLump("savepic.png");
if (pic != nullptr)
auto pic = resf->FindEntry("savepic.png");
if (pic >= 0)
{
FileReader picreader;
picreader.OpenMemoryArray([=](std::vector<uint8_t> &array)
{
auto cache = pic->Lock();
array.resize(pic->LumpSize);
memcpy(&array[0], cache, pic->LumpSize);
pic->Unlock();
return true;
});
picreader.OpenMemoryArray([=](std::vector<uint8_t>& array)
{
auto rd = resf->GetEntryReader(pic, false);
array.resize(resf->Length(pic));
rd.Read(array.data(), array.size());
return true;
});
PNGHandle *png = M_VerifyPNG(picreader);
if (png != nullptr)
{

1
src/common/models/model.cpp
View file

@ -47,6 +47,7 @@
TArray<FString> savedModelFiles;
TDeletingArray<FModel*> Models;
TArray<FSpriteModelFrame> SpriteModelFrames;
TMap<void*, FSpriteModelFrame> BaseSpriteModelFrames;
/////////////////////////////////////////////////////////////////////////////

18
src/common/models/model.h
View file

@ -4,7 +4,10 @@
#include "textureid.h"
#include "i_modelvertexbuffer.h"
#include "matrix.h"
#include "palettecontainer.h"
#include "TRS.h"
#include "tarray.h"
#include "name.h"
class DBoneComponents;
class FModelRenderer;
@ -18,6 +21,7 @@ void FlushModels();
extern TArray<FString> savedModelFiles;
extern TDeletingArray<FModel*> Models;
extern TArray<FSpriteModelFrame> SpriteModelFrames;
extern TMap<void*, FSpriteModelFrame> BaseSpriteModelFrames;
#define MD3_MAX_SURFACES 32
#define MIN_MODELS 4
@ -71,13 +75,20 @@ public:
virtual ~FModel();
virtual bool Load(const char * fn, int lumpnum, const char * buffer, int length) = 0;
virtual int FindFrame(const char * name, bool nodefault = false) = 0;
virtual void RenderFrame(FModelRenderer *renderer, FGameTexture * skin, int frame, int frame2, double inter, int translation, const FTextureID* surfaceskinids, const TArray<VSMatrix>& boneData, int boneStartPosition) = 0;
// [RL0] these are used for decoupled iqm animations
virtual int FindFirstFrame(FName name) { return FErr_NotFound; }
virtual int FindLastFrame(FName name) { return FErr_NotFound; }
virtual double FindFramerate(FName name) { return FErr_NotFound; }
virtual void RenderFrame(FModelRenderer *renderer, FGameTexture * skin, int frame, int frame2, double inter, FTranslationID translation, const FTextureID* surfaceskinids, const TArray<VSMatrix>& boneData, int boneStartPosition) = 0;
virtual void BuildVertexBuffer(FModelRenderer *renderer) = 0;
virtual void AddSkins(uint8_t *hitlist, const FTextureID* surfaceskinids) = 0;
virtual float getAspectFactor(float vscale) { return 1.f; }
virtual const TArray<TRS>* AttachAnimationData() { return nullptr; };
virtual const TArray<VSMatrix> CalculateBones(int frame1, int frame2, double inter, const TArray<TRS>* animationData, DBoneComponents* bones, int index) { return {}; };
virtual const TArray<VSMatrix> CalculateBones(int frame1, int frame2, float inter, int frame1_prev, float inter1_prev, int frame2_prev, float inter2_prev, const TArray<TRS>* animationData, DBoneComponents* bones, int index) { return {}; };
void SetVertexBuffer(int type, IModelVertexBuffer *buffer) { mVBuf[type] = buffer; }
IModelVertexBuffer *GetVertexBuffer(int type) const { return mVBuf[type]; }
@ -85,7 +96,8 @@ public:
bool hasSurfaces = false;
FString mFileName;
FSpriteModelFrame *baseFrame;
private:
IModelVertexBuffer *mVBuf[NumModelRendererTypes];
};

11
src/common/models/model_iqm.h
View file

@ -6,6 +6,7 @@
#include "matrix.h"
#include "common/rendering/i_modelvertexbuffer.h"
#include "m_swap.h"
#include "name.h"
class DBoneComponents;
@ -112,11 +113,14 @@ public:
bool Load(const char* fn, int lumpnum, const char* buffer, int length) override;
int FindFrame(const char* name, bool nodefault) override;
void RenderFrame(FModelRenderer* renderer, FGameTexture* skin, int frame, int frame2, double inter, int translation, const FTextureID* surfaceskinids, const TArray<VSMatrix>& boneData, int boneStartPosition) override;
int FindFirstFrame(FName name) override;
int FindLastFrame(FName name) override;
double FindFramerate(FName name) override;
void RenderFrame(FModelRenderer* renderer, FGameTexture* skin, int frame, int frame2, double inter, FTranslationID translation, const FTextureID* surfaceskinids, const TArray<VSMatrix>& boneData, int boneStartPosition) override;
void BuildVertexBuffer(FModelRenderer* renderer) override;
void AddSkins(uint8_t* hitlist, const FTextureID* surfaceskinids) override;
const TArray<TRS>* AttachAnimationData() override;
const TArray<VSMatrix> CalculateBones(int frame1, int frame2, double inter, const TArray<TRS>* animationData, DBoneComponents* bones, int index) override;
const TArray<VSMatrix> CalculateBones(int frame1, int frame2, float inter, int frame1_prev, float inter1_prev, int frame2_prev, float inter2_prev, const TArray<TRS>* animationData, DBoneComponents* bones, int index) override;
private:
void LoadGeometry();
@ -129,6 +133,9 @@ private:
void LoadBlendWeights(IQMFileReader& reader, const IQMVertexArray& vertexArray);
int mLumpNum = -1;
TMap<FName, int> NamedAnimations;
TArray<IQMMesh> Meshes;
TArray<IQMTriangle> Triangles;
TArray<IQMAdjacency> Adjacency;

2
src/common/models/model_kvx.h
View file

@ -59,7 +59,7 @@ public:
bool Load(const char * fn, int lumpnum, const char * buffer, int length) override;
void Initialize();
virtual int FindFrame(const char* name, bool nodefault) override;
virtual void RenderFrame(FModelRenderer *renderer, FGameTexture * skin, int frame, int frame2, double inter, int translation, const FTextureID* surfaceskinids, const TArray<VSMatrix>& boneData, int boneStartPosition) override;
virtual void RenderFrame(FModelRenderer *renderer, FGameTexture * skin, int frame, int frame2, double inter, FTranslationID translation, const FTextureID* surfaceskinids, const TArray<VSMatrix>& boneData, int boneStartPosition) override;
virtual void AddSkins(uint8_t *hitlist, const FTextureID* surfaceskinids) override;
FTextureID GetPaletteTexture() const { return mPalette; }
void BuildVertexBuffer(FModelRenderer *renderer) override;

2
src/common/models/model_md2.h
View file

@ -113,7 +113,7 @@ public:
virtual bool Load(const char * fn, int lumpnum, const char * buffer, int length) override;
virtual int FindFrame(const char* name, bool nodefault) override;
virtual void RenderFrame(FModelRenderer *renderer, FGameTexture * skin, int frame, int frame2, double inter, int translation, const FTextureID* surfaceskinids, const TArray<VSMatrix>& boneData, int boneStartPosition) override;
virtual void RenderFrame(FModelRenderer *renderer, FGameTexture * skin, int frame, int frame2, double inter, FTranslationID translation, const FTextureID* surfaceskinids, const TArray<VSMatrix>& boneData, int boneStartPosition) override;
virtual void LoadGeometry();
virtual void AddSkins(uint8_t *hitlist, const FTextureID* surfaceskinids) override;

2
src/common/models/model_md3.h
View file

@ -67,7 +67,7 @@ public:
virtual bool Load(const char * fn, int lumpnum, const char * buffer, int length) override;
virtual int FindFrame(const char* name, bool nodefault) override;
virtual void RenderFrame(FModelRenderer *renderer, FGameTexture * skin, int frame, int frame2, double inter, int translation, const FTextureID* surfaceskinids, const TArray<VSMatrix>& boneData, int boneStartPosition) override;
virtual void RenderFrame(FModelRenderer *renderer, FGameTexture * skin, int frame, int frame2, double inter, FTranslationID translation, const FTextureID* surfaceskinids, const TArray<VSMatrix>& boneData, int boneStartPosition) override;
void LoadGeometry();
void BuildVertexBuffer(FModelRenderer *renderer);
virtual void AddSkins(uint8_t *hitlist, const FTextureID* surfaceskinids) override;

2
src/common/models/model_obj.h
View file

@ -98,7 +98,7 @@ public:
~FOBJModel();
bool Load(const char* fn, int lumpnum, const char* buffer, int length) override;
int FindFrame(const char* name, bool nodefault) override;
void RenderFrame(FModelRenderer* renderer, FGameTexture* skin, int frame, int frame2, double inter, int translation, const FTextureID* surfaceskinids, const TArray<VSMatrix>& boneData, int boneStartPosition) override;
void RenderFrame(FModelRenderer* renderer, FGameTexture* skin, int frame, int frame2, double inter, FTranslationID translation, const FTextureID* surfaceskinids, const TArray<VSMatrix>& boneData, int boneStartPosition) override;
void BuildVertexBuffer(FModelRenderer* renderer) override;
void AddSkins(uint8_t* hitlist, const FTextureID* surfaceskinids) override;
};

2
src/common/models/model_ue1.h
View file

@ -26,7 +26,7 @@ public:
bool Load(const char * fn, int lumpnum, const char * buffer, int length) override;
int FindFrame(const char* name, bool nodefault) override;
void RenderFrame(FModelRenderer *renderer, FGameTexture * skin, int frame, int frame2, double inter, int translation, const FTextureID* surfaceskinids, const TArray<VSMatrix>& boneData, int boneStartPosition) override;
void RenderFrame(FModelRenderer *renderer, FGameTexture * skin, int frame, int frame2, double inter, FTranslationID translation, const FTextureID* surfaceskinids, const TArray<VSMatrix>& boneData, int boneStartPosition) override;
void BuildVertexBuffer(FModelRenderer *renderer) override;
void AddSkins(uint8_t *hitlist, const FTextureID* surfaceskinids) override;
void LoadGeometry();

2
src/common/models/modelrenderer.h
View file

@ -21,7 +21,7 @@ public:
virtual void EndDrawHUDModel(FRenderStyle style, FSpriteModelFrame *smf) = 0;
virtual void SetInterpolation(double interpolation) = 0;
virtual void SetMaterial(FGameTexture *skin, bool clampNoFilter, int translation) = 0;
virtual void SetMaterial(FGameTexture *skin, bool clampNoFilter, FTranslationID translation) = 0;
virtual void DrawArrays(int start, int count) = 0;
virtual void DrawElements(int numIndices, size_t offset) = 0;
virtual int SetupFrame(FModel* model, unsigned int frame1, unsigned int frame2, unsigned int size, const TArray<VSMatrix>& bones, int boneStartIndex) { return -1; };

85
src/common/models/models_iqm.cpp
View file

@ -137,9 +137,12 @@ bool IQMModel::Load(const char* path, int lumpnum, const char* buffer, int lengt
}
reader.SeekTo(ofs_anims);
for (IQMAnim& anim : Anims)
for(int i = 0; i < Anims.Size(); i++)
{
IQMAnim& anim = Anims[i];
anim.Name = reader.ReadName(text);
NamedAnimations.Insert(anim.Name, i);
anim.FirstFrame = reader.ReadUInt32();
anim.NumFrames = reader.ReadUInt32();
anim.Framerate = reader.ReadFloat();
@ -445,7 +448,28 @@ int IQMModel::FindFrame(const char* name, bool nodefault)
return FErr_NotFound;
}
void IQMModel::RenderFrame(FModelRenderer* renderer, FGameTexture* skin, int frame1, int frame2, double inter, int translation, const FTextureID* surfaceskinids, const TArray<VSMatrix>& boneData, int boneStartPosition)
int IQMModel::FindFirstFrame(FName name)
{
int * nam = NamedAnimations.CheckKey(name);
if(nam) return Anims[*nam].FirstFrame;
return FErr_NotFound;
}
int IQMModel::FindLastFrame(FName name)
{
int * nam = NamedAnimations.CheckKey(name);
if(nam) return Anims[*nam].FirstFrame + Anims[*nam].NumFrames;
return FErr_NotFound;
}
double IQMModel::FindFramerate(FName name)
{
int * nam = NamedAnimations.CheckKey(name);
if(nam) return Anims[*nam].Framerate;
return FErr_NotFound;
}
void IQMModel::RenderFrame(FModelRenderer* renderer, FGameTexture* skin, int frame1, int frame2, double inter, FTranslationID translation, const FTextureID* surfaceskinids, const TArray<VSMatrix>& boneData, int boneStartPosition)
{
renderer->SetupFrame(this, 0, 0, NumVertices, boneData, boneStartPosition);
@ -517,7 +541,26 @@ const TArray<TRS>* IQMModel::AttachAnimationData()
return &TRSData;
}
const TArray<VSMatrix> IQMModel::CalculateBones(int frame1, int frame2, double inter, const TArray<TRS>* animationData, DBoneComponents* boneComponentData, int index)
static TRS InterpolateBone(const TRS &from, const TRS &to, float t, float invt)
{
TRS bone;
bone.translation = from.translation * invt + to.translation * t;
bone.rotation = from.rotation * invt;
if ((bone.rotation | to.rotation * t) < 0)
{
bone.rotation.X *= -1; bone.rotation.Y *= -1; bone.rotation.Z *= -1; bone.rotation.W *= -1;
}
bone.rotation += to.rotation * t;
bone.rotation.MakeUnit();
bone.scaling = from.scaling * invt + to.scaling * t;
return bone;
}
const TArray<VSMatrix> IQMModel::CalculateBones(int frame1, int frame2, float inter, int frame1_prev, float inter1_prev, int frame2_prev, float inter2_prev, const TArray<TRS>* animationData, DBoneComponents* boneComponentData, int index)
{
const TArray<TRS>& animationFrames = animationData ? *animationData : TRSData;
if (Joints.Size() > 0)
@ -534,8 +577,13 @@ const TArray<VSMatrix> IQMModel::CalculateBones(int frame1, int frame2, double i
int offset1 = frame1 * numbones;
int offset2 = frame2 * numbones;
float t = (float)inter;
float invt = 1.0f - t;
int offset1_1 = frame1_prev * numbones;
int offset2_1 = frame2_prev * numbones;
float invt = 1.0f - inter;
float invt1 = 1.0f - inter1_prev;
float invt2 = 1.0f - inter2_prev;
float swapYZ[16] = { 0.0f };
swapYZ[0 + 0 * 4] = 1.0f;
@ -547,19 +595,26 @@ const TArray<VSMatrix> IQMModel::CalculateBones(int frame1, int frame2, double i
TArray<bool> modifiedBone(numbones, true);
for (int i = 0; i < numbones; i++)
{
TRS bone;
TRS from = animationFrames[offset1 + i];
TRS to = animationFrames[offset2 + i];
TRS prev;
bone.translation = from.translation * invt + to.translation * t;
bone.rotation = from.rotation * invt;
if ((bone.rotation | to.rotation * t) < 0)
if(frame1 >= 0 && (frame1_prev >= 0 || inter1_prev < 0))
{
bone.rotation.X *= -1; bone.rotation.Y *= -1; bone.rotation.Z *= -1; bone.rotation.W *= -1;
prev = inter1_prev < 0 ? animationFrames[offset1 + i] : InterpolateBone(animationFrames[offset1_1 + i], animationFrames[offset1 + i], inter1_prev, invt1);
}
TRS next;
if(frame2 >= 0 && (frame2_prev >= 0 || inter2_prev < 0))
{
next = inter2_prev < 0 ? animationFrames[offset2 + i] : InterpolateBone(animationFrames[offset2_1 + i], animationFrames[offset2 + i], inter2_prev, invt2);
}
TRS bone;
if(frame1 >= 0 || inter < 0)
{
bone = inter < 0 ? animationFrames[offset1 + i] : InterpolateBone(prev, next , inter, invt);
}
bone.rotation += to.rotation * t;
bone.rotation.MakeUnit();
bone.scaling = from.scaling * invt + to.scaling * t;
if (Joints[i].Parent >= 0 && modifiedBone[Joints[i].Parent])
{

2
src/common/models/models_md2.cpp
View file

@ -364,7 +364,7 @@ int FDMDModel::FindFrame(const char* name, bool nodefault)
//
//===========================================================================
void FDMDModel::RenderFrame(FModelRenderer *renderer, FGameTexture * skin, int frameno, int frameno2, double inter, int translation, const FTextureID*, const TArray<VSMatrix>& boneData, int boneStartPosition)
void FDMDModel::RenderFrame(FModelRenderer *renderer, FGameTexture * skin, int frameno, int frameno2, double inter, FTranslationID translation, const FTextureID*, const TArray<VSMatrix>& boneData, int boneStartPosition)
{
if (frameno >= info.numFrames || frameno2 >= info.numFrames) return;

2
src/common/models/models_md3.cpp
View file

@ -345,7 +345,7 @@ int FMD3Model::FindFrame(const char* name, bool nodefault)
//
//===========================================================================
void FMD3Model::RenderFrame(FModelRenderer *renderer, FGameTexture * skin, int frameno, int frameno2, double inter, int translation, const FTextureID* surfaceskinids, const TArray<VSMatrix>& boneData, int boneStartPosition)
void FMD3Model::RenderFrame(FModelRenderer *renderer, FGameTexture * skin, int frameno, int frameno2, double inter, FTranslationID translation, const FTextureID* surfaceskinids, const TArray<VSMatrix>& boneData, int boneStartPosition)
{
if ((unsigned)frameno >= Frames.Size() || (unsigned)frameno2 >= Frames.Size()) return;

2
src/common/models/models_obj.cpp
View file

@ -630,7 +630,7 @@ int FOBJModel::FindFrame(const char* name, bool nodefault)
* @param inter The amount to interpolate the two frames.
* @param translation The translation for the skin
*/
void FOBJModel::RenderFrame(FModelRenderer *renderer, FGameTexture * skin, int frameno, int frameno2, double inter, int translation, const FTextureID* surfaceskinids, const TArray<VSMatrix>& boneData, int boneStartPosition)
void FOBJModel::RenderFrame(FModelRenderer *renderer, FGameTexture * skin, int frameno, int frameno2, double inter, FTranslationID translation, const FTextureID* surfaceskinids, const TArray<VSMatrix>& boneData, int boneStartPosition)
{
// Prevent the model from rendering if the frame number is < 0
if (frameno < 0 || frameno2 < 0) return;

2
src/common/models/models_ue1.cpp
View file

@ -232,7 +232,7 @@ int FUE1Model::FindFrame(const char* name, bool nodefault)
return index;
}
void FUE1Model::RenderFrame( FModelRenderer *renderer, FGameTexture *skin, int frame, int frame2, double inter, int translation, const FTextureID* surfaceskinids, const TArray<VSMatrix>& boneData, int boneStartPosition)
void FUE1Model::RenderFrame( FModelRenderer *renderer, FGameTexture *skin, int frame, int frame2, double inter, FTranslationID translation, const FTextureID* surfaceskinids, const TArray<VSMatrix>& boneData, int boneStartPosition)
{
// the moment of magic
if ( (frame < 0) || (frame2 < 0) || (frame >= numFrames) || (frame2 >= numFrames) ) return;

2
src/common/models/models_voxel.cpp
View file

@ -400,7 +400,7 @@ float FVoxelModel::getAspectFactor(float stretch)
//
//===========================================================================
void FVoxelModel::RenderFrame(FModelRenderer *renderer, FGameTexture * skin, int frame, int frame2, double inter, int translation, const FTextureID*, const TArray<VSMatrix>& boneData, int boneStartPosition)
void FVoxelModel::RenderFrame(FModelRenderer *renderer, FGameTexture * skin, int frame, int frame2, double inter, FTranslationID translation, const FTextureID*, const TArray<VSMatrix>& boneData, int boneStartPosition)
{
renderer->SetMaterial(skin, true, translation);
renderer->SetupFrame(this, 0, 0, 0, {}, -1);

16
src/common/objects/dobjgc.h
View file

@ -210,8 +210,8 @@ class TObjPtr
{
union
{
T pp;
DObject *o;
mutable T pp;
mutable DObject *o;
};
public:
@ -273,16 +273,22 @@ public:
{
return GC::ReadBarrier(pp);
}
constexpr bool operator!=(T u) noexcept
constexpr const T operator->() const noexcept
{
return GC::ReadBarrier(pp);
}
constexpr bool operator!=(T u) const noexcept
{
return GC::ReadBarrier(o) != u;
}
constexpr bool operator==(T u) noexcept
constexpr bool operator==(T u) const noexcept
{
return GC::ReadBarrier(o) == u;
}
constexpr bool operator==(TObjPtr<T> u) noexcept
constexpr bool operator==(TObjPtr<T> u) const noexcept
{
return ForceGet() == u.ForceGet();
}

9
src/common/platform/posix/sdl/i_system.cpp
View file

@ -218,20 +218,21 @@ void RedrawProgressBar(int CurPos, int MaxPos)
CleanProgressBar();
struct winsize sizeOfWindow;
ioctl(STDOUT_FILENO, TIOCGWINSZ, &sizeOfWindow);
int windowColClamped = std::min((int)sizeOfWindow.ws_col, 512);
double progVal = std::clamp((double)CurPos / (double)MaxPos,0.0,1.0);
int curProgVal = std::clamp(int(sizeOfWindow.ws_col * progVal),0,(int)sizeOfWindow.ws_col);
int curProgVal = std::clamp(int(windowColClamped * progVal),0,windowColClamped);
char progressBuffer[512];
memset(progressBuffer,'.',512);
progressBuffer[sizeOfWindow.ws_col - 1] = 0;
progressBuffer[windowColClamped - 1] = 0;
int lengthOfStr = 0;
while (curProgVal-- > 0)
{
progressBuffer[lengthOfStr++] = '=';
if (lengthOfStr >= sizeOfWindow.ws_col - 1) break;
if (lengthOfStr >= windowColClamped - 1) break;
}
fprintf(stdout, "\0337\033[%d;%dH\033[2K[%s\033[%d;%dH]\0338", sizeOfWindow.ws_row, 0, progressBuffer, sizeOfWindow.ws_row, sizeOfWindow.ws_col);
fprintf(stdout, "\0337\033[%d;%dH\033[2K[%s\033[%d;%dH]\0338", sizeOfWindow.ws_row, 0, progressBuffer, sizeOfWindow.ws_row, windowColClamped);
fflush(stdout);
ProgressBarCurPos = CurPos;
ProgressBarMaxPos = MaxPos;

31
src/common/platform/win32/i_system.cpp
View file

@ -830,37 +830,6 @@ FString I_GetLongPathName(const FString &shortpath)
return longpath;
}
#ifdef _USING_V110_SDK71_
//==========================================================================
//
// _stat64i32
//
// Work around an issue where stat() function doesn't work
// with Windows XP compatible toolset.
// It uses GetFileInformationByHandleEx() which requires Windows Vista.
//
//==========================================================================
int _wstat64i32(const wchar_t *path, struct _stat64i32 *buffer)
{
WIN32_FILE_ATTRIBUTE_DATA data;
if(!GetFileAttributesExW(path, GetFileExInfoStandard, &data))
return -1;
buffer->st_ino = 0;
buffer->st_mode = ((data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) ? S_IFDIR : S_IFREG)|
((data.dwFileAttributes & FILE_ATTRIBUTE_READONLY) ? S_IREAD : S_IREAD|S_IWRITE);
buffer->st_dev = buffer->st_rdev = 0;
buffer->st_nlink = 1;
buffer->st_uid = 0;
buffer->st_gid = 0;
buffer->st_size = data.nFileSizeLow;
buffer->st_atime = (*(uint64_t*)&data.ftLastAccessTime) / 10000000 - 11644473600LL;
buffer->st_mtime = (*(uint64_t*)&data.ftLastWriteTime) / 10000000 - 11644473600LL;
buffer->st_ctime = (*(uint64_t*)&data.ftCreationTime) / 10000000 - 11644473600LL;
return 0;
}
#endif
struct NumaNode
{

6
src/common/rendering/hwrenderer/data/hw_renderstate.h
View file

@ -549,6 +549,12 @@ public:
SetMaterial(mat, clampmode, translation, overrideshader);
}
void SetMaterial(FGameTexture* tex, EUpscaleFlags upscalemask, int scaleflags, int clampmode, FTranslationID translation, int overrideshader)
{
SetMaterial(tex, upscalemask, scaleflags, clampmode, translation.index(), overrideshader);
}
void SetClipSplit(float bottom, float top)
{
mSurfaceUniforms.uClipSplit.X = bottom;

609
src/common/scripting/backend/codegen.cpp
View file

@ -43,11 +43,13 @@
#include "texturemanager.h"
#include "m_random.h"
#include "v_font.h"
#include "palettecontainer.h"
extern FRandom pr_exrandom;
FMemArena FxAlloc(65536);
CompileEnvironment compileEnvironment;
FTranslationID R_FindCustomTranslation(FName name);
struct FLOP
{
@ -161,6 +163,12 @@ void FCompileContext::CheckReturn(PPrototype *proto, FScriptPosition &pos)
PType* expected = ReturnProto->ReturnTypes[i];
PType* actual = proto->ReturnTypes[i];
if (swapped) std::swap(expected, actual);
// this must pass for older ZScripts.
if (Version < MakeVersion(4, 12, 0))
{
if (expected == TypeTranslationID) expected = TypeSInt32;
if (actual == TypeTranslationID) actual = TypeSInt32;
}
if (expected != actual && !AreCompatiblePointerTypes(expected, actual))
{ // Incompatible
@ -916,6 +924,17 @@ FxExpression *FxBoolCast::Resolve(FCompileContext &ctx)
ExpEmit FxBoolCast::Emit(VMFunctionBuilder *build)
{
ExpEmit from = basex->Emit(build);
if(from.Konst && from.RegType == REGT_INT)
{ // this is needed here because the int const assign optimization returns a constant
ExpEmit to;
to.Konst = true;
to.RegType = REGT_INT;
to.RegNum = build->GetConstantInt(!!build->FindConstantInt(from.RegNum));
return to;
}
assert(!from.Konst);
assert(basex->ValueType->GetRegType() == REGT_INT || basex->ValueType->GetRegType() == REGT_FLOAT || basex->ValueType->GetRegType() == REGT_POINTER);
@ -982,6 +1001,19 @@ FxExpression *FxIntCast::Resolve(FCompileContext &ctx)
if (basex->ValueType->GetRegType() == REGT_INT)
{
if (basex->ValueType == TypeTranslationID)
{
// translation IDs must be entirely incompatible with ints, not even allowing an explicit conversion,
// but since the type was only introduced in version 4.12, older ZScript versions must allow this conversion.
if (ctx.Version < MakeVersion(4, 12, 0))
{
FxExpression* x = basex;
x->ValueType = ValueType;
basex = nullptr;
delete this;
return x;
}
}
if (basex->ValueType->isNumeric() || Explicit) // names can be converted to int, but only with an explicit type cast.
{
FxExpression *x = basex;
@ -995,7 +1027,7 @@ FxExpression *FxIntCast::Resolve(FCompileContext &ctx)
// Ugh. This should abort, but too many mods fell into this logic hole somewhere, so this serious error needs to be reduced to a warning. :(
// At least in ZScript, MSG_OPTERROR always means to report an error, not a warning so the problem only exists in DECORATE.
if (!basex->isConstant())
ScriptPosition.Message(MSG_OPTERROR, "Numeric type expected, got a name");
ScriptPosition.Message(MSG_OPTERROR, "Numeric type expected, got a %s", basex->ValueType->DescriptiveName());
else ScriptPosition.Message(MSG_OPTERROR, "Numeric type expected, got \"%s\"", static_cast<FxConstant*>(basex)->GetValue().GetName().GetChars());
FxExpression * x = new FxConstant(0, ScriptPosition);
delete this;
@ -1116,7 +1148,8 @@ FxExpression *FxFloatCast::Resolve(FCompileContext &ctx)
{
// Ugh. This should abort, but too many mods fell into this logic hole somewhere, so this seroious error needs to be reduced to a warning. :(
// At least in ZScript, MSG_OPTERROR always means to report an error, not a warning so the problem only exists in DECORATE.
if (!basex->isConstant()) ScriptPosition.Message(MSG_OPTERROR, "Numeric type expected, got a name");
if (!basex->isConstant())
ScriptPosition.Message(MSG_OPTERROR, "Numeric type expected, got a %s", basex->ValueType->DescriptiveName());
else ScriptPosition.Message(MSG_OPTERROR, "Numeric type expected, got \"%s\"", static_cast<FxConstant*>(basex)->GetValue().GetName().GetChars());
FxExpression *x = new FxConstant(0.0, ScriptPosition);
delete this;
@ -1140,7 +1173,15 @@ FxExpression *FxFloatCast::Resolve(FCompileContext &ctx)
ExpEmit FxFloatCast::Emit(VMFunctionBuilder *build)
{
ExpEmit from = basex->Emit(build);
//assert(!from.Konst);
if(from.Konst && from.RegType == REGT_INT)
{ // this is needed here because the int const assign optimization returns a constant
ExpEmit to;
to.Konst = true;
to.RegType = REGT_FLOAT;
to.RegNum = build->GetConstantFloat(build->FindConstantInt(from.RegNum));
return to;
}
assert(basex->ValueType->GetRegType() == REGT_INT);
from.Free(build);
ExpEmit to(build, REGT_FLOAT);
@ -1515,6 +1556,107 @@ ExpEmit FxSoundCast::Emit(VMFunctionBuilder *build)
return to;
}
//==========================================================================
//
//
//
//==========================================================================
FxTranslationCast::FxTranslationCast(FxExpression* x)
: FxExpression(EFX_TranslationCast, x->ScriptPosition)
{
basex = x;
ValueType = TypeTranslationID;
}
//==========================================================================
//
//
//
//==========================================================================
FxTranslationCast::~FxTranslationCast()
{
SAFE_DELETE(basex);
}
//==========================================================================
//
//
//
//==========================================================================
FxExpression* FxTranslationCast::Resolve(FCompileContext& ctx)
{
CHECKRESOLVED();
SAFE_RESOLVE(basex, ctx);
if (basex->ValueType->isInt())
{
// 0 is a valid constant for translations, meaning 'no translation at all'. note that this conversion ONLY allows a constant!
if (basex->isConstant() && static_cast<FxConstant*>(basex)->GetValue().GetInt() == 0)
{
FxExpression* x = basex;
x->ValueType = TypeTranslationID;
basex = nullptr;
delete this;
return x;
}
if (ctx.Version < MakeVersion(4, 12, 0))
{
// only allow this conversion as a fallback
FxExpression* x = basex;
x->ValueType = TypeTranslationID;
basex = nullptr;
delete this;
return x;
}
}
else if (basex->ValueType == TypeString || basex->ValueType == TypeName)
{
if (basex->isConstant())
{
ExpVal constval = static_cast<FxConstant*>(basex)->GetValue();
FxExpression* x = new FxConstant(R_FindCustomTranslation(constval.GetName()), ScriptPosition);
x->ValueType = TypeTranslationID;
delete this;
return x;
}
else if (basex->ValueType == TypeString)
{
basex = new FxNameCast(basex, true);
basex = basex->Resolve(ctx);
}
return this;
}
ScriptPosition.Message(MSG_ERROR, "Cannot convert to translation ID");
delete this;
return nullptr;
}
//==========================================================================
//
//
//
//==========================================================================
ExpEmit FxTranslationCast::Emit(VMFunctionBuilder* build)
{
ExpEmit to(build, REGT_POINTER);
VMFunction* callfunc;
auto sym = FindBuiltinFunction(NAME_BuiltinFindTranslation);
assert(sym);
callfunc = sym->Variants[0].Implementation;
FunctionCallEmitter emitters(callfunc);
emitters.AddParameter(build, basex);
emitters.AddReturn(REGT_INT);
return emitters.EmitCall(build);
}
//==========================================================================
//
//
@ -1747,6 +1889,14 @@ FxExpression *FxTypeCast::Resolve(FCompileContext &ctx)
delete this;
return x;
}
else if (ValueType == TypeTranslationID)
{
FxExpression* x = new FxTranslationCast(basex);
x = x->Resolve(ctx);
basex = nullptr;
delete this;
return x;
}
else if (ValueType == TypeColor)
{
FxExpression *x = new FxColorCast(basex);
@ -1973,7 +2123,17 @@ ExpEmit FxMinusSign::Emit(VMFunctionBuilder *build)
{
//assert(ValueType == Operand->ValueType);
ExpEmit from = Operand->Emit(build);
if(from.Konst && from.RegType == REGT_INT)
{ // this is needed here because the int const assign optimization returns a constant
ExpEmit to;
to.Konst = true;
to.RegType = REGT_INT;
to.RegNum = build->GetConstantInt(-build->FindConstantInt(from.RegNum));
return to;
}
ExpEmit to;
assert(from.Konst == 0);
assert(ValueType->GetRegCount() == from.RegCount);
// Do it in-place, unless a local variable
@ -2093,6 +2253,16 @@ ExpEmit FxUnaryNotBitwise::Emit(VMFunctionBuilder *build)
{
assert(Operand->ValueType->GetRegType() == REGT_INT);
ExpEmit from = Operand->Emit(build);
if(from.Konst && from.RegType == REGT_INT)
{ // this is needed here because the int const assign optimization returns a constant
ExpEmit to;
to.Konst = true;
to.RegType = REGT_INT;
to.RegNum = build->GetConstantInt(~build->FindConstantInt(from.RegNum));
return to;
}
from.Free(build);
ExpEmit to(build, REGT_INT);
assert(!from.Konst);
@ -2164,6 +2334,16 @@ ExpEmit FxUnaryNotBoolean::Emit(VMFunctionBuilder *build)
assert(Operand->ValueType == TypeBool);
assert(ValueType == TypeBool || IsInteger()); // this may have been changed by an int cast.
ExpEmit from = Operand->Emit(build);
if(from.Konst && from.RegType == REGT_INT)
{ // this is needed here because the int const assign optimization returns a constant
ExpEmit to;
to.Konst = true;
to.RegType = REGT_INT;
to.RegNum = build->GetConstantInt(!build->FindConstantInt(from.RegNum));
return to;
}
from.Free(build);
ExpEmit to(build, REGT_INT);
assert(!from.Konst);
@ -2548,7 +2728,7 @@ FxExpression *FxAssign::Resolve(FCompileContext &ctx)
{
FArgumentList args;
args.Push(Right);
auto call = new FxMemberFunctionCall(Base, NAME_Copy, args, ScriptPosition);
auto call = new FxMemberFunctionCall(Base, NAME_Copy, std::move(args), ScriptPosition);
Right = Base = nullptr;
delete this;
return call->Resolve(ctx);
@ -2576,7 +2756,7 @@ FxExpression *FxAssign::Resolve(FCompileContext &ctx)
{
FArgumentList args;
args.Push(Right);
auto call = new FxMemberFunctionCall(Base, NAME_Copy, args, ScriptPosition);
auto call = new FxMemberFunctionCall(Base, NAME_Copy, std::move(args), ScriptPosition);
Right = Base = nullptr;
delete this;
return call->Resolve(ctx);
@ -3646,6 +3826,16 @@ FxExpression *FxCompareRel::Resolve(FCompileContext& ctx)
delete left;
left = x;
}
else if (left->IsNumeric() && right->ValueType == TypeTranslationID && ctx.Version < MakeVersion(4, 12))
{
right = new FxTypeCast(right, TypeSInt32, true);
SAFE_RESOLVE(right, ctx);
}
else if (right->IsNumeric() && left->ValueType == TypeTranslationID && ctx.Version < MakeVersion(4, 12))
{
left = new FxTypeCast(left, TypeSInt32, true);
SAFE_RESOLVE(left, ctx);
}
if (left->ValueType == TypeString || right->ValueType == TypeString)
{
@ -3938,6 +4128,17 @@ FxExpression *FxCompareEq::Resolve(FCompileContext& ctx)
delete left;
left = x;
}
else if (left->IsNumeric() && right->ValueType == TypeTranslationID && ctx.Version < MakeVersion(4, 12))
{
right = new FxIntCast(right, true, true);
SAFE_RESOLVE(right, ctx);
}
else if (right->IsNumeric() && left->ValueType == TypeTranslationID && ctx.Version < MakeVersion(4, 12))
{
left = new FxTypeCast(left, TypeSInt32, true);
SAFE_RESOLVE(left, ctx);
}
// Special cases: Compare strings and names with names, sounds, colors, state labels and class types.
// These are all types a string can be implicitly cast into, so for convenience, so they should when doing a comparison.
if ((left->ValueType == TypeString || left->ValueType == TypeName) &&
@ -4590,6 +4791,7 @@ ExpEmit FxConcat::Emit(VMFunctionBuilder *build)
else if (left->ValueType == TypeColor) cast = CAST_Co2S;
else if (left->ValueType == TypeSpriteID) cast = CAST_SID2S;
else if (left->ValueType == TypeTextureID) cast = CAST_TID2S;
else if (left->ValueType == TypeTranslationID) cast = CAST_U2S;
else if (op1.RegType == REGT_POINTER) cast = CAST_P2S;
else if (op1.RegType == REGT_INT) cast = CAST_I2S;
else assert(false && "Bad type for string concatenation");
@ -4623,6 +4825,7 @@ ExpEmit FxConcat::Emit(VMFunctionBuilder *build)
else if (right->ValueType == TypeColor) cast = CAST_Co2S;
else if (right->ValueType == TypeSpriteID) cast = CAST_SID2S;
else if (right->ValueType == TypeTextureID) cast = CAST_TID2S;
else if (right->ValueType == TypeTranslationID) cast = CAST_U2S;
else if (op2.RegType == REGT_POINTER) cast = CAST_P2S;
else if (op2.RegType == REGT_INT) cast = CAST_I2S;
else assert(false && "Bad type for string concatenation");
@ -5094,11 +5297,24 @@ FxExpression *FxConditional::Resolve(FCompileContext& ctx)
ValueType = truex->ValueType;
else if (falsex->IsPointer() && truex->ValueType == TypeNullPtr)
ValueType = falsex->ValueType;
// translation IDs need a bit of glue for compatibility and the 0 literal.
else if (truex->IsInteger() && falsex->ValueType == TypeTranslationID)
{
truex = new FxTranslationCast(truex);
truex = truex->Resolve(ctx);
ValueType = ctx.Version < MakeVersion(4, 12, 0)? TypeSInt32 : TypeTranslationID;
}
else if (falsex->IsInteger() && truex->ValueType == TypeTranslationID)
{
falsex = new FxTranslationCast(falsex);
falsex = falsex->Resolve(ctx);
ValueType = ctx.Version < MakeVersion(4, 12, 0) ? TypeSInt32 : TypeTranslationID;
}
else
ValueType = TypeVoid;
//else if (truex->ValueType != falsex->ValueType)
if (ValueType->GetRegType() == REGT_NIL)
if (truex == nullptr || falsex == nullptr || ValueType->GetRegType() == REGT_NIL)
{
ScriptPosition.Message(MSG_ERROR, "Incompatible types for ?: operator");
delete this;
@ -7986,7 +8202,7 @@ static bool CheckFunctionCompatiblity(FScriptPosition &ScriptPosition, PFunction
//
//==========================================================================
FxFunctionCall::FxFunctionCall(FName methodname, FName rngname, FArgumentList &args, const FScriptPosition &pos)
FxFunctionCall::FxFunctionCall(FName methodname, FName rngname, FArgumentList &&args, const FScriptPosition &pos)
: FxExpression(EFX_FunctionCall, pos)
{
MethodName = methodname;
@ -8230,6 +8446,7 @@ FxExpression *FxFunctionCall::Resolve(FCompileContext& ctx)
MethodName == NAME_Name ? TypeName :
MethodName == NAME_SpriteID ? TypeSpriteID :
MethodName == NAME_TextureID ? TypeTextureID :
MethodName == NAME_TranslationID ? TypeTranslationID :
MethodName == NAME_State ? TypeState :
MethodName == NAME_Color ? TypeColor : (PType*)TypeSound;
@ -8412,7 +8629,7 @@ FxExpression *FxFunctionCall::Resolve(FCompileContext& ctx)
//
//==========================================================================
FxMemberFunctionCall::FxMemberFunctionCall(FxExpression *self, FName methodname, FArgumentList &args, const FScriptPosition &pos)
FxMemberFunctionCall::FxMemberFunctionCall(FxExpression *self, FName methodname, FArgumentList &&args, const FScriptPosition &pos)
: FxExpression(EFX_MemberFunctionCall, pos)
{
Self = self;
@ -8957,7 +9174,7 @@ FxExpression *FxMemberFunctionCall::Resolve(FCompileContext& ctx)
if (a->IsMap())
{
// Copy and Move must turn their parameter into a pointer to the backing struct type.
auto o = static_cast<PMapIterator*>(a->ValueType);
auto o = static_cast<PMap*>(a->ValueType);
auto backingtype = o->BackingType;
if (mapKeyType != o->KeyType || mapValueType != o->ValueType)
{
@ -11165,12 +11382,14 @@ ExpEmit FxForLoop::Emit(VMFunctionBuilder *build)
//
//==========================================================================
FxForEachLoop::FxForEachLoop(FName vn, FxExpression* arrayvar, FxExpression* arrayvar2, FxExpression* code, const FScriptPosition& pos)
: FxLoopStatement(EFX_ForEachLoop, pos), loopVarName(vn), Array(arrayvar), Array2(arrayvar2), Code(code)
FxForEachLoop::FxForEachLoop(FName vn, FxExpression* arrayvar, FxExpression* arrayvar2, FxExpression* arrayvar3, FxExpression* arrayvar4, FxExpression* code, const FScriptPosition& pos)
: FxLoopStatement(EFX_ForEachLoop, pos), loopVarName(vn), Array(arrayvar), Array2(arrayvar2), Array3(arrayvar3), Array4(arrayvar4), Code(code)
{
ValueType = TypeVoid;
if (Array != nullptr) Array->NeedResult = false;
if (Array2 != nullptr) Array2->NeedResult = false;
if (Array3 != nullptr) Array3->NeedResult = false;
if (Array4 != nullptr) Array4->NeedResult = false;
if (Code != nullptr) Code->NeedResult = false;
}
@ -11178,57 +11397,347 @@ FxForEachLoop::~FxForEachLoop()
{
SAFE_DELETE(Array);
SAFE_DELETE(Array2);
SAFE_DELETE(Array3);
SAFE_DELETE(Array4);
SAFE_DELETE(Code);
}
extern bool IsGameSpecificForEachLoop(FxForEachLoop *);
extern FxExpression * ResolveGameSpecificForEachLoop(FxForEachLoop *);
FxExpression* FxForEachLoop::DoResolve(FCompileContext& ctx)
{
CHECKRESOLVED();
SAFE_RESOLVE(Array, ctx);
SAFE_RESOLVE(Array2, ctx);
SAFE_RESOLVE(Array3, ctx);
SAFE_RESOLVE(Array4, ctx);
// Instead of writing a new code generator for this, convert this into
//
// int @size = array.Size();
// for(int @i = 0; @i < @size; @i++)
// {
// let var = array[i];
// body
// }
// and let the existing 'for' loop code sort out the rest.
FName sizevar = "@size";
FName itvar = "@i";
FArgumentList al;
auto block = new FxCompoundStatement(ScriptPosition);
auto arraysize = new FxMemberFunctionCall(Array, NAME_Size, al, ScriptPosition);
auto size = new FxLocalVariableDeclaration(TypeSInt32, sizevar, arraysize, 0, ScriptPosition);
auto it = new FxLocalVariableDeclaration(TypeSInt32, itvar, new FxConstant(0, ScriptPosition), 0, ScriptPosition);
block->Add(size);
block->Add(it);
if(Array->ValueType->isMap() || Array->ValueType->isMapIterator())
{
auto mapLoop = new FxTwoArgForEachLoop(NAME_None, loopVarName, Array, Array2, Array3, Array4, Code, ScriptPosition);
Array = Array2 = Array3 = Array4 = Code = nullptr;
delete this;
return mapLoop->Resolve(ctx);
}
else if(IsGameSpecificForEachLoop(this))
{
return ResolveGameSpecificForEachLoop(this)->Resolve(ctx);
}
else
{
// Instead of writing a new code generator for this, convert this into
//
// int @size = array.Size();
// for(int @i = 0; @i < @size; @i++)
// {
// let var = array[i];
// body
// }
// and let the existing 'for' loop code sort out the rest.
auto cit = new FxLocalVariable(it, ScriptPosition);
auto csiz = new FxLocalVariable(size, ScriptPosition);
auto comp = new FxCompareRel('<', cit, csiz); // new FxIdentifier(itvar, ScriptPosition), new FxIdentifier(sizevar, ScriptPosition));
FName sizevar = "@size";
FName itvar = "@i";
auto iit = new FxLocalVariable(it, ScriptPosition);
auto bump = new FxPreIncrDecr(iit, TK_Incr);
auto block = new FxCompoundStatement(ScriptPosition);
auto arraysize = new FxMemberFunctionCall(Array, NAME_Size, {}, ScriptPosition);
auto size = new FxLocalVariableDeclaration(TypeSInt32, sizevar, arraysize, 0, ScriptPosition);
auto it = new FxLocalVariableDeclaration(TypeSInt32, itvar, new FxConstant(0, ScriptPosition), 0, ScriptPosition);
block->Add(size);
block->Add(it);
auto ait = new FxLocalVariable(it, ScriptPosition);
auto access = new FxArrayElement(Array2, ait, true); // Note: Array must be a separate copy because these nodes cannot share the same element.
auto cit = new FxLocalVariable(it, ScriptPosition);
auto csiz = new FxLocalVariable(size, ScriptPosition);
auto comp = new FxCompareRel('<', cit, csiz); // new FxIdentifier(itvar, ScriptPosition), new FxIdentifier(sizevar, ScriptPosition));
auto assign = new FxLocalVariableDeclaration(TypeAuto, loopVarName, access, 0, ScriptPosition);
auto body = new FxCompoundStatement(ScriptPosition);
body->Add(assign);
body->Add(Code);
auto forloop = new FxForLoop(nullptr, comp, bump, body, ScriptPosition);
block->Add(forloop);
Array2 = Array = nullptr;
Code = nullptr;
delete this;
return block->Resolve(ctx);
auto iit = new FxLocalVariable(it, ScriptPosition);
auto bump = new FxPreIncrDecr(iit, TK_Incr);
auto ait = new FxLocalVariable(it, ScriptPosition);
auto access = new FxArrayElement(Array2, ait, true); // Note: Array must be a separate copy because these nodes cannot share the same element.
auto assign = new FxLocalVariableDeclaration(TypeAuto, loopVarName, access, 0, ScriptPosition);
auto body = new FxCompoundStatement(ScriptPosition);
body->Add(assign);
body->Add(Code);
auto forloop = new FxForLoop(nullptr, comp, bump, body, ScriptPosition);
block->Add(forloop);
Array2 = Array = nullptr;
Code = nullptr;
delete this;
return block->Resolve(ctx);
}
}
//==========================================================================
//
// FxMapForEachLoop
//
//==========================================================================
FxTwoArgForEachLoop::FxTwoArgForEachLoop(FName kv, FName vv, FxExpression* mapexpr, FxExpression* mapexpr2, FxExpression* mapexpr3, FxExpression* mapexpr4, FxExpression* code, const FScriptPosition& pos)
: FxExpression(EFX_TwoArgForEachLoop,pos), keyVarName(kv), valueVarName(vv), MapExpr(mapexpr), MapExpr2(mapexpr2), MapExpr3(mapexpr3), MapExpr4(mapexpr4), Code(code)
{
ValueType = TypeVoid;
if (MapExpr != nullptr) MapExpr->NeedResult = false;
if (MapExpr2 != nullptr) MapExpr2->NeedResult = false;
if (MapExpr3 != nullptr) MapExpr3->NeedResult = false;
if (MapExpr4 != nullptr) MapExpr4->NeedResult = false;
if (Code != nullptr) Code->NeedResult = false;
}
FxTwoArgForEachLoop::~FxTwoArgForEachLoop()
{
SAFE_DELETE(MapExpr);
SAFE_DELETE(MapExpr2);
SAFE_DELETE(MapExpr3);
SAFE_DELETE(MapExpr4);
SAFE_DELETE(Code);
}
extern bool HasGameSpecificTwoArgForEachLoopTypeNames();
extern const char * GetGameSpecificTwoArgForEachLoopTypeNames();
extern bool IsGameSpecificTwoArgForEachLoop(FxTwoArgForEachLoop *);
extern FxExpression * ResolveGameSpecificTwoArgForEachLoop(FxTwoArgForEachLoop *);
FxExpression *FxTwoArgForEachLoop::Resolve(FCompileContext &ctx)
{
CHECKRESOLVED();
SAFE_RESOLVE(MapExpr, ctx);
SAFE_RESOLVE(MapExpr2, ctx);
SAFE_RESOLVE(MapExpr3, ctx);
SAFE_RESOLVE(MapExpr4, ctx);
bool is_iterator = false;
if(IsGameSpecificTwoArgForEachLoop(this))
{
return ResolveGameSpecificTwoArgForEachLoop(this)->Resolve(ctx);
}
else if(!(MapExpr->ValueType->isMap() || (is_iterator = MapExpr->ValueType->isMapIterator())))
{
if(HasGameSpecificTwoArgForEachLoopTypeNames())
{
ScriptPosition.Message(MSG_ERROR, "foreach( k, v : m ) - 'm' must be %s a map, or a map iterator, but is a %s", GetGameSpecificTwoArgForEachLoopTypeNames(), MapExpr->ValueType->DescriptiveName());
delete this;
return nullptr;
}
else
{
ScriptPosition.Message(MSG_ERROR, "foreach( k, v : m ) - 'm' must be a map or a map iterator, but is a %s", MapExpr->ValueType->DescriptiveName());
delete this;
return nullptr;
}
}
else if(valueVarName == NAME_None)
{
ScriptPosition.Message(MSG_ERROR, "missing value for foreach( k, v : m )");
delete this;
return nullptr;
}
auto block = new FxCompoundStatement(ScriptPosition);
auto valType = is_iterator ? static_cast<PMapIterator*>(MapExpr->ValueType)->ValueType : static_cast<PMap*>(MapExpr->ValueType)->ValueType;
auto keyType = is_iterator ? static_cast<PMapIterator*>(MapExpr->ValueType)->KeyType : static_cast<PMap*>(MapExpr->ValueType)->KeyType;
auto v = new FxLocalVariableDeclaration(valType, valueVarName, nullptr, 0, ScriptPosition);
block->Add(v);
if(keyVarName != NAME_None)
{
auto k = new FxLocalVariableDeclaration(keyType, keyVarName, nullptr, 0, ScriptPosition);
block->Add(k);
}
if(MapExpr->ValueType->isMapIterator())
{
/*
{
KeyType k;
ValueType v;
if(it.ReInit()) while(it.Next())
{
k = it.GetKey();
v = it.GetValue();
body
}
}
*/
auto inner_block = new FxCompoundStatement(ScriptPosition);
if(keyVarName != NAME_None)
{
inner_block->Add(new FxAssign(new FxIdentifier(keyVarName, ScriptPosition), new FxMemberFunctionCall(MapExpr, "GetKey", {}, ScriptPosition), true));
}
inner_block->Add(new FxAssign(new FxIdentifier(valueVarName, ScriptPosition), new FxMemberFunctionCall(MapExpr2, "GetValue", {}, ScriptPosition), true));
inner_block->Add(Code);
auto reInit = new FxMemberFunctionCall(MapExpr3, "ReInit", {}, ScriptPosition);
block->Add(new FxIfStatement(reInit, new FxWhileLoop(new FxMemberFunctionCall(MapExpr4, "Next", {}, ScriptPosition), inner_block, ScriptPosition), nullptr, ScriptPosition));
MapExpr = MapExpr2 = MapExpr3 = MapExpr4 = Code = nullptr;
delete this;
return block->Resolve(ctx);
}
else
{
/*
{
KeyType k;
ValueType v;
MapIterator<KeyType, ValueType> @it;
@it.Init(map);
while(@it.Next())
{
k = @it.GetKey();
v = @it.GetValue();
body
}
}
*/
PType * itType = NewMapIterator(keyType, valType);
auto it = new FxLocalVariableDeclaration(itType, "@it", nullptr, 0, ScriptPosition);
block->Add(it);
FArgumentList al_map;
al_map.Push(MapExpr);
block->Add(new FxMemberFunctionCall(new FxIdentifier("@it", ScriptPosition), "Init", std::move(al_map), ScriptPosition));
auto inner_block = new FxCompoundStatement(ScriptPosition);
if(keyVarName != NAME_None)
{
inner_block->Add(new FxAssign(new FxIdentifier(keyVarName, ScriptPosition), new FxMemberFunctionCall(new FxIdentifier("@it", ScriptPosition), "GetKey", {}, ScriptPosition), true));
}
inner_block->Add(new FxAssign(new FxIdentifier(valueVarName, ScriptPosition), new FxMemberFunctionCall(new FxIdentifier("@it", ScriptPosition), "GetValue", {}, ScriptPosition), true));
inner_block->Add(Code);
block->Add(new FxWhileLoop(new FxMemberFunctionCall(new FxIdentifier("@it", ScriptPosition), "Next", {}, ScriptPosition), inner_block, ScriptPosition));
delete MapExpr2;
delete MapExpr3;
delete MapExpr4;
MapExpr = MapExpr2 = MapExpr3 = MapExpr4 = Code = nullptr;
delete this;
return block->Resolve(ctx);
}
}
//==========================================================================
//
// FxThreeArgForEachLoop
//
//==========================================================================
FxThreeArgForEachLoop::FxThreeArgForEachLoop(FName vv, FName pv, FName fv, FxExpression* blockiteartorexpr, FxExpression* code, const FScriptPosition& pos)
: FxExpression(EFX_ThreeArgForEachLoop, pos), varVarName(vv), posVarName(pv), flagsVarName(fv), BlockIteratorExpr(blockiteartorexpr), Code(code)
{
ValueType = TypeVoid;
if (BlockIteratorExpr != nullptr) BlockIteratorExpr->NeedResult = false;
if (Code != nullptr) Code->NeedResult = false;
}
FxThreeArgForEachLoop::~FxThreeArgForEachLoop()
{
SAFE_DELETE(BlockIteratorExpr);
SAFE_DELETE(Code);
}
extern bool HasGameSpecificThreeArgForEachLoopTypeNames();
extern const char * GetGameSpecificThreeArgForEachLoopTypeNames();
extern bool IsGameSpecificThreeArgForEachLoop(FxThreeArgForEachLoop *);
extern FxExpression * ResolveGameSpecificThreeArgForEachLoop(FxThreeArgForEachLoop *);
FxExpression *FxThreeArgForEachLoop::Resolve(FCompileContext &ctx)
{
CHECKRESOLVED();
SAFE_RESOLVE(BlockIteratorExpr, ctx);
if(IsGameSpecificThreeArgForEachLoop(this))
{
return ResolveGameSpecificThreeArgForEachLoop(this)->Resolve(ctx);
}
else
{
//put any non-game-specific typed for-each loops here
}
if(HasGameSpecificThreeArgForEachLoopTypeNames())
{
ScriptPosition.Message(MSG_ERROR, "foreach( a, b, c : it ) - 'it' must be % but is a %s", GetGameSpecificThreeArgForEachLoopTypeNames(), BlockIteratorExpr->ValueType->DescriptiveName());
delete this;
return nullptr;
}
else
{
ScriptPosition.Message(MSG_ERROR, "foreach( a, b, c : it ) - three-arg foreach loops not supported");
delete this;
return nullptr;
}
}
//==========================================================================
//
// FxCastForEachLoop
//
//==========================================================================
FxTypedForEachLoop::FxTypedForEachLoop(FName cv, FName vv, FxExpression* castiteartorexpr, FxExpression* code, const FScriptPosition& pos)
: FxExpression(EFX_TypedForEachLoop, pos), className(cv), varName(vv), Expr(castiteartorexpr), Code(code)
{
ValueType = TypeVoid;
if (Expr != nullptr) Expr->NeedResult = false;
if (Code != nullptr) Code->NeedResult = false;
}
FxTypedForEachLoop::~FxTypedForEachLoop()
{
SAFE_DELETE(Expr);
SAFE_DELETE(Code);
}
extern bool HasGameSpecificTypedForEachLoopTypeNames();
extern const char * GetGameSpecificTypedForEachLoopTypeNames();
extern bool IsGameSpecificTypedForEachLoop(FxTypedForEachLoop *);
extern FxExpression * ResolveGameSpecificTypedForEachLoop(FxTypedForEachLoop *);
FxExpression *FxTypedForEachLoop::Resolve(FCompileContext &ctx)
{
CHECKRESOLVED();
SAFE_RESOLVE(Expr, ctx);
if(IsGameSpecificTypedForEachLoop(this))
{
return ResolveGameSpecificTypedForEachLoop(this)->Resolve(ctx);
}
else
{
//put any non-game-specific typed for-each loops here
}
if(HasGameSpecificTypedForEachLoopTypeNames())
{
ScriptPosition.Message(MSG_ERROR, "foreach(Type var : it ) - 'it' must be % but is a %s",GetGameSpecificTypedForEachLoopTypeNames(), Expr->ValueType->DescriptiveName());
delete this;
return nullptr;
}
else
{
ScriptPosition.Message(MSG_ERROR, "foreach(Type var : it ) - typed foreach loops not supported");
delete this;
return nullptr;
}
}
//==========================================================================
//
@ -12071,8 +12580,7 @@ FxExpression *FxLocalVariableDeclaration::Resolve(FCompileContext &ctx)
if (IsDynamicArray())
{
auto stackVar = new FxStackVariable(ValueType, StackOffset, ScriptPosition);
FArgumentList argsList;
clearExpr = new FxMemberFunctionCall(stackVar, "Clear", argsList, ScriptPosition);
clearExpr = new FxMemberFunctionCall(stackVar, "Clear", {}, ScriptPosition);
SAFE_RESOLVE(clearExpr, ctx);
}
@ -12390,10 +12898,9 @@ FxExpression *FxLocalArrayDeclaration::Resolve(FCompileContext &ctx)
else
{
FArgumentList argsList;
argsList.Clear();
argsList.Push(v);
FxExpression *funcCall = new FxMemberFunctionCall(stackVar, NAME_Push, argsList, (const FScriptPosition) v->ScriptPosition);
FxExpression *funcCall = new FxMemberFunctionCall(stackVar, NAME_Push, std::move(argsList), (const FScriptPosition) v->ScriptPosition);
SAFE_RESOLVE(funcCall, ctx);
v = funcCall;

100
src/common/scripting/backend/codegen.h
View file

@ -48,6 +48,7 @@
#include "types.h"
#include "vmintern.h"
#include "c_cvars.h"
#include "palettecontainer.h"
struct FState; // needed for FxConstant. Maybe move the state constructor to a subclass later?
@ -232,6 +233,7 @@ enum EFxType
EFX_StringCast,
EFX_ColorCast,
EFX_SoundCast,
EFX_TranslationCast,
EFX_TypeCast,
EFX_PlusSign,
EFX_MinusSign,
@ -273,6 +275,9 @@ enum EFxType
EFX_DoWhileLoop,
EFX_ForLoop,
EFX_ForEachLoop,
EFX_TwoArgForEachLoop,
EFX_ThreeArgForEachLoop,
EFX_TypedForEachLoop,
EFX_JumpStatement,
EFX_ReturnStatement,
EFX_ClassTypeCast,
@ -507,6 +512,13 @@ public:
isresolved = true;
}
FxConstant(FTranslationID state, const FScriptPosition& pos) : FxExpression(EFX_Constant, pos)
{
value.Int = state.index();
ValueType = value.Type = TypeTranslationID;
isresolved = true;
}
FxConstant(VMFunction* state, const FScriptPosition& pos) : FxExpression(EFX_Constant, pos)
{
value.pointer = state;
@ -715,6 +727,19 @@ public:
ExpEmit Emit(VMFunctionBuilder *build);
};
class FxTranslationCast : public FxExpression
{
FxExpression* basex;
public:
FxTranslationCast(FxExpression* x);
~FxTranslationCast();
FxExpression* Resolve(FCompileContext&);
ExpEmit Emit(VMFunctionBuilder* build);
};
class FxFontCast : public FxExpression
{
FxExpression *basex;
@ -1599,7 +1624,7 @@ public:
FName MethodName;
FArgumentList ArgList;
FxFunctionCall(FName methodname, FName rngname, FArgumentList &args, const FScriptPosition &pos);
FxFunctionCall(FName methodname, FName rngname, FArgumentList &&args, const FScriptPosition &pos);
~FxFunctionCall();
FxExpression *Resolve(FCompileContext&);
};
@ -1616,10 +1641,11 @@ class FxMemberFunctionCall : public FxExpression
FxExpression *Self;
FName MethodName;
FArgumentList ArgList;
bool ResolveSelf;
public:
FxMemberFunctionCall(FxExpression *self, FName methodname, FArgumentList &args, const FScriptPosition &pos);
FxMemberFunctionCall(FxExpression *self, FName methodname, FArgumentList &&args, const FScriptPosition &pos);
~FxMemberFunctionCall();
FxExpression *Resolve(FCompileContext&);
};
@ -2047,17 +2073,83 @@ public:
class FxForEachLoop : public FxLoopStatement
{
public:
FName loopVarName;
FxExpression* Array;
FxExpression* Array2;
FxExpression* Array3;
FxExpression* Array4;
FxExpression* Code;
public:
FxForEachLoop(FName vn, FxExpression* arrayvar, FxExpression* arrayvar2, FxExpression* code, const FScriptPosition& pos);
FxForEachLoop(FName vn, FxExpression* arrayvar, FxExpression* arrayvar2, FxExpression* arrayvar3, FxExpression* arrayvar4, FxExpression* code, const FScriptPosition& pos);
~FxForEachLoop();
FxExpression* DoResolve(FCompileContext&);
};
//==========================================================================
//
// FxTwoArgForEachLoop
//
//==========================================================================
class FxTwoArgForEachLoop : public FxExpression
{
public:
FName keyVarName;
FName valueVarName;
FxExpression* MapExpr;
FxExpression* MapExpr2;
FxExpression* MapExpr3;
FxExpression* MapExpr4;
FxExpression* Code;
FxTwoArgForEachLoop(FName kv, FName vv, FxExpression* mapexpr, FxExpression* mapexpr2, FxExpression* mapexpr3, FxExpression* mapexpr4, FxExpression* code, const FScriptPosition& pos);
~FxTwoArgForEachLoop();
FxExpression *Resolve(FCompileContext&);
//ExpEmit Emit(VMFunctionBuilder *build); This node is transformed, so it won't ever be emitted itself
};
//==========================================================================
//
// FxThreeArgForEachLoop
//
//==========================================================================
class FxThreeArgForEachLoop : public FxExpression
{
public:
FName varVarName;
FName posVarName;
FName flagsVarName;
FxExpression* BlockIteratorExpr;
FxExpression* Code;
FxThreeArgForEachLoop(FName vv, FName pv, FName fv, FxExpression* blockiteartorexpr, FxExpression* code, const FScriptPosition& pos);
~FxThreeArgForEachLoop();
FxExpression *Resolve(FCompileContext&);
//ExpEmit Emit(VMFunctionBuilder *build); This node is transformed, so it won't ever be emitted itself
};
//==========================================================================
//
// FxTypedForEachLoop
//
//==========================================================================
class FxTypedForEachLoop : public FxExpression
{
public:
FName className;
FName varName;
FxExpression* Expr;
FxExpression* Code;
FxTypedForEachLoop(FName cv, FName vv, FxExpression* castiteartorexpr, FxExpression* code, const FScriptPosition& pos);
~FxTypedForEachLoop();
FxExpression *Resolve(FCompileContext&);
//ExpEmit Emit(VMFunctionBuilder *build); This node is transformed, so it won't ever be emitted itself
};
//==========================================================================
//
// FxJumpStatement

38
src/common/scripting/backend/vmbuilder.cpp
View file

@ -41,6 +41,9 @@
CVAR(Bool, strictdecorate, false, CVAR_GLOBALCONFIG | CVAR_ARCHIVE)
EXTERN_CVAR(Bool, vm_jit)
EXTERN_CVAR(Bool, vm_jit_aot)
struct VMRemap
{
uint8_t altOp, kReg, kType;
@ -290,6 +293,24 @@ unsigned VMFunctionBuilder::GetConstantAddress(void *ptr)
}
}
//==========================================================================
//
// VMFunctionBuilder :: FindConstantInt
//
// Returns a constant register initialized with the given value.
//
//==========================================================================
int VMFunctionBuilder::FindConstantInt(unsigned index)
{
if(IntConstantList.Size() < index)
{
return IntConstantList[index];
}
return 0;
}
//==========================================================================
//
// VMFunctionBuilder :: AllocConstants*
@ -298,7 +319,7 @@ unsigned VMFunctionBuilder::GetConstantAddress(void *ptr)
//
//==========================================================================
unsigned VMFunctionBuilder::AllocConstantsInt(unsigned count, int *values)
unsigned VMFunctionBuilder::AllocConstantsInt(unsigned int count, int *values)
{
unsigned addr = IntConstantList.Reserve(count);
memcpy(&IntConstantList[addr], values, count * sizeof(int));
@ -309,7 +330,7 @@ unsigned VMFunctionBuilder::AllocConstantsInt(unsigned count, int *values)
return addr;
}
unsigned VMFunctionBuilder::AllocConstantsFloat(unsigned count, double *values)
unsigned VMFunctionBuilder::AllocConstantsFloat(unsigned int count, double *values)
{
unsigned addr = FloatConstantList.Reserve(count);
memcpy(&FloatConstantList[addr], values, count * sizeof(double));
@ -320,7 +341,7 @@ unsigned VMFunctionBuilder::AllocConstantsFloat(unsigned count, double *values)
return addr;
}
unsigned VMFunctionBuilder::AllocConstantsAddress(unsigned count, void **ptrs)
unsigned VMFunctionBuilder::AllocConstantsAddress(unsigned int count, void **ptrs)
{
unsigned addr = AddressConstantList.Reserve(count);
memcpy(&AddressConstantList[addr], ptrs, count * sizeof(void *));
@ -331,7 +352,7 @@ unsigned VMFunctionBuilder::AllocConstantsAddress(unsigned count, void **ptrs)
return addr;
}
unsigned VMFunctionBuilder::AllocConstantsString(unsigned count, FString *ptrs)
unsigned VMFunctionBuilder::AllocConstantsString(unsigned int count, FString *ptrs)
{
unsigned addr = StringConstantList.Reserve(count);
for (unsigned i = 0; i < count; i++)
@ -910,6 +931,13 @@ void FFunctionBuildList::Build()
disasmdump.Write(sfunc, item.PrintableName);
sfunc->Unsafe = ctx.Unsafe;
#if HAVE_VM_JIT
if(vm_jit && vm_jit_aot)
{
sfunc->JitCompile();
}
#endif
}
catch (CRecoverableError &err)
{
@ -1076,8 +1104,6 @@ void FunctionCallEmitter::AddParameterStringConst(const FString &konst)
});
}
EXTERN_CVAR(Bool, vm_jit)
ExpEmit FunctionCallEmitter::EmitCall(VMFunctionBuilder *build, TArray<ExpEmit> *ReturnRegs)
{
unsigned paramcount = 0;

5
src/common/scripting/backend/vmbuilder.h
View file

@ -65,6 +65,11 @@ public:
unsigned GetConstantAddress(void *ptr);
unsigned GetConstantString(FString str);
int FindConstantInt(unsigned index);
//double FindConstantFloat(unsigned index);
//void * FindConstantAddress(unsigned index);
//const FString& FindConstantString(unsigned index);
unsigned AllocConstantsInt(unsigned int count, int *values);
unsigned AllocConstantsFloat(unsigned int count, double *values);
unsigned AllocConstantsAddress(unsigned int count, void **ptrs);

187
src/common/scripting/core/types.cpp
View file

@ -41,6 +41,9 @@
#include "printf.h"
#include "textureid.h"
#include "maps.h"
#include "palettecontainer.h"
#include "texturemanager.h"
#include "i_interface.h"
FTypeTable TypeTable;
@ -58,6 +61,7 @@ PName *TypeName;
PSound *TypeSound;
PColor *TypeColor;
PTextureID *TypeTextureID;
PTranslationID* TypeTranslationID;
PSpriteID *TypeSpriteID;
PStatePointer *TypeState;
PPointer *TypeFont;
@ -322,6 +326,7 @@ void PType::StaticInit()
TypeTable.AddType(TypeNullPtr = new PPointer, NAME_Pointer);
TypeTable.AddType(TypeSpriteID = new PSpriteID, NAME_SpriteID);
TypeTable.AddType(TypeTextureID = new PTextureID, NAME_TextureID);
TypeTable.AddType(TypeTranslationID = new PTranslationID, NAME_TranslationID);
TypeVoidPtr = NewPointer(TypeVoid, false);
TypeRawFunction = new PPointer;
@ -1321,6 +1326,48 @@ bool PTextureID::ReadValue(FSerializer &ar, const char *key, void *addr) const
return true;
}
/* PTranslationID ******************************************************************/
//==========================================================================
//
// PTranslationID Default Constructor
//
//==========================================================================
PTranslationID::PTranslationID()
: PInt(sizeof(FTranslationID), true, false)
{
mDescriptiveName = "TranslationID";
Flags |= TYPE_IntNotInt;
static_assert(sizeof(FTranslationID) == alignof(FTranslationID), "TranslationID not properly aligned");
}
//==========================================================================
//
// PTranslationID :: WriteValue
//
//==========================================================================
void PTranslationID::WriteValue(FSerializer& ar, const char* key, const void* addr) const
{
FTranslationID val = *(FTranslationID*)addr;
ar(key, val);
}
//==========================================================================
//
// PTranslationID :: ReadValue
//
//==========================================================================
bool PTranslationID::ReadValue(FSerializer& ar, const char* key, void* addr) const
{
FTranslationID val;
ar(key, val);
*(FTranslationID*)addr = val;
return true;
}
/* PSound *****************************************************************/
//==========================================================================
@ -1614,7 +1661,8 @@ PClassPointer::PClassPointer(PClass *restrict)
loadOp = OP_LP;
storeOp = OP_SP;
Flags |= TYPE_ClassPointer;
mVersion = restrict->VMType->mVersion;
if (restrict) mVersion = restrict->VMType->mVersion;
else mVersion = 0;
}
//==========================================================================
@ -2504,9 +2552,63 @@ static void PMapValueWriter(FSerializer &ar, const M *map, const PMap *m)
}
else if constexpr(std::is_same_v<typename M::KeyType,uint32_t>)
{
FString key;
key.Format("%u",p->Key);
m->ValueType->WriteValue(ar,key.GetChars(),static_cast<const void *>(&p->Value));
if(m->KeyType->Flags & 8 /*TYPE_IntNotInt*/)
{
if(m->KeyType == TypeName)
{
m->ValueType->WriteValue(ar,FName(ENamedName(p->Key)).GetChars(),static_cast<const void *>(&p->Value));
}
else if(m->KeyType == TypeSound)
{
m->ValueType->WriteValue(ar,soundEngine->GetSoundName(FSoundID::fromInt(p->Key)),static_cast<const void *>(&p->Value));
}
else if(m->KeyType == TypeTextureID)
{
if(!!(p->Key & 0x8000000))
{ // invalid
m->ValueType->WriteValue(ar,"invalid",static_cast<const void *>(&p->Value));
}
else if(p->Key == 0 || p->Key >= TexMan.NumTextures())
{ // null
m->ValueType->WriteValue(ar,"null",static_cast<const void *>(&p->Value));
}
else
{
FTextureID tid;
tid.SetIndex(p->Key);
FGameTexture *tex = TexMan.GetGameTexture(tid);
int lump = tex->GetSourceLump();
unsigned useType = static_cast<unsigned>(tex->GetUseType());
FString name;
if (TexMan.GetLinkedTexture(lump) == tex)
{
name = fileSystem.GetFileFullName(lump);
}
else
{
name = tex->GetName().GetChars();
}
name.AppendFormat(":%u",useType);
m->ValueType->WriteValue(ar,name.GetChars(),static_cast<const void *>(&p->Value));
}
}
else
{ // bool/color/enum/sprite/translationID
FString key;
key.Format("%u",p->Key);
m->ValueType->WriteValue(ar,key.GetChars(),static_cast<const void *>(&p->Value));
}
}
else
{
FString key;
key.Format("%u",p->Key);
m->ValueType->WriteValue(ar,key.GetChars(),static_cast<const void *>(&p->Value));
}
}
//else unknown key type
}
@ -2539,20 +2641,85 @@ static bool PMapValueReader(FSerializer &ar, M *map, const PMap *m)
const char * k;
while((k = ar.GetKey()))
{
typename M::ValueType * val;
typename M::ValueType * val = nullptr;
if constexpr(std::is_same_v<typename M::KeyType,FString>)
{
val = &map->InsertNew(k);
}
else if constexpr(std::is_same_v<typename M::KeyType,uint32_t>)
{
FString s(k);
if(!s.IsInt())
if(m->KeyType->Flags & 8 /*TYPE_IntNotInt*/)
{
ar.EndObject();
return false;
if(m->KeyType == TypeName)
{
val = &map->InsertNew(FName(k).GetIndex());
}
else if(m->KeyType == TypeSound)
{
val = &map->InsertNew(S_FindSound(k).index());
}
else if(m->KeyType == TypeTextureID)
{
FString s(k);
FTextureID tex;
if(s.Compare("invalid") == 0)
{
tex.SetInvalid();
}
else if(s.Compare("null") == 0)
{
tex.SetNull();
}
else
{
ptrdiff_t sep = s.LastIndexOf(":");
if(sep < 0)
{
ar.EndObject();
return false;
}
FString texName = s.Left(sep);
FString useType = s.Mid(sep + 1);
tex = TexMan.GetTextureID(texName.GetChars(), (ETextureType) useType.ToULong());
}
val = &map->InsertNew(tex.GetIndex());
}
else if(m->KeyType == TypeTranslationID)
{
FString s(k);
if(!s.IsInt())
{
ar.EndObject();
return false;
}
int v = s.ToULong();
if (sysCallbacks.RemapTranslation) v = sysCallbacks.RemapTranslation(FTranslationID::fromInt(v)).index();
val = &map->InsertNew(v);
}
else
{ // bool/color/enum/sprite
FString s(k);
if(!s.IsInt())
{
ar.EndObject();
return false;
}
val = &map->InsertNew(static_cast<uint32_t>(s.ToULong()));
}
}
else
{
FString s(k);
if(!s.IsInt())
{
ar.EndObject();
return false;
}
val = &map->InsertNew(static_cast<uint32_t>(s.ToULong()));
}
val = &map->InsertNew(static_cast<uint32_t>(s.ToULong()));
}
if (!m->ValueType->ReadValue(ar,nullptr,static_cast<void*>(val)))
{

10
src/common/scripting/core/types.h
View file

@ -383,6 +383,15 @@ public:
bool ReadValue(FSerializer &ar, const char *key, void *addr) const override;
};
class PTranslationID : public PInt
{
public:
PTranslationID();
void WriteValue(FSerializer& ar, const char* key, const void* addr) const override;
bool ReadValue(FSerializer& ar, const char* key, void* addr) const override;
};
class PColor : public PInt
{
public:
@ -712,6 +721,7 @@ extern PName *TypeName;
extern PSound *TypeSound;
extern PColor *TypeColor;
extern PTextureID *TypeTextureID;
extern PTranslationID* TypeTranslationID;
extern PSpriteID *TypeSpriteID;
extern PStruct* TypeVector2;
extern PStruct* TypeVector3;

50
src/common/scripting/frontend/ast.cpp
View file

@ -983,6 +983,53 @@ static void PrintArrayIterationStmt(FLispString &out, const ZCC_TreeNode *node)
out.Close();
}
static void PrintTwoArgIterationStmt(FLispString &out, const ZCC_TreeNode *node)
{
auto inode = (ZCC_TwoArgIterationStmt *)node;
out.Break();
out.Open("map-iteration-stmt");
PrintVarName(out, inode->ItKey);
out.Break();
PrintVarName(out, inode->ItValue);
out.Break();
PrintNodes(out, inode->ItMap);
out.Break();
PrintNodes(out, inode->LoopStatement);
out.Close();
}
static void PrintThreeArgIterationStmt(FLispString &out, const ZCC_TreeNode *node)
{
auto inode = (ZCC_ThreeArgIterationStmt *)node;
out.Break();
out.Open("block-iteration-stmt");
PrintVarName(out, inode->ItVar);
out.Break();
PrintVarName(out, inode->ItPos);
out.Break();
PrintVarName(out, inode->ItFlags);
out.Break();
PrintNodes(out, inode->ItBlock);
out.Break();
PrintNodes(out, inode->LoopStatement);
out.Close();
}
static void PrintTypedIterationStmt(FLispString &out, const ZCC_TreeNode *node)
{
auto inode = (ZCC_TypedIterationStmt *)node;
out.Break();
out.Open("cast-iteration-stmt");
PrintVarName(out, inode->ItType);
out.Break();
PrintVarName(out, inode->ItVar);
out.Break();
PrintNodes(out, inode->ItExpr);
out.Break();
PrintNodes(out, inode->LoopStatement);
out.Close();
}
static const NodePrinterFunc TreeNodePrinter[] =
{
PrintIdentifier,
@ -1050,6 +1097,9 @@ static const NodePrinterFunc TreeNodePrinter[] =
PrintMixinDef,
PrintMixinStmt,
PrintArrayIterationStmt,
PrintTwoArgIterationStmt,
PrintThreeArgIterationStmt,
PrintTypedIterationStmt,
};
FString ZCC_PrintAST(const ZCC_TreeNode *root)

40
src/common/scripting/frontend/zcc-parse.lemon
View file

@ -1956,6 +1956,9 @@ statement(X) ::= expression_statement(A) SEMICOLON. { X = A; /*X-overwrites-A*/
statement(X) ::= selection_statement(X).
statement(X) ::= iteration_statement(X).
statement(X) ::= array_iteration_statement(X).
statement(X) ::= two_arg_iteration_statement(X).
statement(X) ::= three_arg_iteration_statement(X).
statement(X) ::= typed_iteration_statement(X).
statement(X) ::= jump_statement(X).
statement(X) ::= assign_statement(A) SEMICOLON. { X = A; /*X-overwrites-A*/ }
statement(X) ::= assign_decl_statement(A) SEMICOLON.{ X = A; /*X-overwrites-A*/ }
@ -2125,6 +2128,43 @@ array_iteration_statement(X) ::= FOREACH(T) LPAREN variable_name(IN) COLON expr(
X = iter;
}
%type two_arg_iteration_statement{ZCC_Statement *}
two_arg_iteration_statement(X) ::= FOREACH(T) LPAREN variable_name(KEY) COMMA variable_name(VAL) COLON expr(EX) RPAREN statement(ST).
{
NEW_AST_NODE(TwoArgIterationStmt, iter, T);
iter->ItKey = KEY;
iter->ItValue = VAL;
iter->ItMap = EX;
iter->LoopStatement = ST;
X = iter;
}
%type three_arg_iteration_statement{ZCC_Statement *}
three_arg_iteration_statement(X) ::= FOREACH(T) LPAREN variable_name(VAR) COMMA variable_name(POS) COMMA variable_name(FLAGS) COLON expr(EX) RPAREN statement(ST).
{
NEW_AST_NODE(ThreeArgIterationStmt, iter, T);
iter->ItVar = VAR;
iter->ItPos = POS;
iter->ItFlags = FLAGS;
iter->ItBlock = EX;
iter->LoopStatement = ST;
X = iter;
}
%type typed_iteration_statement{ZCC_Statement *}
typed_iteration_statement(X) ::= FOREACH(T) LPAREN variable_name(TYPE) variable_name(VAR) COLON expr(EX) RPAREN statement(ST).
{
NEW_AST_NODE(TypedIterationStmt, iter, T);
iter->ItType = TYPE;
iter->ItVar = VAR;
iter->ItExpr = EX;
iter->LoopStatement = ST;
X = iter;
}
while_or_until(X) ::= WHILE(T).
{
X.Int = ZCC_WHILE;

51
src/common/scripting/frontend/zcc_compile.cpp
View file

@ -1886,6 +1886,12 @@ PType *ZCCCompiler::DetermineType(PType *outertype, ZCC_TreeNode *field, FName n
retval = TypeTextureID;
break;
case NAME_TranslationID:
retval = TypeTranslationID;
break;
default:
retval = ResolveUserType(btype, btype->UserType, outertype ? &outertype->Symbols : nullptr, false);
break;
@ -2993,12 +2999,12 @@ FxExpression *ZCCCompiler::ConvertNode(ZCC_TreeNode *ast, bool substitute)
{
case AST_ExprID:
// The function name is a simple identifier.
return new FxFunctionCall(static_cast<ZCC_ExprID *>(fcall->Function)->Identifier, NAME_None, ConvertNodeList(args, fcall->Parameters), *ast);
return new FxFunctionCall(static_cast<ZCC_ExprID *>(fcall->Function)->Identifier, NAME_None, std::move(ConvertNodeList(args, fcall->Parameters)), *ast);
case AST_ExprMemberAccess:
{
auto ema = static_cast<ZCC_ExprMemberAccess *>(fcall->Function);
return new FxMemberFunctionCall(ConvertNode(ema->Left, true), ema->Right, ConvertNodeList(args, fcall->Parameters), *ast);
return new FxMemberFunctionCall(ConvertNode(ema->Left, true), ema->Right, std::move(ConvertNodeList(args, fcall->Parameters)), *ast);
}
case AST_ExprBinary:
@ -3008,7 +3014,7 @@ FxExpression *ZCCCompiler::ConvertNode(ZCC_TreeNode *ast, bool substitute)
auto binary = static_cast<ZCC_ExprBinary *>(fcall->Function);
if (binary->Left->NodeType == AST_ExprID && binary->Right->NodeType == AST_ExprID)
{
return new FxFunctionCall(static_cast<ZCC_ExprID *>(binary->Left)->Identifier, static_cast<ZCC_ExprID *>(binary->Right)->Identifier, ConvertNodeList(args, fcall->Parameters), *ast);
return new FxFunctionCall(static_cast<ZCC_ExprID *>(binary->Left)->Identifier, static_cast<ZCC_ExprID *>(binary->Right)->Identifier, std::move(ConvertNodeList(args, fcall->Parameters)), *ast);
}
}
// fall through if this isn't an array access node.
@ -3382,10 +3388,45 @@ FxExpression *ZCCCompiler::ConvertNode(ZCC_TreeNode *ast, bool substitute)
auto iter = static_cast<ZCC_ArrayIterationStmt*>(ast);
auto var = iter->ItName->Name;
FxExpression* const itArray = ConvertNode(iter->ItArray);
FxExpression* const itArray2 = ConvertNode(iter->ItArray); // the handler needs two copies of this - here's the easiest place to create them.
FxExpression* const itArray2 = ConvertNode(iter->ItArray);
FxExpression* const itArray3 = ConvertNode(iter->ItArray);
FxExpression* const itArray4 = ConvertNode(iter->ItArray); // the handler needs copies of this - here's the easiest place to create them.
FxExpression* const body = ConvertImplicitScopeNode(ast, iter->LoopStatement);
return new FxForEachLoop(iter->ItName->Name, itArray, itArray2, body, *ast);
return new FxForEachLoop(iter->ItName->Name, itArray, itArray2, itArray3, itArray4, body, *ast);
}
case AST_TwoArgIterationStmt:
{
auto iter = static_cast<ZCC_TwoArgIterationStmt*>(ast);
auto key = iter->ItKey->Name;
auto var = iter->ItValue->Name;
FxExpression* const itMap = ConvertNode(iter->ItMap);
FxExpression* const itMap2 = ConvertNode(iter->ItMap);
FxExpression* const itMap3 = ConvertNode(iter->ItMap);
FxExpression* const itMap4 = ConvertNode(iter->ItMap);
FxExpression* const body = ConvertImplicitScopeNode(ast, iter->LoopStatement);
return new FxTwoArgForEachLoop(key, var, itMap, itMap2, itMap3, itMap4, body, *ast);
}
case AST_ThreeArgIterationStmt:
{
auto iter = static_cast<ZCC_ThreeArgIterationStmt*>(ast);
auto var = iter->ItVar->Name;
auto pos = iter->ItPos->Name;
auto flags = iter->ItFlags->Name;
FxExpression* const itBlock = ConvertNode(iter->ItBlock);
FxExpression* const body = ConvertImplicitScopeNode(ast, iter->LoopStatement);
return new FxThreeArgForEachLoop(var, pos, flags, itBlock, body, *ast);
}
case AST_TypedIterationStmt:
{
auto iter = static_cast<ZCC_TypedIterationStmt*>(ast);
auto cls = iter->ItType->Name;
auto var = iter->ItVar->Name;
FxExpression* const itExpr = ConvertNode(iter->ItExpr);
FxExpression* const body = ConvertImplicitScopeNode(ast, iter->LoopStatement);
return new FxTypedForEachLoop(cls, var, itExpr, body, *ast);
}
case AST_IterationStmt:

40
src/common/scripting/frontend/zcc_parser.cpp
View file

@ -1177,6 +1177,46 @@ ZCC_TreeNode *TreeNodeDeepCopy_Internal(ZCC_AST *ast, ZCC_TreeNode *orig, bool c
break;
}
case AST_TwoArgIterationStmt:
{
TreeNodeDeepCopy_Start(TwoArgIterationStmt);
// ZCC_TwoArgIterationStmt
copy->ItKey = static_cast<ZCC_VarName*>(TreeNodeDeepCopy_Internal(ast, origCasted->ItKey, true, copiedNodesList));
copy->ItValue = static_cast<ZCC_VarName*>(TreeNodeDeepCopy_Internal(ast, origCasted->ItValue, true, copiedNodesList));
copy->LoopStatement = static_cast<ZCC_Statement*>(TreeNodeDeepCopy_Internal(ast, origCasted->LoopStatement, true, copiedNodesList));
copy->ItMap = static_cast<ZCC_Expression*>(TreeNodeDeepCopy_Internal(ast, origCasted->ItMap, true, copiedNodesList));
break;
}
case AST_ThreeArgIterationStmt:
{
TreeNodeDeepCopy_Start(ThreeArgIterationStmt);
// ZCC_TwoArgIterationStmt
copy->ItVar = static_cast<ZCC_VarName*>(TreeNodeDeepCopy_Internal(ast, origCasted->ItVar, true, copiedNodesList));
copy->ItPos = static_cast<ZCC_VarName*>(TreeNodeDeepCopy_Internal(ast, origCasted->ItPos, true, copiedNodesList));
copy->ItFlags = static_cast<ZCC_VarName*>(TreeNodeDeepCopy_Internal(ast, origCasted->ItFlags, true, copiedNodesList));
copy->LoopStatement = static_cast<ZCC_Statement*>(TreeNodeDeepCopy_Internal(ast, origCasted->LoopStatement, true, copiedNodesList));
copy->ItBlock = static_cast<ZCC_Expression*>(TreeNodeDeepCopy_Internal(ast, origCasted->ItBlock, true, copiedNodesList));
break;
}
case AST_TypedIterationStmt:
{
TreeNodeDeepCopy_Start(TypedIterationStmt);
// ZCC_TwoArgIterationStmt
copy->ItType = static_cast<ZCC_VarName*>(TreeNodeDeepCopy_Internal(ast, origCasted->ItType, true, copiedNodesList));
copy->ItVar = static_cast<ZCC_VarName*>(TreeNodeDeepCopy_Internal(ast, origCasted->ItVar, true, copiedNodesList));
copy->LoopStatement = static_cast<ZCC_Statement*>(TreeNodeDeepCopy_Internal(ast, origCasted->LoopStatement, true, copiedNodesList));
copy->ItExpr = static_cast<ZCC_Expression*>(TreeNodeDeepCopy_Internal(ast, origCasted->ItExpr, true, copiedNodesList));
break;
}
case AST_IfStmt:
{
TreeNodeDeepCopy_Start(IfStmt);

28
src/common/scripting/frontend/zcc_parser.h
View file

@ -145,6 +145,9 @@ enum EZCCTreeNodeType
AST_MixinDef,
AST_MixinStmt,
AST_ArrayIterationStmt,
AST_TwoArgIterationStmt,
AST_ThreeArgIterationStmt,
AST_TypedIterationStmt,
NUM_AST_NODE_TYPES
};
@ -532,6 +535,31 @@ struct ZCC_ArrayIterationStmt : ZCC_Statement
ZCC_Statement* LoopStatement;
};
struct ZCC_TwoArgIterationStmt : ZCC_Statement
{
ZCC_VarName* ItKey;
ZCC_VarName* ItValue;
ZCC_Expression* ItMap;
ZCC_Statement* LoopStatement;
};
struct ZCC_ThreeArgIterationStmt : ZCC_Statement
{
ZCC_VarName* ItVar;
ZCC_VarName* ItPos;
ZCC_VarName* ItFlags;
ZCC_Expression* ItBlock;
ZCC_Statement* LoopStatement;
};
struct ZCC_TypedIterationStmt : ZCC_Statement
{
ZCC_VarName* ItType;
ZCC_VarName* ItVar;
ZCC_Expression* ItExpr;
ZCC_Statement* LoopStatement;
};
struct ZCC_IfStmt : ZCC_Statement
{
ZCC_Expression *Condition;

41
src/common/scripting/interface/vmnatives.cpp
View file

@ -668,7 +668,7 @@ DEFINE_ACTION_FUNCTION_NATIVE(FFont, GetBottomAlignOffset, GetBottomAlignOffset)
ACTION_RETURN_FLOAT(GetBottomAlignOffset(self, code));
}
static int StringWidth(FFont *font, const FString &str, bool localize)
static int StringWidth(FFont *font, const FString &str, int localize)
{
const char *txt = (localize && str[0] == '$') ? GStrings(&str[1]) : str.GetChars();
return font->StringWidth(txt);
@ -682,7 +682,7 @@ DEFINE_ACTION_FUNCTION_NATIVE(FFont, StringWidth, StringWidth)
ACTION_RETURN_INT(StringWidth(self, str, localize));
}
static int GetMaxAscender(FFont* font, const FString& str, bool localize)
static int GetMaxAscender(FFont* font, const FString& str, int localize)
{
const char* txt = (localize && str[0] == '$') ? GStrings(&str[1]) : str.GetChars();
return font->GetMaxAscender(txt);
@ -696,7 +696,7 @@ DEFINE_ACTION_FUNCTION_NATIVE(FFont, GetMaxAscender, GetMaxAscender)
ACTION_RETURN_INT(GetMaxAscender(self, str, localize));
}
static int CanPrint(FFont *font, const FString &str, bool localize)
static int CanPrint(FFont *font, const FString &str, int localize)
{
const char *txt = (localize && str[0] == '$') ? GStrings(&str[1]) : str.GetChars();
return font->CanPrint(txt);
@ -1372,3 +1372,38 @@ DEFINE_ACTION_FUNCTION_NATIVE(DObject, FindFunction, FindFunctionPointer)
ACTION_RETURN_POINTER(FindFunctionPointer(cls, fn.GetIndex()));
}
FTranslationID R_FindCustomTranslation(FName name);
static int ZFindTranslation(int intname)
{
return R_FindCustomTranslation(ENamedName(intname)).index();
}
static int MakeTransID(int g, int s)
{
return TRANSLATION(g, s).index();
}
DEFINE_ACTION_FUNCTION_NATIVE(_Translation, GetID, ZFindTranslation)
{
PARAM_PROLOGUE;
PARAM_INT(t);
ACTION_RETURN_INT(ZFindTranslation(t));
}
// same as above for the compiler which needs a class to look this up.
DEFINE_ACTION_FUNCTION_NATIVE(DObject, BuiltinFindTranslation, ZFindTranslation)
{
PARAM_PROLOGUE;
PARAM_INT(t);
ACTION_RETURN_INT(ZFindTranslation(t));
}
DEFINE_ACTION_FUNCTION_NATIVE(_Translation, MakeID, MakeTransID)
{
PARAM_PROLOGUE;
PARAM_INT(g);
PARAM_INT(t);
ACTION_RETURN_INT(MakeTransID(g, t));
}

39
src/common/scripting/vm/vmframe.cpp
View file

@ -54,8 +54,14 @@ CUSTOM_CVAR(Bool, vm_jit, true, CVAR_NOINITCALL)
Printf("You must restart " GAMENAME " for this change to take effect.\n");
Printf("This cvar is currently not saved. You must specify it on the command line.");
}
CUSTOM_CVAR(Bool, vm_jit_aot, true, CVAR_NOINITCALL)
{
Printf("You must restart " GAMENAME " for this change to take effect.\n");
Printf("This cvar is currently not saved. You must specify it on the command line.");
}
#else
CVAR(Bool, vm_jit, false, CVAR_NOINITCALL|CVAR_NOSET)
CVAR(Bool, vm_jit_aot, false, CVAR_NOINITCALL|CVAR_NOSET)
FString JitCaptureStackTrace(int framesToSkip, bool includeNativeFrames, int maxFrames) { return FString(); }
void JitRelease() {}
#endif
@ -282,6 +288,25 @@ static bool CanJit(VMScriptFunction *func)
return false;
}
void VMScriptFunction::JitCompile()
{
if(!(VarFlags & VARF_Abstract))
{
#ifdef HAVE_VM_JIT
if (vm_jit && CanJit(this))
{
ScriptCall = ::JitCompile(this);
if (!ScriptCall)
ScriptCall = VMExec;
}
else
#endif // HAVE_VM_JIT
{
ScriptCall = VMExec;
}
}
}
int VMScriptFunction::FirstScriptCall(VMFunction *func, VMValue *params, int numparams, VMReturn *ret, int numret)
{
// [Player701] Check that we aren't trying to call an abstract function.
@ -291,18 +316,8 @@ int VMScriptFunction::FirstScriptCall(VMFunction *func, VMValue *params, int num
{
ThrowAbortException(X_OTHER, "attempt to call abstract function %s.", func->PrintableName);
}
#ifdef HAVE_VM_JIT
if (vm_jit && CanJit(static_cast<VMScriptFunction*>(func)))
{
func->ScriptCall = JitCompile(static_cast<VMScriptFunction*>(func));
if (!func->ScriptCall)
func->ScriptCall = VMExec;
}
else
#endif // HAVE_VM_JIT
{
func->ScriptCall = VMExec;
}
static_cast<VMScriptFunction*>(func)->JitCompile();
return func->ScriptCall(func, params, numparams, ret, numret);
}

2
src/common/scripting/vm/vmintern.h
View file

@ -481,4 +481,6 @@ public:
private:
static int FirstScriptCall(VMFunction *func, VMValue *params, int numparams, VMReturn *ret, int numret);
void JitCompile();
friend class FFunctionBuildList;
};

13
src/common/statusbar/base_sbar.cpp
View file

@ -601,7 +601,7 @@ void DStatusBarCore::DrawGraphic(FGameTexture* tex, double x, double y, int flag
DTA_ClipBottom, twod->GetHeight(),
DTA_ClipRight, clipwidth < 0? twod->GetWidth() : int(x + boxwidth * clipwidth),
DTA_Color, color,
DTA_TranslationIndex, translation? translation : (flags & DI_TRANSLATABLE) ? GetTranslation() : 0,
DTA_TranslationIndex, translation? translation : (flags & DI_TRANSLATABLE) ? GetTranslation().index() : 0,
DTA_ColorOverlay, (flags & DI_DIM) ? MAKEARGB(170, 0, 0, 0) : 0,
DTA_Alpha, Alpha,
DTA_AlphaChannel, !!(flags & DI_ALPHAMAPPED),
@ -686,7 +686,7 @@ void DStatusBarCore::DrawRotated(FGameTexture* tex, double x, double y, int flag
DTA_Color, color,
DTA_CenterOffsetRel, !!(flags & DI_ITEM_RELCENTER),
DTA_Rotate, angle,
DTA_TranslationIndex, translation ? translation : (flags & DI_TRANSLATABLE) ? GetTranslation() : 0,
DTA_TranslationIndex, translation ? translation : (flags & DI_TRANSLATABLE) ? GetTranslation().index() : 0,
DTA_ColorOverlay, (flags & DI_DIM) ? MAKEARGB(170, 0, 0, 0) : 0,
DTA_Alpha, Alpha,
DTA_AlphaChannel, !!(flags & DI_ALPHAMAPPED),
@ -704,7 +704,7 @@ void DStatusBarCore::DrawRotated(FGameTexture* tex, double x, double y, int flag
//
//============================================================================
void DStatusBarCore::DrawString(FFont* font, const FString& cstring, double x, double y, int flags, double Alpha, int translation, int spacing, EMonospacing monospacing, int shadowX, int shadowY, double scaleX, double scaleY, int pt, int style)
void DStatusBarCore::DrawString(FFont* font, const FString& cstring, double x, double y, int flags, double Alpha, int translation, int spacing, EMonospacing monospacing, int shadowX, int shadowY, double scaleX, double scaleY, FTranslationID pt, int style)
{
bool monospaced = monospacing != EMonospacing::Off;
double dx = 0;
@ -823,11 +823,11 @@ void DStatusBarCore::DrawString(FFont* font, const FString& cstring, double x, d
DTA_FillColor, 0,
TAG_DONE);
}
DrawChar(twod, font, pt == 0? fontcolor : CR_NATIVEPAL, rx, ry, ch,
DrawChar(twod, font, pt == NO_TRANSLATION? fontcolor : CR_NATIVEPAL, rx, ry, ch,
DTA_DestWidthF, rw,
DTA_DestHeightF, rh,
DTA_Alpha, Alpha,
DTA_TranslationIndex, pt,
DTA_TranslationIndex, pt.index(),
DTA_LegacyRenderStyle, ERenderStyle(style),
TAG_DONE);
@ -840,10 +840,11 @@ void DStatusBarCore::DrawString(FFont* font, const FString& cstring, double x, d
}
}
void SBar_DrawString(DStatusBarCore* self, DHUDFont* font, const FString& string, double x, double y, int flags, int trans, double alpha, int wrapwidth, int linespacing, double scaleX, double scaleY, int pt, int style)
void SBar_DrawString(DStatusBarCore* self, DHUDFont* font, const FString& string, double x, double y, int flags, int trans, double alpha, int wrapwidth, int linespacing, double scaleX, double scaleY, int pt_, int style)
{
if (font == nullptr || font->mFont == nullptr) ThrowAbortException(X_READ_NIL, nullptr);
if (!twod->HasBegun2D()) ThrowAbortException(X_OTHER, "Attempt to draw to screen outside a draw function");
auto pt = FTranslationID::fromInt(pt_);
// resolve auto-alignment before making any adjustments to the position values.
if (!(flags & DI_SCREEN_MANUAL_ALIGN))

4
src/common/statusbar/base_sbar.h
View file

@ -179,7 +179,7 @@ public:
void BeginHUD(int resW, int resH, double Alpha, bool forceScaled = false);
void SetSize(int reltop = 32, int hres = 320, int vres = 200, int hhres = -1, int hvres = -1);
virtual DVector2 GetHUDScale() const;
virtual uint32_t GetTranslation() const { return 0; }
virtual FTranslationID GetTranslation() const { return NO_TRANSLATION; }
void SetDrawSize(int reltop, int hres, int vres);
virtual void SetScale();
void ValidateResolution(int& hres, int& vres) const;
@ -188,7 +188,7 @@ public:
void DrawGraphic(FTextureID texture, double x, double y, int flags, double Alpha, double boxwidth, double boxheight, double scaleX, double scaleY, ERenderStyle style = STYLE_Translucent, PalEntry color = 0xffffffff, int translation = 0, double clipwidth = -1.0);
void DrawRotated(FTextureID texture, double x, double y, int flags, double angle, double Alpha, double scaleX, double scaleY, PalEntry color = 0xffffffff, int translation = 0, ERenderStyle style = STYLE_Translucent);
void DrawRotated(FGameTexture* tex, double x, double y, int flags, double angle, double Alpha, double scaleX, double scaleY, PalEntry color = 0xffffffff, int translation = 0, ERenderStyle style = STYLE_Translucent);
void DrawString(FFont* font, const FString& cstring, double x, double y, int flags, double Alpha, int translation, int spacing, EMonospacing monospacing, int shadowX, int shadowY, double scaleX, double scaleY, int pt, int style);
void DrawString(FFont* font, const FString& cstring, double x, double y, int flags, double Alpha, int translation, int spacing, EMonospacing monospacing, int shadowX, int shadowY, double scaleX, double scaleY, FTranslationID pt, int style);
void TransformRect(double& x, double& y, double& w, double& h, int flags = 0);
void Fill(PalEntry color, double x, double y, double w, double h, int flags = 0);
void SetClipRect(double x, double y, double w, double h, int flags = 0);

3
src/common/textures/formats/anmtexture.cpp
View file

@ -72,9 +72,10 @@ FImageSource *AnmImage_TryCreate(FileReader & file, int lumpnum)
if (memcmp(check, "LPF ", 4)) return nullptr;
file.Seek(0, FileReader::SeekSet);
auto buffer = file.ReadPadded(1);
if (buffer.size() < 4) return nullptr;
std::unique_ptr<anim_t> anim = std::make_unique<anim_t>(); // note that this struct is very large and should not go onto the stack!
if (ANIM_LoadAnim(anim.get(), buffer.data(), buffer.size() - 1) < 0)
if (ANIM_LoadAnim(anim.get(), buffer.bytes(), buffer.size() - 1) < 0)
{
return nullptr;
}

8
src/common/textures/formats/pcxtexture.cpp
View file

@ -161,7 +161,7 @@ void FPCXTexture::ReadPCX1bit (uint8_t *dst, FileReader & lump, PCXHeader *hdr)
uint8_t rle_value = 0;
auto srcp = lump.Read(lump.GetLength() - sizeof(PCXHeader));
uint8_t * src = srcp.data();
const uint8_t * src = srcp.bytes();
for (y = 0; y < Height; ++y)
{
@ -211,7 +211,7 @@ void FPCXTexture::ReadPCX4bits (uint8_t *dst, FileReader & lump, PCXHeader *hdr)
TArray<uint8_t> colorIndex(Width, true);
auto srcp = lump.Read(lump.GetLength() - sizeof(PCXHeader));
uint8_t * src = srcp.data();
const uint8_t * src = srcp.bytes();
for (y = 0; y < Height; ++y)
{
@ -265,7 +265,7 @@ void FPCXTexture::ReadPCX8bits (uint8_t *dst, FileReader & lump, PCXHeader *hdr)
int y, bytes;
auto srcp = lump.Read(lump.GetLength() - sizeof(PCXHeader));
uint8_t * src = srcp.data();
const uint8_t * src = srcp.bytes();
for (y = 0; y < Height; ++y)
{
@ -306,7 +306,7 @@ void FPCXTexture::ReadPCX24bits (uint8_t *dst, FileReader & lump, PCXHeader *hdr
int bytes;
auto srcp = lump.Read(lump.GetLength() - sizeof(PCXHeader));
uint8_t * src = srcp.data();
const uint8_t * src = srcp.bytes();
for (y = 0; y < Height; ++y)
{

4
src/common/textures/formats/webptexture.cpp
View file

@ -65,9 +65,9 @@ FImageSource *WebPImage_TryCreate(FileReader &file, int lumpnum)
file.Seek(0, FileReader::SeekSet);
auto bytes = file.Read();
if (WebPGetInfo(bytes.data(), bytes.size(), &width, &height))
if (WebPGetInfo(bytes.bytes(), bytes.size(), &width, &height))
{
WebPData data{ bytes.data(), bytes.size() };
WebPData data{ bytes.bytes(), bytes.size() };
WebPData chunk_data;
auto mux = WebPMuxCreate(&data, 0);
if (mux)

4
src/common/textures/texture.cpp
View file

@ -50,6 +50,7 @@
#include "c_cvars.h"
#include "imagehelpers.h"
#include "v_video.h"
#include "v_font.h"
// Wrappers to keep the definitions of these classes out of here.
IHardwareTexture* CreateHardwareTexture(int numchannels);
@ -321,7 +322,6 @@ bool FTexture::ProcessData(unsigned char* buffer, int w, int h, bool ispatch)
// Initializes the buffer for the texture data
//
//===========================================================================
void V_ApplyLuminosityTranslation(int translation, uint8_t *buffer, int size);
FTextureBuffer FTexture::CreateTexBuffer(int translation, int flags)
{
@ -367,7 +367,7 @@ FTextureBuffer FTexture::CreateTexBuffer(int translation, int flags)
bmp.Blit(exx, exx, Pixels);
if (IsLuminosityTranslation(translation))
{
V_ApplyLuminosityTranslation(translation, buffer, W * H);
V_ApplyLuminosityTranslation(LuminosityTranslationDesc::fromInt(translation), buffer, W * H);
}
if (remap == nullptr)

3
src/common/textures/textureid.h
View file

@ -1,5 +1,7 @@
#pragma once
#include <cstddef>
enum class ETextureType : uint8_t
{
Any,
@ -28,6 +30,7 @@ class FTextureID
public:
FTextureID() = default;
FTextureID(std::nullptr_t) : texnum(0) {}
bool isNull() const { return texnum == 0; }
bool isValid() const { return texnum > 0; }
bool Exists() const { return texnum >= 0; }

93
src/common/thirdparty/utf8proc/LICENSE.md vendored Normal file
View file

@ -0,0 +1,93 @@
## utf8proc license ##
**utf8proc** is a software package originally developed
by Jan Behrens and the rest of the Public Software Group, who
deserve nearly all of the credit for this library, that is now maintained by the Julia-language developers. Like the original utf8proc,
whose copyright and license statements are reproduced below, all new
work on the utf8proc library is licensed under the [MIT "expat"
license](http://opensource.org/licenses/MIT):
*Copyright &copy; 2014-2021 by Steven G. Johnson, Jiahao Chen, Tony Kelman, Jonas Fonseca, and other contributors listed in the git history.*
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
## Original utf8proc license ##
*Copyright (c) 2009, 2013 Public Software Group e. V., Berlin, Germany*
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
## Unicode data license ##
This software contains data (`utf8proc_data.c`) derived from processing
the Unicode data files. The following license applies to that data:
**COPYRIGHT AND PERMISSION NOTICE**
*Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed
under the Terms of Use in http://www.unicode.org/copyright.html.*
Permission is hereby granted, free of charge, to any person obtaining a
copy of the Unicode data files and any associated documentation (the "Data
Files") or Unicode software and any associated documentation (the
"Software") to deal in the Data Files or Software without restriction,
including without limitation the rights to use, copy, modify, merge,
publish, distribute, and/or sell copies of the Data Files or Software, and
to permit persons to whom the Data Files or Software are furnished to do
so, provided that (a) the above copyright notice(s) and this permission
notice appear with all copies of the Data Files or Software, (b) both the
above copyright notice(s) and this permission notice appear in associated
documentation, and (c) there is clear notice in each modified Data File or
in the Software as well as in the documentation associated with the Data
File(s) or Software that the data or software has been modified.
THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS
INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR
CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF
USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THE DATA FILES OR SOFTWARE.
Except as contained in this notice, the name of a copyright holder shall
not be used in advertising or otherwise to promote the sale, use or other
dealings in these Data Files or Software without prior written
authorization of the copyright holder.
Unicode and the Unicode logo are trademarks of Unicode, Inc., and may be
registered in some jurisdictions. All other trademarks and registered
trademarks mentioned herein are the property of their respective owners.

815
src/common/thirdparty/utf8proc/utf8proc.c vendored Normal file
View file

@ -0,0 +1,815 @@
/* -*- mode: c; c-basic-offset: 2; tab-width: 2; indent-tabs-mode: nil -*- */
/*
* Copyright (c) 2014-2021 Steven G. Johnson, Jiahao Chen, Peter Colberg, Tony Kelman, Scott P. Jones, and other contributors.
* Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
/*
* This library contains derived data from a modified version of the
* Unicode data files.
*
* The original data files are available at
* https://www.unicode.org/Public/UNIDATA/
*
* Please notice the copyright statement in the file "utf8proc_data.c".
*/
/*
* File name: utf8proc.c
*
* Description:
* Implementation of libutf8proc.
*/
#include "utf8proc.h"
#ifndef SSIZE_MAX
#define SSIZE_MAX ((size_t)SIZE_MAX/2)
#endif
#ifndef UINT16_MAX
# define UINT16_MAX 65535U
#endif
#include "utf8proc_data.c"
UTF8PROC_DLLEXPORT const utf8proc_int8_t utf8proc_utf8class[256] = {
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0 };
#define UTF8PROC_HANGUL_SBASE 0xAC00
#define UTF8PROC_HANGUL_LBASE 0x1100
#define UTF8PROC_HANGUL_VBASE 0x1161
#define UTF8PROC_HANGUL_TBASE 0x11A7
#define UTF8PROC_HANGUL_LCOUNT 19
#define UTF8PROC_HANGUL_VCOUNT 21
#define UTF8PROC_HANGUL_TCOUNT 28
#define UTF8PROC_HANGUL_NCOUNT 588
#define UTF8PROC_HANGUL_SCOUNT 11172
/* END is exclusive */
#define UTF8PROC_HANGUL_L_START 0x1100
#define UTF8PROC_HANGUL_L_END 0x115A
#define UTF8PROC_HANGUL_L_FILLER 0x115F
#define UTF8PROC_HANGUL_V_START 0x1160
#define UTF8PROC_HANGUL_V_END 0x11A3
#define UTF8PROC_HANGUL_T_START 0x11A8
#define UTF8PROC_HANGUL_T_END 0x11FA
#define UTF8PROC_HANGUL_S_START 0xAC00
#define UTF8PROC_HANGUL_S_END 0xD7A4
/* Should follow semantic-versioning rules (semver.org) based on API
compatibility. (Note that the shared-library version number will
be different, being based on ABI compatibility.): */
#define STRINGIZEx(x) #x
#define STRINGIZE(x) STRINGIZEx(x)
UTF8PROC_DLLEXPORT const char *utf8proc_version(void) {
return STRINGIZE(UTF8PROC_VERSION_MAJOR) "." STRINGIZE(UTF8PROC_VERSION_MINOR) "." STRINGIZE(UTF8PROC_VERSION_PATCH) "";
}
UTF8PROC_DLLEXPORT const char *utf8proc_unicode_version(void) {
return "15.1.0";
}
UTF8PROC_DLLEXPORT const char *utf8proc_errmsg(utf8proc_ssize_t errcode) {
switch (errcode) {
case UTF8PROC_ERROR_NOMEM:
return "Memory for processing UTF-8 data could not be allocated.";
case UTF8PROC_ERROR_OVERFLOW:
return "UTF-8 string is too long to be processed.";
case UTF8PROC_ERROR_INVALIDUTF8:
return "Invalid UTF-8 string";
case UTF8PROC_ERROR_NOTASSIGNED:
return "Unassigned Unicode code point found in UTF-8 string.";
case UTF8PROC_ERROR_INVALIDOPTS:
return "Invalid options for UTF-8 processing chosen.";
default:
return "An unknown error occurred while processing UTF-8 data.";
}
}
#define utf_cont(ch) (((ch) & 0xc0) == 0x80)
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_iterate(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *dst
) {
utf8proc_int32_t uc;
const utf8proc_uint8_t *end;
*dst = -1;
if (!strlen) return 0;
end = str + ((strlen < 0) ? 4 : strlen);
uc = *str++;
if (uc < 0x80) {
*dst = uc;
return 1;
}
// Must be between 0xc2 and 0xf4 inclusive to be valid
if ((utf8proc_uint32_t)(uc - 0xc2) > (0xf4-0xc2)) return UTF8PROC_ERROR_INVALIDUTF8;
if (uc < 0xe0) { // 2-byte sequence
// Must have valid continuation character
if (str >= end || !utf_cont(*str)) return UTF8PROC_ERROR_INVALIDUTF8;
*dst = ((uc & 0x1f)<<6) | (*str & 0x3f);
return 2;
}
if (uc < 0xf0) { // 3-byte sequence
if ((str + 1 >= end) || !utf_cont(*str) || !utf_cont(str[1]))
return UTF8PROC_ERROR_INVALIDUTF8;
// Check for surrogate chars
if (uc == 0xed && *str > 0x9f)
return UTF8PROC_ERROR_INVALIDUTF8;
uc = ((uc & 0xf)<<12) | ((*str & 0x3f)<<6) | (str[1] & 0x3f);
if (uc < 0x800)
return UTF8PROC_ERROR_INVALIDUTF8;
*dst = uc;
return 3;
}
// 4-byte sequence
// Must have 3 valid continuation characters
if ((str + 2 >= end) || !utf_cont(*str) || !utf_cont(str[1]) || !utf_cont(str[2]))
return UTF8PROC_ERROR_INVALIDUTF8;
// Make sure in correct range (0x10000 - 0x10ffff)
if (uc == 0xf0) {
if (*str < 0x90) return UTF8PROC_ERROR_INVALIDUTF8;
} else if (uc == 0xf4) {
if (*str > 0x8f) return UTF8PROC_ERROR_INVALIDUTF8;
}
*dst = ((uc & 7)<<18) | ((*str & 0x3f)<<12) | ((str[1] & 0x3f)<<6) | (str[2] & 0x3f);
return 4;
}
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_codepoint_valid(utf8proc_int32_t uc) {
return (((utf8proc_uint32_t)uc)-0xd800 > 0x07ff) && ((utf8proc_uint32_t)uc < 0x110000);
}
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst) {
if (uc < 0x00) {
return 0;
} else if (uc < 0x80) {
dst[0] = (utf8proc_uint8_t) uc;
return 1;
} else if (uc < 0x800) {
dst[0] = (utf8proc_uint8_t)(0xC0 + (uc >> 6));
dst[1] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
return 2;
// Note: we allow encoding 0xd800-0xdfff here, so as not to change
// the API, however, these are actually invalid in UTF-8
} else if (uc < 0x10000) {
dst[0] = (utf8proc_uint8_t)(0xE0 + (uc >> 12));
dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
dst[2] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
return 3;
} else if (uc < 0x110000) {
dst[0] = (utf8proc_uint8_t)(0xF0 + (uc >> 18));
dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 12) & 0x3F));
dst[2] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
dst[3] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
return 4;
} else return 0;
}
/* internal version used for inserting 0xff bytes between graphemes */
static utf8proc_ssize_t charbound_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst) {
if (uc < 0x00) {
if (uc == -1) { /* internal value used for grapheme breaks */
dst[0] = (utf8proc_uint8_t)0xFF;
return 1;
}
return 0;
} else if (uc < 0x80) {
dst[0] = (utf8proc_uint8_t)uc;
return 1;
} else if (uc < 0x800) {
dst[0] = (utf8proc_uint8_t)(0xC0 + (uc >> 6));
dst[1] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
return 2;
} else if (uc < 0x10000) {
dst[0] = (utf8proc_uint8_t)(0xE0 + (uc >> 12));
dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
dst[2] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
return 3;
} else if (uc < 0x110000) {
dst[0] = (utf8proc_uint8_t)(0xF0 + (uc >> 18));
dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 12) & 0x3F));
dst[2] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
dst[3] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
return 4;
} else return 0;
}
/* internal "unsafe" version that does not check whether uc is in range */
static const utf8proc_property_t *unsafe_get_property(utf8proc_int32_t uc) {
/* ASSERT: uc >= 0 && uc < 0x110000 */
return utf8proc_properties + (
utf8proc_stage2table[
utf8proc_stage1table[uc >> 8] + (uc & 0xFF)
]
);
}
UTF8PROC_DLLEXPORT const utf8proc_property_t *utf8proc_get_property(utf8proc_int32_t uc) {
return uc < 0 || uc >= 0x110000 ? utf8proc_properties : unsafe_get_property(uc);
}
/* return whether there is a grapheme break between boundclasses lbc and tbc
(according to the definition of extended grapheme clusters)
Rule numbering refers to TR29 Version 29 (Unicode 9.0.0):
http://www.unicode.org/reports/tr29/tr29-29.html
CAVEATS:
Please note that evaluation of GB10 (grapheme breaks between emoji zwj sequences)
and GB 12/13 (regional indicator code points) require knowledge of previous characters
and are thus not handled by this function. This may result in an incorrect break before
an E_Modifier class codepoint and an incorrectly missing break between two
REGIONAL_INDICATOR class code points if such support does not exist in the caller.
See the special support in grapheme_break_extended, for required bookkeeping by the caller.
*/
static utf8proc_bool grapheme_break_simple(int lbc, int tbc) {
return
(lbc == UTF8PROC_BOUNDCLASS_START) ? true : // GB1
(lbc == UTF8PROC_BOUNDCLASS_CR && // GB3
tbc == UTF8PROC_BOUNDCLASS_LF) ? false : // ---
(lbc >= UTF8PROC_BOUNDCLASS_CR && lbc <= UTF8PROC_BOUNDCLASS_CONTROL) ? true : // GB4
(tbc >= UTF8PROC_BOUNDCLASS_CR && tbc <= UTF8PROC_BOUNDCLASS_CONTROL) ? true : // GB5
(lbc == UTF8PROC_BOUNDCLASS_L && // GB6
(tbc == UTF8PROC_BOUNDCLASS_L || // ---
tbc == UTF8PROC_BOUNDCLASS_V || // ---
tbc == UTF8PROC_BOUNDCLASS_LV || // ---
tbc == UTF8PROC_BOUNDCLASS_LVT)) ? false : // ---
((lbc == UTF8PROC_BOUNDCLASS_LV || // GB7
lbc == UTF8PROC_BOUNDCLASS_V) && // ---
(tbc == UTF8PROC_BOUNDCLASS_V || // ---
tbc == UTF8PROC_BOUNDCLASS_T)) ? false : // ---
((lbc == UTF8PROC_BOUNDCLASS_LVT || // GB8
lbc == UTF8PROC_BOUNDCLASS_T) && // ---
tbc == UTF8PROC_BOUNDCLASS_T) ? false : // ---
(tbc == UTF8PROC_BOUNDCLASS_EXTEND || // GB9
tbc == UTF8PROC_BOUNDCLASS_ZWJ || // ---
tbc == UTF8PROC_BOUNDCLASS_SPACINGMARK || // GB9a
lbc == UTF8PROC_BOUNDCLASS_PREPEND) ? false : // GB9b
(lbc == UTF8PROC_BOUNDCLASS_E_ZWG && // GB11 (requires additional handling below)
tbc == UTF8PROC_BOUNDCLASS_EXTENDED_PICTOGRAPHIC) ? false : // ----
(lbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR && // GB12/13 (requires additional handling below)
tbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR) ? false : // ----
true; // GB999
}
static utf8proc_bool grapheme_break_extended(int lbc, int tbc, int licb, int ticb, utf8proc_int32_t *state)
{
if (state) {
int state_bc, state_icb; /* boundclass and indic_conjunct_break state */
if (*state == 0) { /* state initialization */
state_bc = lbc;
state_icb = licb == UTF8PROC_INDIC_CONJUNCT_BREAK_CONSONANT ? licb : UTF8PROC_INDIC_CONJUNCT_BREAK_NONE;
}
else { /* lbc and licb are already encoded in *state */
state_bc = *state & 0xff; // 1st byte of state is bound class
state_icb = *state >> 8; // 2nd byte of state is indic conjunct break
}
utf8proc_bool break_permitted = grapheme_break_simple(state_bc, tbc) &&
!(state_icb == UTF8PROC_INDIC_CONJUNCT_BREAK_LINKER
&& ticb == UTF8PROC_INDIC_CONJUNCT_BREAK_CONSONANT); // GB9c
// Special support for GB9c. Don't break between two consonants
// separated 1+ linker characters and 0+ extend characters in any order.
// After a consonant, we enter LINKER state after at least one linker.
if (ticb == UTF8PROC_INDIC_CONJUNCT_BREAK_CONSONANT
|| state_icb == UTF8PROC_INDIC_CONJUNCT_BREAK_CONSONANT
|| state_icb == UTF8PROC_INDIC_CONJUNCT_BREAK_EXTEND)
state_icb = ticb;
else if (state_icb == UTF8PROC_INDIC_CONJUNCT_BREAK_LINKER)
state_icb = ticb == UTF8PROC_INDIC_CONJUNCT_BREAK_EXTEND ?
UTF8PROC_INDIC_CONJUNCT_BREAK_LINKER : ticb;
// Special support for GB 12/13 made possible by GB999. After two RI
// class codepoints we want to force a break. Do this by resetting the
// second RI's bound class to UTF8PROC_BOUNDCLASS_OTHER, to force a break
// after that character according to GB999 (unless of course such a break is
// forbidden by a different rule such as GB9).
if (state_bc == tbc && tbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR)
state_bc = UTF8PROC_BOUNDCLASS_OTHER;
// Special support for GB11 (emoji extend* zwj / emoji)
else if (state_bc == UTF8PROC_BOUNDCLASS_EXTENDED_PICTOGRAPHIC) {
if (tbc == UTF8PROC_BOUNDCLASS_EXTEND) // fold EXTEND codepoints into emoji
state_bc = UTF8PROC_BOUNDCLASS_EXTENDED_PICTOGRAPHIC;
else if (tbc == UTF8PROC_BOUNDCLASS_ZWJ)
state_bc = UTF8PROC_BOUNDCLASS_E_ZWG; // state to record emoji+zwg combo
else
state_bc = tbc;
}
else
state_bc = tbc;
*state = state_bc + (state_icb << 8);
return break_permitted;
}
else
return grapheme_break_simple(lbc, tbc);
}
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break_stateful(
utf8proc_int32_t c1, utf8proc_int32_t c2, utf8proc_int32_t *state) {
const utf8proc_property_t *p1 = utf8proc_get_property(c1);
const utf8proc_property_t *p2 = utf8proc_get_property(c2);
return grapheme_break_extended(p1->boundclass,
p2->boundclass,
p1->indic_conjunct_break,
p2->indic_conjunct_break,
state);
}
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break(
utf8proc_int32_t c1, utf8proc_int32_t c2) {
return utf8proc_grapheme_break_stateful(c1, c2, NULL);
}
static utf8proc_int32_t seqindex_decode_entry(const utf8proc_uint16_t **entry)
{
utf8proc_int32_t entry_cp = **entry;
if ((entry_cp & 0xF800) == 0xD800) {
*entry = *entry + 1;
entry_cp = ((entry_cp & 0x03FF) << 10) | (**entry & 0x03FF);
entry_cp += 0x10000;
}
return entry_cp;
}
static utf8proc_int32_t seqindex_decode_index(const utf8proc_uint32_t seqindex)
{
const utf8proc_uint16_t *entry = &utf8proc_sequences[seqindex];
return seqindex_decode_entry(&entry);
}
static utf8proc_ssize_t seqindex_write_char_decomposed(utf8proc_uint16_t seqindex, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize, utf8proc_option_t options, int *last_boundclass) {
utf8proc_ssize_t written = 0;
const utf8proc_uint16_t *entry = &utf8proc_sequences[seqindex & 0x3FFF];
int len = seqindex >> 14;
if (len >= 3) {
len = *entry;
entry++;
}
for (; len >= 0; entry++, len--) {
utf8proc_int32_t entry_cp = seqindex_decode_entry(&entry);
written += utf8proc_decompose_char(entry_cp, dst+written,
(bufsize > written) ? (bufsize - written) : 0, options,
last_boundclass);
if (written < 0) return UTF8PROC_ERROR_OVERFLOW;
}
return written;
}
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_tolower(utf8proc_int32_t c)
{
utf8proc_int32_t cl = utf8proc_get_property(c)->lowercase_seqindex;
return cl != UINT16_MAX ? seqindex_decode_index((utf8proc_uint32_t)cl) : c;
}
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_toupper(utf8proc_int32_t c)
{
utf8proc_int32_t cu = utf8proc_get_property(c)->uppercase_seqindex;
return cu != UINT16_MAX ? seqindex_decode_index((utf8proc_uint32_t)cu) : c;
}
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_totitle(utf8proc_int32_t c)
{
utf8proc_int32_t cu = utf8proc_get_property(c)->titlecase_seqindex;
return cu != UINT16_MAX ? seqindex_decode_index((utf8proc_uint32_t)cu) : c;
}
UTF8PROC_DLLEXPORT int utf8proc_islower(utf8proc_int32_t c)
{
const utf8proc_property_t *p = utf8proc_get_property(c);
return p->lowercase_seqindex != p->uppercase_seqindex && p->lowercase_seqindex == UINT16_MAX;
}
UTF8PROC_DLLEXPORT int utf8proc_isupper(utf8proc_int32_t c)
{
const utf8proc_property_t *p = utf8proc_get_property(c);
return p->lowercase_seqindex != p->uppercase_seqindex && p->uppercase_seqindex == UINT16_MAX && p->category != UTF8PROC_CATEGORY_LT;
}
/* return a character width analogous to wcwidth (except portable and
hopefully less buggy than most system wcwidth functions). */
UTF8PROC_DLLEXPORT int utf8proc_charwidth(utf8proc_int32_t c) {
return utf8proc_get_property(c)->charwidth;
}
UTF8PROC_DLLEXPORT utf8proc_category_t utf8proc_category(utf8proc_int32_t c) {
return (utf8proc_category_t) utf8proc_get_property(c)->category;
}
UTF8PROC_DLLEXPORT const char *utf8proc_category_string(utf8proc_int32_t c) {
static const char s[][3] = {"Cn","Lu","Ll","Lt","Lm","Lo","Mn","Mc","Me","Nd","Nl","No","Pc","Pd","Ps","Pe","Pi","Pf","Po","Sm","Sc","Sk","So","Zs","Zl","Zp","Cc","Cf","Cs","Co"};
return s[utf8proc_category(c)];
}
#define utf8proc_decompose_lump(replacement_uc) \
return utf8proc_decompose_char((replacement_uc), dst, bufsize, \
options & ~(unsigned int)UTF8PROC_LUMP, last_boundclass)
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char(utf8proc_int32_t uc, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize, utf8proc_option_t options, int *last_boundclass) {
const utf8proc_property_t *property;
utf8proc_propval_t category;
utf8proc_int32_t hangul_sindex;
if (uc < 0 || uc >= 0x110000) return UTF8PROC_ERROR_NOTASSIGNED;
property = unsafe_get_property(uc);
category = property->category;
hangul_sindex = uc - UTF8PROC_HANGUL_SBASE;
if (options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) {
if (hangul_sindex >= 0 && hangul_sindex < UTF8PROC_HANGUL_SCOUNT) {
utf8proc_int32_t hangul_tindex;
if (bufsize >= 1) {
dst[0] = UTF8PROC_HANGUL_LBASE +
hangul_sindex / UTF8PROC_HANGUL_NCOUNT;
if (bufsize >= 2) dst[1] = UTF8PROC_HANGUL_VBASE +
(hangul_sindex % UTF8PROC_HANGUL_NCOUNT) / UTF8PROC_HANGUL_TCOUNT;
}
hangul_tindex = hangul_sindex % UTF8PROC_HANGUL_TCOUNT;
if (!hangul_tindex) return 2;
if (bufsize >= 3) dst[2] = UTF8PROC_HANGUL_TBASE + hangul_tindex;
return 3;
}
}
if (options & UTF8PROC_REJECTNA) {
if (!category) return UTF8PROC_ERROR_NOTASSIGNED;
}
if (options & UTF8PROC_IGNORE) {
if (property->ignorable) return 0;
}
if (options & UTF8PROC_STRIPNA) {
if (!category) return 0;
}
if (options & UTF8PROC_LUMP) {
if (category == UTF8PROC_CATEGORY_ZS) utf8proc_decompose_lump(0x0020);
if (uc == 0x2018 || uc == 0x2019 || uc == 0x02BC || uc == 0x02C8)
utf8proc_decompose_lump(0x0027);
if (category == UTF8PROC_CATEGORY_PD || uc == 0x2212)
utf8proc_decompose_lump(0x002D);
if (uc == 0x2044 || uc == 0x2215) utf8proc_decompose_lump(0x002F);
if (uc == 0x2236) utf8proc_decompose_lump(0x003A);
if (uc == 0x2039 || uc == 0x2329 || uc == 0x3008)
utf8proc_decompose_lump(0x003C);
if (uc == 0x203A || uc == 0x232A || uc == 0x3009)
utf8proc_decompose_lump(0x003E);
if (uc == 0x2216) utf8proc_decompose_lump(0x005C);
if (uc == 0x02C4 || uc == 0x02C6 || uc == 0x2038 || uc == 0x2303)
utf8proc_decompose_lump(0x005E);
if (category == UTF8PROC_CATEGORY_PC || uc == 0x02CD)
utf8proc_decompose_lump(0x005F);
if (uc == 0x02CB) utf8proc_decompose_lump(0x0060);
if (uc == 0x2223) utf8proc_decompose_lump(0x007C);
if (uc == 0x223C) utf8proc_decompose_lump(0x007E);
if ((options & UTF8PROC_NLF2LS) && (options & UTF8PROC_NLF2PS)) {
if (category == UTF8PROC_CATEGORY_ZL ||
category == UTF8PROC_CATEGORY_ZP)
utf8proc_decompose_lump(0x000A);
}
}
if (options & UTF8PROC_STRIPMARK) {
if (category == UTF8PROC_CATEGORY_MN ||
category == UTF8PROC_CATEGORY_MC ||
category == UTF8PROC_CATEGORY_ME) return 0;
}
if (options & UTF8PROC_CASEFOLD) {
if (property->casefold_seqindex != UINT16_MAX) {
return seqindex_write_char_decomposed(property->casefold_seqindex, dst, bufsize, options, last_boundclass);
}
}
if (options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) {
if (property->decomp_seqindex != UINT16_MAX &&
(!property->decomp_type || (options & UTF8PROC_COMPAT))) {
return seqindex_write_char_decomposed(property->decomp_seqindex, dst, bufsize, options, last_boundclass);
}
}
if (options & UTF8PROC_CHARBOUND) {
utf8proc_bool boundary;
boundary = grapheme_break_extended(0, property->boundclass, 0, property->indic_conjunct_break,
last_boundclass);
if (boundary) {
if (bufsize >= 1) dst[0] = -1; /* sentinel value for grapheme break */
if (bufsize >= 2) dst[1] = uc;
return 2;
}
}
if (bufsize >= 1) *dst = uc;
return 1;
}
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options
) {
return utf8proc_decompose_custom(str, strlen, buffer, bufsize, options, NULL, NULL);
}
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_custom(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options,
utf8proc_custom_func custom_func, void *custom_data
) {
/* strlen will be ignored, if UTF8PROC_NULLTERM is set in options */
utf8proc_ssize_t wpos = 0;
if ((options & UTF8PROC_COMPOSE) && (options & UTF8PROC_DECOMPOSE))
return UTF8PROC_ERROR_INVALIDOPTS;
if ((options & UTF8PROC_STRIPMARK) &&
!(options & UTF8PROC_COMPOSE) && !(options & UTF8PROC_DECOMPOSE))
return UTF8PROC_ERROR_INVALIDOPTS;
{
utf8proc_int32_t uc;
utf8proc_ssize_t rpos = 0;
utf8proc_ssize_t decomp_result;
int boundclass = UTF8PROC_BOUNDCLASS_START;
while (1) {
if (options & UTF8PROC_NULLTERM) {
rpos += utf8proc_iterate(str + rpos, -1, &uc);
/* checking of return value is not necessary,
as 'uc' is < 0 in case of error */
if (uc < 0) return UTF8PROC_ERROR_INVALIDUTF8;
if (rpos < 0) return UTF8PROC_ERROR_OVERFLOW;
if (uc == 0) break;
} else {
if (rpos >= strlen) break;
rpos += utf8proc_iterate(str + rpos, strlen - rpos, &uc);
if (uc < 0) return UTF8PROC_ERROR_INVALIDUTF8;
}
if (custom_func != NULL) {
uc = custom_func(uc, custom_data); /* user-specified custom mapping */
}
decomp_result = utf8proc_decompose_char(
uc, buffer + wpos, (bufsize > wpos) ? (bufsize - wpos) : 0, options,
&boundclass
);
if (decomp_result < 0) return decomp_result;
wpos += decomp_result;
/* prohibiting integer overflows due to too long strings: */
if (wpos < 0 ||
wpos > (utf8proc_ssize_t)(SSIZE_MAX/sizeof(utf8proc_int32_t)/2))
return UTF8PROC_ERROR_OVERFLOW;
}
}
if ((options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) && bufsize >= wpos) {
utf8proc_ssize_t pos = 0;
while (pos < wpos-1) {
utf8proc_int32_t uc1, uc2;
const utf8proc_property_t *property1, *property2;
uc1 = buffer[pos];
uc2 = buffer[pos+1];
property1 = unsafe_get_property(uc1);
property2 = unsafe_get_property(uc2);
if (property1->combining_class > property2->combining_class &&
property2->combining_class > 0) {
buffer[pos] = uc2;
buffer[pos+1] = uc1;
if (pos > 0) pos--; else pos++;
} else {
pos++;
}
}
}
return wpos;
}
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_normalize_utf32(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options) {
/* UTF8PROC_NULLTERM option will be ignored, 'length' is never ignored */
if (options & (UTF8PROC_NLF2LS | UTF8PROC_NLF2PS | UTF8PROC_STRIPCC)) {
utf8proc_ssize_t rpos;
utf8proc_ssize_t wpos = 0;
utf8proc_int32_t uc;
for (rpos = 0; rpos < length; rpos++) {
uc = buffer[rpos];
if (uc == 0x000D && rpos < length-1 && buffer[rpos+1] == 0x000A) rpos++;
if (uc == 0x000A || uc == 0x000D || uc == 0x0085 ||
((options & UTF8PROC_STRIPCC) && (uc == 0x000B || uc == 0x000C))) {
if (options & UTF8PROC_NLF2LS) {
if (options & UTF8PROC_NLF2PS) {
buffer[wpos++] = 0x000A;
} else {
buffer[wpos++] = 0x2028;
}
} else {
if (options & UTF8PROC_NLF2PS) {
buffer[wpos++] = 0x2029;
} else {
buffer[wpos++] = 0x0020;
}
}
} else if ((options & UTF8PROC_STRIPCC) &&
(uc < 0x0020 || (uc >= 0x007F && uc < 0x00A0))) {
if (uc == 0x0009) buffer[wpos++] = 0x0020;
} else {
buffer[wpos++] = uc;
}
}
length = wpos;
}
if (options & UTF8PROC_COMPOSE) {
utf8proc_int32_t *starter = NULL;
utf8proc_int32_t current_char;
const utf8proc_property_t *starter_property = NULL, *current_property;
utf8proc_propval_t max_combining_class = -1;
utf8proc_ssize_t rpos;
utf8proc_ssize_t wpos = 0;
utf8proc_int32_t composition;
for (rpos = 0; rpos < length; rpos++) {
current_char = buffer[rpos];
current_property = unsafe_get_property(current_char);
if (starter && current_property->combining_class > max_combining_class) {
/* combination perhaps possible */
utf8proc_int32_t hangul_lindex;
utf8proc_int32_t hangul_sindex;
hangul_lindex = *starter - UTF8PROC_HANGUL_LBASE;
if (hangul_lindex >= 0 && hangul_lindex < UTF8PROC_HANGUL_LCOUNT) {
utf8proc_int32_t hangul_vindex;
hangul_vindex = current_char - UTF8PROC_HANGUL_VBASE;
if (hangul_vindex >= 0 && hangul_vindex < UTF8PROC_HANGUL_VCOUNT) {
*starter = UTF8PROC_HANGUL_SBASE +
(hangul_lindex * UTF8PROC_HANGUL_VCOUNT + hangul_vindex) *
UTF8PROC_HANGUL_TCOUNT;
starter_property = NULL;
continue;
}
}
hangul_sindex = *starter - UTF8PROC_HANGUL_SBASE;
if (hangul_sindex >= 0 && hangul_sindex < UTF8PROC_HANGUL_SCOUNT &&
(hangul_sindex % UTF8PROC_HANGUL_TCOUNT) == 0) {
utf8proc_int32_t hangul_tindex;
hangul_tindex = current_char - UTF8PROC_HANGUL_TBASE;
if (hangul_tindex >= 0 && hangul_tindex < UTF8PROC_HANGUL_TCOUNT) {
*starter += hangul_tindex;
starter_property = NULL;
continue;
}
}
if (!starter_property) {
starter_property = unsafe_get_property(*starter);
}
if (starter_property->comb_index < 0x8000 &&
current_property->comb_index != UINT16_MAX &&
current_property->comb_index >= 0x8000) {
int sidx = starter_property->comb_index;
int idx = current_property->comb_index & 0x3FFF;
if (idx >= utf8proc_combinations[sidx] && idx <= utf8proc_combinations[sidx + 1] ) {
idx += sidx + 2 - utf8proc_combinations[sidx];
if (current_property->comb_index & 0x4000) {
composition = (utf8proc_combinations[idx] << 16) | utf8proc_combinations[idx+1];
} else
composition = utf8proc_combinations[idx];
if (composition > 0 && (!(options & UTF8PROC_STABLE) ||
!(unsafe_get_property(composition)->comp_exclusion))) {
*starter = composition;
starter_property = NULL;
continue;
}
}
}
}
buffer[wpos] = current_char;
if (current_property->combining_class) {
if (current_property->combining_class > max_combining_class) {
max_combining_class = current_property->combining_class;
}
} else {
starter = buffer + wpos;
starter_property = NULL;
max_combining_class = -1;
}
wpos++;
}
length = wpos;
}
return length;
}
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options) {
/* UTF8PROC_NULLTERM option will be ignored, 'length' is never ignored
ASSERT: 'buffer' has one spare byte of free space at the end! */
length = utf8proc_normalize_utf32(buffer, length, options);
if (length < 0) return length;
{
utf8proc_ssize_t rpos, wpos = 0;
utf8proc_int32_t uc;
if (options & UTF8PROC_CHARBOUND) {
for (rpos = 0; rpos < length; rpos++) {
uc = buffer[rpos];
wpos += charbound_encode_char(uc, ((utf8proc_uint8_t *)buffer) + wpos);
}
} else {
for (rpos = 0; rpos < length; rpos++) {
uc = buffer[rpos];
wpos += utf8proc_encode_char(uc, ((utf8proc_uint8_t *)buffer) + wpos);
}
}
((utf8proc_uint8_t *)buffer)[wpos] = 0;
return wpos;
}
}
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options
) {
return utf8proc_map_custom(str, strlen, dstptr, options, NULL, NULL);
}
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map_custom(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options,
utf8proc_custom_func custom_func, void *custom_data
) {
utf8proc_int32_t *buffer;
utf8proc_ssize_t result;
*dstptr = NULL;
result = utf8proc_decompose_custom(str, strlen, NULL, 0, options, custom_func, custom_data);
if (result < 0) return result;
buffer = (utf8proc_int32_t *) malloc(((utf8proc_size_t)result) * sizeof(utf8proc_int32_t) + 1);
if (!buffer) return UTF8PROC_ERROR_NOMEM;
result = utf8proc_decompose_custom(str, strlen, buffer, result, options, custom_func, custom_data);
if (result < 0) {
free(buffer);
return result;
}
result = utf8proc_reencode(buffer, result, options);
if (result < 0) {
free(buffer);
return result;
}
{
utf8proc_int32_t *newptr;
newptr = (utf8proc_int32_t *) realloc(buffer, (size_t)result+1);
if (newptr) buffer = newptr;
}
*dstptr = (utf8proc_uint8_t *)buffer;
return result;
}
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFD(const utf8proc_uint8_t *str) {
utf8proc_uint8_t *retval;
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
UTF8PROC_DECOMPOSE);
return retval;
}
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFC(const utf8proc_uint8_t *str) {
utf8proc_uint8_t *retval;
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
UTF8PROC_COMPOSE);
return retval;
}
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKD(const utf8proc_uint8_t *str) {
utf8proc_uint8_t *retval;
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT);
return retval;
}
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC(const utf8proc_uint8_t *str) {
utf8proc_uint8_t *retval;
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
UTF8PROC_COMPOSE | UTF8PROC_COMPAT);
return retval;
}
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC_Casefold(const utf8proc_uint8_t *str) {
utf8proc_uint8_t *retval;
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
UTF8PROC_COMPOSE | UTF8PROC_COMPAT | UTF8PROC_CASEFOLD | UTF8PROC_IGNORE);
return retval;
}

745
src/common/thirdparty/utf8proc/utf8proc.h vendored Normal file
View file

@ -0,0 +1,745 @@
/*
* Copyright (c) 2014-2021 Steven G. Johnson, Jiahao Chen, Peter Colberg, Tony Kelman, Scott P. Jones, and other contributors.
* Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
/**
* @mainpage
*
* utf8proc is a free/open-source (MIT/expat licensed) C library
* providing Unicode normalization, case-folding, and other operations
* for strings in the UTF-8 encoding, supporting up-to-date Unicode versions.
* See the utf8proc home page (http://julialang.org/utf8proc/)
* for downloads and other information, or the source code on github
* (https://github.com/JuliaLang/utf8proc).
*
* For the utf8proc API documentation, see: @ref utf8proc.h
*
* The features of utf8proc include:
*
* - Transformation of strings (utf8proc_map()) to:
* - decompose (@ref UTF8PROC_DECOMPOSE) or compose (@ref UTF8PROC_COMPOSE) Unicode combining characters (http://en.wikipedia.org/wiki/Combining_character)
* - canonicalize Unicode compatibility characters (@ref UTF8PROC_COMPAT)
* - strip "ignorable" (@ref UTF8PROC_IGNORE) characters, control characters (@ref UTF8PROC_STRIPCC), or combining characters such as accents (@ref UTF8PROC_STRIPMARK)
* - case-folding (@ref UTF8PROC_CASEFOLD)
* - Unicode normalization: utf8proc_NFD(), utf8proc_NFC(), utf8proc_NFKD(), utf8proc_NFKC()
* - Detecting grapheme boundaries (utf8proc_grapheme_break() and @ref UTF8PROC_CHARBOUND)
* - Character-width computation: utf8proc_charwidth()
* - Classification of characters by Unicode category: utf8proc_category() and utf8proc_category_string()
* - Encode (utf8proc_encode_char()) and decode (utf8proc_iterate()) Unicode codepoints to/from UTF-8.
*/
/** @file */
#ifndef UTF8PROC_H
#define UTF8PROC_H
#define UTF8PROC_STATIC
/** @name API version
*
* The utf8proc API version MAJOR.MINOR.PATCH, following
* semantic-versioning rules (http://semver.org) based on API
* compatibility.
*
* This is also returned at runtime by utf8proc_version(); however, the
* runtime version may append a string like "-dev" to the version number
* for prerelease versions.
*
* @note The shared-library version number in the Makefile
* (and CMakeLists.txt, and MANIFEST) may be different,
* being based on ABI compatibility rather than API compatibility.
*/
/** @{ */
/** The MAJOR version number (increased when backwards API compatibility is broken). */
#define UTF8PROC_VERSION_MAJOR 2
/** The MINOR version number (increased when new functionality is added in a backwards-compatible manner). */
#define UTF8PROC_VERSION_MINOR 9
/** The PATCH version (increased for fixes that do not change the API). */
#define UTF8PROC_VERSION_PATCH 0
/** @} */
#include <stdlib.h>
#if defined(_MSC_VER) && _MSC_VER < 1800
// MSVC prior to 2013 lacked stdbool.h and inttypes.h
typedef signed char utf8proc_int8_t;
typedef unsigned char utf8proc_uint8_t;
typedef short utf8proc_int16_t;
typedef unsigned short utf8proc_uint16_t;
typedef int utf8proc_int32_t;
typedef unsigned int utf8proc_uint32_t;
# ifdef _WIN64
typedef __int64 utf8proc_ssize_t;
typedef unsigned __int64 utf8proc_size_t;
# else
typedef int utf8proc_ssize_t;
typedef unsigned int utf8proc_size_t;
# endif
# ifndef __cplusplus
// emulate C99 bool
typedef unsigned char utf8proc_bool;
# ifndef __bool_true_false_are_defined
# define false 0
# define true 1
# define __bool_true_false_are_defined 1
# endif
# else
typedef bool utf8proc_bool;
# endif
#else
# include <stddef.h>
# include <stdbool.h>
# include <inttypes.h>
typedef int8_t utf8proc_int8_t;
typedef uint8_t utf8proc_uint8_t;
typedef int16_t utf8proc_int16_t;
typedef uint16_t utf8proc_uint16_t;
typedef int32_t utf8proc_int32_t;
typedef uint32_t utf8proc_uint32_t;
typedef size_t utf8proc_size_t;
typedef ptrdiff_t utf8proc_ssize_t;
typedef bool utf8proc_bool;
#endif
#include <limits.h>
#ifdef UTF8PROC_STATIC
# define UTF8PROC_DLLEXPORT
#else
# ifdef _WIN32
# ifdef UTF8PROC_EXPORTS
# define UTF8PROC_DLLEXPORT __declspec(dllexport)
# else
# define UTF8PROC_DLLEXPORT __declspec(dllimport)
# endif
# elif __GNUC__ >= 4
# define UTF8PROC_DLLEXPORT __attribute__ ((visibility("default")))
# else
# define UTF8PROC_DLLEXPORT
# endif
#endif
#ifdef __cplusplus
extern "C" {
#endif
/**
* Option flags used by several functions in the library.
*/
typedef enum {
/** The given UTF-8 input is NULL terminated. */
UTF8PROC_NULLTERM = (1<<0),
/** Unicode Versioning Stability has to be respected. */
UTF8PROC_STABLE = (1<<1),
/** Compatibility decomposition (i.e. formatting information is lost). */
UTF8PROC_COMPAT = (1<<2),
/** Return a result with decomposed characters. */
UTF8PROC_COMPOSE = (1<<3),
/** Return a result with decomposed characters. */
UTF8PROC_DECOMPOSE = (1<<4),
/** Strip "default ignorable characters" such as SOFT-HYPHEN or ZERO-WIDTH-SPACE. */
UTF8PROC_IGNORE = (1<<5),
/** Return an error, if the input contains unassigned codepoints. */
UTF8PROC_REJECTNA = (1<<6),
/**
* Indicating that NLF-sequences (LF, CRLF, CR, NEL) are representing a
* line break, and should be converted to the codepoint for line
* separation (LS).
*/
UTF8PROC_NLF2LS = (1<<7),
/**
* Indicating that NLF-sequences are representing a paragraph break, and
* should be converted to the codepoint for paragraph separation
* (PS).
*/
UTF8PROC_NLF2PS = (1<<8),
/** Indicating that the meaning of NLF-sequences is unknown. */
UTF8PROC_NLF2LF = (UTF8PROC_NLF2LS | UTF8PROC_NLF2PS),
/** Strips and/or convers control characters.
*
* NLF-sequences are transformed into space, except if one of the
* NLF2LS/PS/LF options is given. HorizontalTab (HT) and FormFeed (FF)
* are treated as a NLF-sequence in this case. All other control
* characters are simply removed.
*/
UTF8PROC_STRIPCC = (1<<9),
/**
* Performs unicode case folding, to be able to do a case-insensitive
* string comparison.
*/
UTF8PROC_CASEFOLD = (1<<10),
/**
* Inserts 0xFF bytes at the beginning of each sequence which is
* representing a single grapheme cluster (see UAX#29).
*/
UTF8PROC_CHARBOUND = (1<<11),
/** Lumps certain characters together.
*
* E.g. HYPHEN U+2010 and MINUS U+2212 to ASCII "-". See lump.md for details.
*
* If NLF2LF is set, this includes a transformation of paragraph and
* line separators to ASCII line-feed (LF).
*/
UTF8PROC_LUMP = (1<<12),
/** Strips all character markings.
*
* This includes non-spacing, spacing and enclosing (i.e. accents).
* @note This option works only with @ref UTF8PROC_COMPOSE or
* @ref UTF8PROC_DECOMPOSE
*/
UTF8PROC_STRIPMARK = (1<<13),
/**
* Strip unassigned codepoints.
*/
UTF8PROC_STRIPNA = (1<<14),
} utf8proc_option_t;
/** @name Error codes
* Error codes being returned by almost all functions.
*/
/** @{ */
/** Memory could not be allocated. */
#define UTF8PROC_ERROR_NOMEM -1
/** The given string is too long to be processed. */
#define UTF8PROC_ERROR_OVERFLOW -2
/** The given string is not a legal UTF-8 string. */
#define UTF8PROC_ERROR_INVALIDUTF8 -3
/** The @ref UTF8PROC_REJECTNA flag was set and an unassigned codepoint was found. */
#define UTF8PROC_ERROR_NOTASSIGNED -4
/** Invalid options have been used. */
#define UTF8PROC_ERROR_INVALIDOPTS -5
/** @} */
/* @name Types */
/** Holds the value of a property. */
typedef utf8proc_int16_t utf8proc_propval_t;
/** Struct containing information about a codepoint. */
typedef struct utf8proc_property_struct {
/**
* Unicode category.
* @see utf8proc_category_t.
*/
utf8proc_propval_t category;
utf8proc_propval_t combining_class;
/**
* Bidirectional class.
* @see utf8proc_bidi_class_t.
*/
utf8proc_propval_t bidi_class;
/**
* @anchor Decomposition type.
* @see utf8proc_decomp_type_t.
*/
utf8proc_propval_t decomp_type;
utf8proc_uint16_t decomp_seqindex;
utf8proc_uint16_t casefold_seqindex;
utf8proc_uint16_t uppercase_seqindex;
utf8proc_uint16_t lowercase_seqindex;
utf8proc_uint16_t titlecase_seqindex;
utf8proc_uint16_t comb_index;
unsigned bidi_mirrored:1;
unsigned comp_exclusion:1;
/**
* Can this codepoint be ignored?
*
* Used by utf8proc_decompose_char() when @ref UTF8PROC_IGNORE is
* passed as an option.
*/
unsigned ignorable:1;
unsigned control_boundary:1;
/** The width of the codepoint. */
unsigned charwidth:2;
unsigned pad:2;
/**
* Boundclass.
* @see utf8proc_boundclass_t.
*/
unsigned boundclass:6;
unsigned indic_conjunct_break:2;
} utf8proc_property_t;
/** Unicode categories. */
typedef enum {
UTF8PROC_CATEGORY_CN = 0, /**< Other, not assigned */
UTF8PROC_CATEGORY_LU = 1, /**< Letter, uppercase */
UTF8PROC_CATEGORY_LL = 2, /**< Letter, lowercase */
UTF8PROC_CATEGORY_LT = 3, /**< Letter, titlecase */
UTF8PROC_CATEGORY_LM = 4, /**< Letter, modifier */
UTF8PROC_CATEGORY_LO = 5, /**< Letter, other */
UTF8PROC_CATEGORY_MN = 6, /**< Mark, nonspacing */
UTF8PROC_CATEGORY_MC = 7, /**< Mark, spacing combining */
UTF8PROC_CATEGORY_ME = 8, /**< Mark, enclosing */
UTF8PROC_CATEGORY_ND = 9, /**< Number, decimal digit */
UTF8PROC_CATEGORY_NL = 10, /**< Number, letter */
UTF8PROC_CATEGORY_NO = 11, /**< Number, other */
UTF8PROC_CATEGORY_PC = 12, /**< Punctuation, connector */
UTF8PROC_CATEGORY_PD = 13, /**< Punctuation, dash */
UTF8PROC_CATEGORY_PS = 14, /**< Punctuation, open */
UTF8PROC_CATEGORY_PE = 15, /**< Punctuation, close */
UTF8PROC_CATEGORY_PI = 16, /**< Punctuation, initial quote */
UTF8PROC_CATEGORY_PF = 17, /**< Punctuation, final quote */
UTF8PROC_CATEGORY_PO = 18, /**< Punctuation, other */
UTF8PROC_CATEGORY_SM = 19, /**< Symbol, math */
UTF8PROC_CATEGORY_SC = 20, /**< Symbol, currency */
UTF8PROC_CATEGORY_SK = 21, /**< Symbol, modifier */
UTF8PROC_CATEGORY_SO = 22, /**< Symbol, other */
UTF8PROC_CATEGORY_ZS = 23, /**< Separator, space */
UTF8PROC_CATEGORY_ZL = 24, /**< Separator, line */
UTF8PROC_CATEGORY_ZP = 25, /**< Separator, paragraph */
UTF8PROC_CATEGORY_CC = 26, /**< Other, control */
UTF8PROC_CATEGORY_CF = 27, /**< Other, format */
UTF8PROC_CATEGORY_CS = 28, /**< Other, surrogate */
UTF8PROC_CATEGORY_CO = 29, /**< Other, private use */
} utf8proc_category_t;
/** Bidirectional character classes. */
typedef enum {
UTF8PROC_BIDI_CLASS_L = 1, /**< Left-to-Right */
UTF8PROC_BIDI_CLASS_LRE = 2, /**< Left-to-Right Embedding */
UTF8PROC_BIDI_CLASS_LRO = 3, /**< Left-to-Right Override */
UTF8PROC_BIDI_CLASS_R = 4, /**< Right-to-Left */
UTF8PROC_BIDI_CLASS_AL = 5, /**< Right-to-Left Arabic */
UTF8PROC_BIDI_CLASS_RLE = 6, /**< Right-to-Left Embedding */
UTF8PROC_BIDI_CLASS_RLO = 7, /**< Right-to-Left Override */
UTF8PROC_BIDI_CLASS_PDF = 8, /**< Pop Directional Format */
UTF8PROC_BIDI_CLASS_EN = 9, /**< European Number */
UTF8PROC_BIDI_CLASS_ES = 10, /**< European Separator */
UTF8PROC_BIDI_CLASS_ET = 11, /**< European Number Terminator */
UTF8PROC_BIDI_CLASS_AN = 12, /**< Arabic Number */
UTF8PROC_BIDI_CLASS_CS = 13, /**< Common Number Separator */
UTF8PROC_BIDI_CLASS_NSM = 14, /**< Nonspacing Mark */
UTF8PROC_BIDI_CLASS_BN = 15, /**< Boundary Neutral */
UTF8PROC_BIDI_CLASS_B = 16, /**< Paragraph Separator */
UTF8PROC_BIDI_CLASS_S = 17, /**< Segment Separator */
UTF8PROC_BIDI_CLASS_WS = 18, /**< Whitespace */
UTF8PROC_BIDI_CLASS_ON = 19, /**< Other Neutrals */
UTF8PROC_BIDI_CLASS_LRI = 20, /**< Left-to-Right Isolate */
UTF8PROC_BIDI_CLASS_RLI = 21, /**< Right-to-Left Isolate */
UTF8PROC_BIDI_CLASS_FSI = 22, /**< First Strong Isolate */
UTF8PROC_BIDI_CLASS_PDI = 23, /**< Pop Directional Isolate */
} utf8proc_bidi_class_t;
/** Decomposition type. */
typedef enum {
UTF8PROC_DECOMP_TYPE_FONT = 1, /**< Font */
UTF8PROC_DECOMP_TYPE_NOBREAK = 2, /**< Nobreak */
UTF8PROC_DECOMP_TYPE_INITIAL = 3, /**< Initial */
UTF8PROC_DECOMP_TYPE_MEDIAL = 4, /**< Medial */
UTF8PROC_DECOMP_TYPE_FINAL = 5, /**< Final */
UTF8PROC_DECOMP_TYPE_ISOLATED = 6, /**< Isolated */
UTF8PROC_DECOMP_TYPE_CIRCLE = 7, /**< Circle */
UTF8PROC_DECOMP_TYPE_SUPER = 8, /**< Super */
UTF8PROC_DECOMP_TYPE_SUB = 9, /**< Sub */
UTF8PROC_DECOMP_TYPE_VERTICAL = 10, /**< Vertical */
UTF8PROC_DECOMP_TYPE_WIDE = 11, /**< Wide */
UTF8PROC_DECOMP_TYPE_NARROW = 12, /**< Narrow */
UTF8PROC_DECOMP_TYPE_SMALL = 13, /**< Small */
UTF8PROC_DECOMP_TYPE_SQUARE = 14, /**< Square */
UTF8PROC_DECOMP_TYPE_FRACTION = 15, /**< Fraction */
UTF8PROC_DECOMP_TYPE_COMPAT = 16, /**< Compat */
} utf8proc_decomp_type_t;
/** Boundclass property. (TR29) */
typedef enum {
UTF8PROC_BOUNDCLASS_START = 0, /**< Start */
UTF8PROC_BOUNDCLASS_OTHER = 1, /**< Other */
UTF8PROC_BOUNDCLASS_CR = 2, /**< Cr */
UTF8PROC_BOUNDCLASS_LF = 3, /**< Lf */
UTF8PROC_BOUNDCLASS_CONTROL = 4, /**< Control */
UTF8PROC_BOUNDCLASS_EXTEND = 5, /**< Extend */
UTF8PROC_BOUNDCLASS_L = 6, /**< L */
UTF8PROC_BOUNDCLASS_V = 7, /**< V */
UTF8PROC_BOUNDCLASS_T = 8, /**< T */
UTF8PROC_BOUNDCLASS_LV = 9, /**< Lv */
UTF8PROC_BOUNDCLASS_LVT = 10, /**< Lvt */
UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR = 11, /**< Regional indicator */
UTF8PROC_BOUNDCLASS_SPACINGMARK = 12, /**< Spacingmark */
UTF8PROC_BOUNDCLASS_PREPEND = 13, /**< Prepend */
UTF8PROC_BOUNDCLASS_ZWJ = 14, /**< Zero Width Joiner */
/* the following are no longer used in Unicode 11, but we keep
the constants here for backward compatibility */
UTF8PROC_BOUNDCLASS_E_BASE = 15, /**< Emoji Base */
UTF8PROC_BOUNDCLASS_E_MODIFIER = 16, /**< Emoji Modifier */
UTF8PROC_BOUNDCLASS_GLUE_AFTER_ZWJ = 17, /**< Glue_After_ZWJ */
UTF8PROC_BOUNDCLASS_E_BASE_GAZ = 18, /**< E_BASE + GLUE_AFTER_ZJW */
/* the Extended_Pictographic property is used in the Unicode 11
grapheme-boundary rules, so we store it in the boundclass field */
UTF8PROC_BOUNDCLASS_EXTENDED_PICTOGRAPHIC = 19,
UTF8PROC_BOUNDCLASS_E_ZWG = 20, /* UTF8PROC_BOUNDCLASS_EXTENDED_PICTOGRAPHIC + ZWJ */
} utf8proc_boundclass_t;
/** Indic_Conjunct_Break property. (TR44) */
typedef enum {
UTF8PROC_INDIC_CONJUNCT_BREAK_NONE = 0,
UTF8PROC_INDIC_CONJUNCT_BREAK_LINKER = 1,
UTF8PROC_INDIC_CONJUNCT_BREAK_CONSONANT = 2,
UTF8PROC_INDIC_CONJUNCT_BREAK_EXTEND = 3,
} utf8proc_indic_conjunct_break_t;
/**
* Function pointer type passed to utf8proc_map_custom() and
* utf8proc_decompose_custom(), which is used to specify a user-defined
* mapping of codepoints to be applied in conjunction with other mappings.
*/
typedef utf8proc_int32_t (*utf8proc_custom_func)(utf8proc_int32_t codepoint, void *data);
/**
* Array containing the byte lengths of a UTF-8 encoded codepoint based
* on the first byte.
*/
UTF8PROC_DLLEXPORT extern const utf8proc_int8_t utf8proc_utf8class[256];
/**
* Returns the utf8proc API version as a string MAJOR.MINOR.PATCH
* (http://semver.org format), possibly with a "-dev" suffix for
* development versions.
*/
UTF8PROC_DLLEXPORT const char *utf8proc_version(void);
/**
* Returns the utf8proc supported Unicode version as a string MAJOR.MINOR.PATCH.
*/
UTF8PROC_DLLEXPORT const char *utf8proc_unicode_version(void);
/**
* Returns an informative error string for the given utf8proc error code
* (e.g. the error codes returned by utf8proc_map()).
*/
UTF8PROC_DLLEXPORT const char *utf8proc_errmsg(utf8proc_ssize_t errcode);
/**
* Reads a single codepoint from the UTF-8 sequence being pointed to by `str`.
* The maximum number of bytes read is `strlen`, unless `strlen` is
* negative (in which case up to 4 bytes are read).
*
* If a valid codepoint could be read, it is stored in the variable
* pointed to by `codepoint_ref`, otherwise that variable will be set to -1.
* In case of success, the number of bytes read is returned; otherwise, a
* negative error code is returned.
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_iterate(const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *codepoint_ref);
/**
* Check if a codepoint is valid (regardless of whether it has been
* assigned a value by the current Unicode standard).
*
* @return 1 if the given `codepoint` is valid and otherwise return 0.
*/
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_codepoint_valid(utf8proc_int32_t codepoint);
/**
* Encodes the codepoint as an UTF-8 string in the byte array pointed
* to by `dst`. This array must be at least 4 bytes long.
*
* In case of success the number of bytes written is returned, and
* otherwise 0 is returned.
*
* This function does not check whether `codepoint` is valid Unicode.
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t codepoint, utf8proc_uint8_t *dst);
/**
* Look up the properties for a given codepoint.
*
* @param codepoint The Unicode codepoint.
*
* @returns
* A pointer to a (constant) struct containing information about
* the codepoint.
* @par
* If the codepoint is unassigned or invalid, a pointer to a special struct is
* returned in which `category` is 0 (@ref UTF8PROC_CATEGORY_CN).
*/
UTF8PROC_DLLEXPORT const utf8proc_property_t *utf8proc_get_property(utf8proc_int32_t codepoint);
/** Decompose a codepoint into an array of codepoints.
*
* @param codepoint the codepoint.
* @param dst the destination buffer.
* @param bufsize the size of the destination buffer.
* @param options one or more of the following flags:
* - @ref UTF8PROC_REJECTNA - return an error `codepoint` is unassigned
* - @ref UTF8PROC_IGNORE - strip "default ignorable" codepoints
* - @ref UTF8PROC_CASEFOLD - apply Unicode casefolding
* - @ref UTF8PROC_COMPAT - replace certain codepoints with their
* compatibility decomposition
* - @ref UTF8PROC_CHARBOUND - insert 0xFF bytes before each grapheme cluster
* - @ref UTF8PROC_LUMP - lump certain different codepoints together
* - @ref UTF8PROC_STRIPMARK - remove all character marks
* - @ref UTF8PROC_STRIPNA - remove unassigned codepoints
* @param last_boundclass
* Pointer to an integer variable containing
* the previous codepoint's (boundclass + indic_conjunct_break << 1) if the @ref UTF8PROC_CHARBOUND
* option is used. If the string is being processed in order, this can be initialized to 0 for
* the beginning of the string, and is thereafter updated automatically. Otherwise, this parameter is ignored.
*
* @return
* In case of success, the number of codepoints written is returned; in case
* of an error, a negative error code is returned (utf8proc_errmsg()).
* @par
* If the number of written codepoints would be bigger than `bufsize`, the
* required buffer size is returned, while the buffer will be overwritten with
* undefined data.
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char(
utf8proc_int32_t codepoint, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize,
utf8proc_option_t options, int *last_boundclass
);
/**
* The same as utf8proc_decompose_char(), but acts on a whole UTF-8
* string and orders the decomposed sequences correctly.
*
* If the @ref UTF8PROC_NULLTERM flag in `options` is set, processing
* will be stopped, when a NULL byte is encountered, otherwise `strlen`
* bytes are processed. The result (in the form of 32-bit unicode
* codepoints) is written into the buffer being pointed to by
* `buffer` (which must contain at least `bufsize` entries). In case of
* success, the number of codepoints written is returned; in case of an
* error, a negative error code is returned (utf8proc_errmsg()).
* See utf8proc_decompose_custom() to supply additional transformations.
*
* If the number of written codepoints would be bigger than `bufsize`, the
* required buffer size is returned, while the buffer will be overwritten with
* undefined data.
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options
);
/**
* The same as utf8proc_decompose(), but also takes a `custom_func` mapping function
* that is called on each codepoint in `str` before any other transformations
* (along with a `custom_data` pointer that is passed through to `custom_func`).
* The `custom_func` argument is ignored if it is `NULL`. See also utf8proc_map_custom().
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_custom(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options,
utf8proc_custom_func custom_func, void *custom_data
);
/**
* Normalizes the sequence of `length` codepoints pointed to by `buffer`
* in-place (i.e., the result is also stored in `buffer`).
*
* @param buffer the (native-endian UTF-32) unicode codepoints to re-encode.
* @param length the length (in codepoints) of the buffer.
* @param options a bitwise or (`|`) of one or more of the following flags:
* - @ref UTF8PROC_NLF2LS - convert LF, CRLF, CR and NEL into LS
* - @ref UTF8PROC_NLF2PS - convert LF, CRLF, CR and NEL into PS
* - @ref UTF8PROC_NLF2LF - convert LF, CRLF, CR and NEL into LF
* - @ref UTF8PROC_STRIPCC - strip or convert all non-affected control characters
* - @ref UTF8PROC_COMPOSE - try to combine decomposed codepoints into composite
* codepoints
* - @ref UTF8PROC_STABLE - prohibit combining characters that would violate
* the unicode versioning stability
*
* @return
* In case of success, the length (in codepoints) of the normalized UTF-32 string is
* returned; otherwise, a negative error code is returned (utf8proc_errmsg()).
*
* @warning The entries of the array pointed to by `str` have to be in the
* range `0x0000` to `0x10FFFF`. Otherwise, the program might crash!
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_normalize_utf32(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options);
/**
* Reencodes the sequence of `length` codepoints pointed to by `buffer`
* UTF-8 data in-place (i.e., the result is also stored in `buffer`).
* Can optionally normalize the UTF-32 sequence prior to UTF-8 conversion.
*
* @param buffer the (native-endian UTF-32) unicode codepoints to re-encode.
* @param length the length (in codepoints) of the buffer.
* @param options a bitwise or (`|`) of one or more of the following flags:
* - @ref UTF8PROC_NLF2LS - convert LF, CRLF, CR and NEL into LS
* - @ref UTF8PROC_NLF2PS - convert LF, CRLF, CR and NEL into PS
* - @ref UTF8PROC_NLF2LF - convert LF, CRLF, CR and NEL into LF
* - @ref UTF8PROC_STRIPCC - strip or convert all non-affected control characters
* - @ref UTF8PROC_COMPOSE - try to combine decomposed codepoints into composite
* codepoints
* - @ref UTF8PROC_STABLE - prohibit combining characters that would violate
* the unicode versioning stability
* - @ref UTF8PROC_CHARBOUND - insert 0xFF bytes before each grapheme cluster
*
* @return
* In case of success, the length (in bytes) of the resulting nul-terminated
* UTF-8 string is returned; otherwise, a negative error code is returned
* (utf8proc_errmsg()).
*
* @warning The amount of free space pointed to by `buffer` must
* exceed the amount of the input data by one byte, and the
* entries of the array pointed to by `str` have to be in the
* range `0x0000` to `0x10FFFF`. Otherwise, the program might crash!
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options);
/**
* Given a pair of consecutive codepoints, return whether a grapheme break is
* permitted between them (as defined by the extended grapheme clusters in UAX#29).
*
* @param codepoint1 The first codepoint.
* @param codepoint2 The second codepoint, occurring consecutively after `codepoint1`.
* @param state Beginning with Version 29 (Unicode 9.0.0), this algorithm requires
* state to break graphemes. This state can be passed in as a pointer
* in the `state` argument and should initially be set to 0. If the
* state is not passed in (i.e. a null pointer is passed), UAX#29 rules
* GB10/12/13 which require this state will not be applied, essentially
* matching the rules in Unicode 8.0.0.
*
* @warning If the state parameter is used, `utf8proc_grapheme_break_stateful` must
* be called IN ORDER on ALL potential breaks in a string. However, it
* is safe to reset the state to zero after a grapheme break.
*/
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break_stateful(
utf8proc_int32_t codepoint1, utf8proc_int32_t codepoint2, utf8proc_int32_t *state);
/**
* Same as utf8proc_grapheme_break_stateful(), except without support for the
* Unicode 9 additions to the algorithm. Supported for legacy reasons.
*/
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break(
utf8proc_int32_t codepoint1, utf8proc_int32_t codepoint2);
/**
* Given a codepoint `c`, return the codepoint of the corresponding
* lower-case character, if any; otherwise (if there is no lower-case
* variant, or if `c` is not a valid codepoint) return `c`.
*/
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_tolower(utf8proc_int32_t c);
/**
* Given a codepoint `c`, return the codepoint of the corresponding
* upper-case character, if any; otherwise (if there is no upper-case
* variant, or if `c` is not a valid codepoint) return `c`.
*/
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_toupper(utf8proc_int32_t c);
/**
* Given a codepoint `c`, return the codepoint of the corresponding
* title-case character, if any; otherwise (if there is no title-case
* variant, or if `c` is not a valid codepoint) return `c`.
*/
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_totitle(utf8proc_int32_t c);
/**
* Given a codepoint `c`, return `1` if the codepoint corresponds to a lower-case character
* and `0` otherwise.
*/
UTF8PROC_DLLEXPORT int utf8proc_islower(utf8proc_int32_t c);
/**
* Given a codepoint `c`, return `1` if the codepoint corresponds to an upper-case character
* and `0` otherwise.
*/
UTF8PROC_DLLEXPORT int utf8proc_isupper(utf8proc_int32_t c);
/**
* Given a codepoint, return a character width analogous to `wcwidth(codepoint)`,
* except that a width of 0 is returned for non-printable codepoints
* instead of -1 as in `wcwidth`.
*
* @note
* If you want to check for particular types of non-printable characters,
* (analogous to `isprint` or `iscntrl`), use utf8proc_category(). */
UTF8PROC_DLLEXPORT int utf8proc_charwidth(utf8proc_int32_t codepoint);
/**
* Return the Unicode category for the codepoint (one of the
* @ref utf8proc_category_t constants.)
*/
UTF8PROC_DLLEXPORT utf8proc_category_t utf8proc_category(utf8proc_int32_t codepoint);
/**
* Return the two-letter (nul-terminated) Unicode category string for
* the codepoint (e.g. `"Lu"` or `"Co"`).
*/
UTF8PROC_DLLEXPORT const char *utf8proc_category_string(utf8proc_int32_t codepoint);
/**
* Maps the given UTF-8 string pointed to by `str` to a new UTF-8
* string, allocated dynamically by `malloc` and returned via `dstptr`.
*
* If the @ref UTF8PROC_NULLTERM flag in the `options` field is set,
* the length is determined by a NULL terminator, otherwise the
* parameter `strlen` is evaluated to determine the string length, but
* in any case the result will be NULL terminated (though it might
* contain NULL characters with the string if `str` contained NULL
* characters). Other flags in the `options` field are passed to the
* functions defined above, and regarded as described. See also
* utf8proc_map_custom() to supply a custom codepoint transformation.
*
* In case of success the length of the new string is returned,
* otherwise a negative error code is returned.
*
* @note The memory of the new UTF-8 string will have been allocated
* with `malloc`, and should therefore be deallocated with `free`.
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options
);
/**
* Like utf8proc_map(), but also takes a `custom_func` mapping function
* that is called on each codepoint in `str` before any other transformations
* (along with a `custom_data` pointer that is passed through to `custom_func`).
* The `custom_func` argument is ignored if it is `NULL`.
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map_custom(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options,
utf8proc_custom_func custom_func, void *custom_data
);
/** @name Unicode normalization
*
* Returns a pointer to newly allocated memory of a NFD, NFC, NFKD, NFKC or
* NFKC_Casefold normalized version of the null-terminated string `str`. These
* are shortcuts to calling utf8proc_map() with @ref UTF8PROC_NULLTERM
* combined with @ref UTF8PROC_STABLE and flags indicating the normalization.
*/
/** @{ */
/** NFD normalization (@ref UTF8PROC_DECOMPOSE). */
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFD(const utf8proc_uint8_t *str);
/** NFC normalization (@ref UTF8PROC_COMPOSE). */
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFC(const utf8proc_uint8_t *str);
/** NFKD normalization (@ref UTF8PROC_DECOMPOSE and @ref UTF8PROC_COMPAT). */
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKD(const utf8proc_uint8_t *str);
/** NFKC normalization (@ref UTF8PROC_COMPOSE and @ref UTF8PROC_COMPAT). */
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC(const utf8proc_uint8_t *str);
/**
* NFKC_Casefold normalization (@ref UTF8PROC_COMPOSE and @ref UTF8PROC_COMPAT
* and @ref UTF8PROC_CASEFOLD and @ref UTF8PROC_IGNORE).
**/
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC_Casefold(const utf8proc_uint8_t *str);
/** @} */
#ifdef __cplusplus
}
#endif
#endif

16960
src/common/thirdparty/utf8proc/utf8proc_data.c vendored Normal file
View file

File diff suppressed because it is too large Load diff

3
src/common/utility/geometry.h
View file

@ -150,7 +150,7 @@ inline double InterceptLineSegments(double v2x, double v2y, double v2dx, double
den = 1 / den;
double factor1 = ((v2x - v1x) * v2dy + (v1y - v2y) * v2dx) * -den;
if (factor1 < 0 || factor1 > 1) return -FLT_MAX; // no intersection
if (factor1 < 0 || factor1 >= 1) return -FLT_MAX; // no intersection
if (pfactor1) *pfactor1 = factor1;
return ((v1x - v2x) * v1dy + (v2y - v1y) * v1dx) * den; // this one's for the line segment where we want to get the intercept factor for so it needs to be last.
@ -232,4 +232,3 @@ inline bool BoxInRange(const DVector2& boxtl, const DVector2& boxbr, const DVect
boxtl.Y < max(start.Y, end.Y) &&
boxbr.Y > min(start.Y, end.Y);
}

9
src/common/utility/m_alloc.cpp
View file

@ -77,6 +77,7 @@ void *M_Realloc(void *memblock, size_t size)
GC::ReportRealloc(oldsize, _msize(block));
return block;
}
#else
void *M_Malloc(size_t size)
{
@ -113,6 +114,14 @@ void *M_Realloc(void *memblock, size_t size)
return block;
}
#endif
void* M_Calloc(size_t v1, size_t v2)
{
auto p = M_Malloc(v1 * v2);
memset(p, 0, v1 * v2);
return p;
}
#else
#ifdef _MSC_VER
#include <crtdbg.h>

7
src/common/utility/m_alloc.h
View file

@ -72,12 +72,7 @@ inline void* M_Calloc_Dbg(size_t v1, size_t v2, const char* file, int lineno)
#else
void *M_Malloc (size_t size);
void *M_Realloc (void *memblock, size_t size);
inline void* M_Calloc(size_t v1, size_t v2)
{
auto p = M_Malloc(v1 * v2);
memset(p, 0, v1 * v2);
return p;
}
void* M_Calloc(size_t v1, size_t v2);
#endif

38
src/common/utility/tarray.h
View file

@ -657,6 +657,44 @@ public:
std::swap(Most, other.Most);
}
// aliases with STL compliant names to allow using TArrays with templates designed for STL containers
size_t size() const
{
return Count;
}
T* data() const
{
return Data();
}
T& front() const
{
return *Data();
}
T& back() const
{
return Last();
}
void resize(size_t i)
{
Resize(i);
}
void push_back(T& elem)
{
Push(elem);
}
void clear()
{
Clear();
}
private:
T *Array;
unsigned int Count;

124
src/common/utility/utf8.cpp
View file

@ -823,44 +823,52 @@ static const uint16_t loweruppercase[] = {
0x0584,0x0554,
0x0585,0x0555,
0x0586,0x0556,
0x10D0,0x10A0,
0x10D1,0x10A1,
0x10D2,0x10A2,
0x10D3,0x10A3,
0x10D4,0x10A4,
0x10D5,0x10A5,
0x10D6,0x10A6,
0x10D7,0x10A7,
0x10D8,0x10A8,
0x10D9,0x10A9,
0x10DA,0x10AA,
0x10DB,0x10AB,
0x10DC,0x10AC,
0x10DD,0x10AD,
0x10DE,0x10AE,
0x10DF,0x10AF,
0x10E0,0x10B0,
0x10E1,0x10B1,
0x10E2,0x10B2,
0x10E3,0x10B3,
0x10E4,0x10B4,
0x10E5,0x10B5,
0x10E6,0x10B6,
0x10E7,0x10B7,
0x10E8,0x10B8,
0x10E9,0x10B9,
0x10EA,0x10BA,
0x10EB,0x10BB,
0x10EC,0x10BC,
0x10ED,0x10BD,
0x10EE,0x10BE,
0x10EF,0x10BF,
0x10F0,0x10C0,
0x10F1,0x10C1,
0x10F2,0x10C2,
0x10F3,0x10C3,
0x10F4,0x10C4,
0x10F5,0x10C5,
0x10D0,0x1C90,
0x10D1,0x1C91,
0x10D2,0x1C92,
0x10D3,0x1C93,
0x10D4,0x1C94,
0x10D5,0x1C95,
0x10D6,0x1C96,
0x10D7,0x1C97,
0x10D8,0x1C98,
0x10D9,0x1C99,
0x10DA,0x1C9A,
0x10DB,0x1C9B,
0x10DC,0x1C9C,
0x10DD,0x1C9D,
0x10DE,0x1C9E,
0x10DF,0x1C9F,
0x10E0,0x1CA0,
0x10E1,0x1CA1,
0x10E2,0x1CA2,
0x10E3,0x1CA3,
0x10E4,0x1CA4,
0x10E5,0x1CA5,
0x10E6,0x1CA6,
0x10E7,0x1CA7,
0x10E8,0x1CA8,
0x10E9,0x1CA9,
0x10EA,0x1CAA,
0x10EB,0x1CAB,
0x10EC,0x1CAC,
0x10ED,0x1CAD,
0x10EE,0x1CAE,
0x10EF,0x1CAF,
0x10F0,0x1CB0,
0x10F1,0x1CB1,
0x10F2,0x1CB2,
0x10F3,0x1CB3,
0x10F4,0x1CB4,
0x10F5,0x1CB5,
0x10F6,0x1CB6,
0x10F7,0x1CB7,
0x10F8,0x1CB8,
0x10F9,0x1CB9,
0x10FA,0x1CBA,
0x10FD,0x1CBD,
0x10FE,0x1CBE,
0x10FF,0x1CBF,
0x1E01,0x1E00,
0x1E03,0x1E02,
0x1E05,0x1E04,
@ -1085,6 +1093,46 @@ static const uint16_t loweruppercase[] = {
0x24E7, 0x24CD,
0x24E8, 0x24CE,
0x24E9, 0x24CF,
0x2D00, 0x10A0,
0x2D01, 0x10A1,
0x2D02, 0x10A2,
0x2D03, 0x10A3,
0x2D04, 0x10A4,
0x2D05, 0x10A5,
0x2D06, 0x10A6,
0x2D07, 0x10A7,
0x2D08, 0x10A8,
0x2D09, 0x10A9,
0x2D0A, 0x10AA,
0x2D0B, 0x10AB,
0x2D0C, 0x10AC,
0x2D0D, 0x10AD,
0x2D0E, 0x10AE,
0x2D0F, 0x10AF,
0x2D10, 0x10B0,
0x2D11, 0x10B1,
0x2D12, 0x10B2,
0x2D13, 0x10B3,
0x2D14, 0x10B4,
0x2D15, 0x10B5,
0x2D16, 0x10B6,
0x2D17, 0x10B7,
0x2D18, 0x10B8,
0x2D19, 0x10B9,
0x2D1A, 0x10BA,
0x2D1B, 0x10BB,
0x2D1C, 0x10BC,
0x2D1D, 0x10BD,
0x2D1E, 0x10BE,
0x2D1F, 0x10BF,
0x2D20, 0x10C0,
0x2D21, 0x10C1,
0x2D22, 0x10C2,
0x2D23, 0x10C3,
0x2D24, 0x10C4,
0x2D25, 0x10C5,
0x2D27, 0x10C7,
0x2D2D, 0x10CD,
0xFF41, 0xFF21,
0xFF42, 0xFF22,
0xFF43, 0xFF23,

Some files were not shown because too many files have changed in this diff Show more