diff --git a/src/swrenderer/plane/r_visibleplanelist.cpp b/src/swrenderer/plane/r_visibleplanelist.cpp
index dcb3756f8..8cb29d938 100644
--- a/src/swrenderer/plane/r_visibleplanelist.cpp
+++ b/src/swrenderer/plane/r_visibleplanelist.cpp
@@ -102,6 +102,8 @@ namespace swrenderer
 
 		FTransform nulltransform;
 
+		RenderPortal *renderportal = Thread->Portal.get();
+
 		if (picnum == skyflatnum)	// killough 10/98
 		{ // most skies map together
 			lightlevel = 0;
@@ -115,9 +117,9 @@ namespace swrenderer
 			// same column but separated by a wall. If they both try to reside in the
 			// same visplane, then only the floor sky will be drawn.
 			plane.set(0., 0., height.fC(), 0.);
-			isskybox = portal != nullptr && !(portal->mFlags & PORTSF_INSKYBOX);
+			isskybox = portal != nullptr && !renderportal->InSkyBox(portal);
 		}
-		else if (portal != nullptr && !(portal->mFlags & PORTSF_INSKYBOX))
+		else if (portal != nullptr && !renderportal->InSkyBox(portal))
 		{
 			plane = height;
 			isskybox = true;
@@ -139,8 +141,6 @@ namespace swrenderer
 		// New visplane algorithm uses hash table -- killough
 		hash = isskybox ? ((unsigned)MAXVISPLANES) : CalcHash(picnum.GetIndex(), lightlevel, height);
 		
-		RenderPortal *renderportal = Thread->Portal.get();
-
 		for (check = visplanes[hash]; check; check = check->next)	// killough
 		{
 			if (isskybox)
@@ -265,7 +265,7 @@ namespace swrenderer
 			// make a new visplane
 			unsigned hash;
 
-			if (pl->portal != nullptr && !(pl->portal->mFlags & PORTSF_INSKYBOX) && viewactive)
+			if (pl->portal != nullptr && !Thread->Portal->InSkyBox(pl->portal) && viewactive)
 			{
 				hash = MAXVISPLANES;
 			}
diff --git a/src/swrenderer/scene/r_portal.cpp b/src/swrenderer/scene/r_portal.cpp
index 28084edcd..b34fe374a 100644
--- a/src/swrenderer/scene/r_portal.cpp
+++ b/src/swrenderer/scene/r_portal.cpp
@@ -156,8 +156,8 @@ namespace swrenderer
 				continue;
 			}
 
-			port->mFlags |= PORTSF_INSKYBOX;
-			if (port->mPartner > 0) level.sectorPortals[port->mPartner].mFlags |= PORTSF_INSKYBOX;
+			SetInSkyBox(port);
+			if (port->mPartner > 0) SetInSkyBox(&level.sectorPortals[port->mPartner]);
 			Thread->Viewport->viewpoint.camera = nullptr;
 			Thread->Viewport->viewpoint.sector = port->mDestination;
 			assert(Thread->Viewport->viewpoint.sector != nullptr);
@@ -217,8 +217,8 @@ namespace swrenderer
 			Thread->Clip3D->ResetClip(); // reset clips (floor/ceiling)
 			planes->Render();
 
-			port->mFlags &= ~PORTSF_INSKYBOX;
-			if (port->mPartner > 0) level.sectorPortals[port->mPartner].mFlags &= ~PORTSF_INSKYBOX;
+			ClearInSkyBox(port);
+			if (port->mPartner > 0) SetInSkyBox(&level.sectorPortals[port->mPartner]);
 		}
 
 		// Draw all the masked textures in a second pass, in the reverse order they
@@ -528,6 +528,7 @@ namespace swrenderer
 		CurrentPortal = nullptr;
 		CurrentPortalUniq = 0;
 		WallPortals.Clear();
+		SectorPortalsInSkyBox.clear();
 	}
 
 	void RenderPortal::AddLinePortal(line_t *linedef, int x1, int x2, const short *topclip, const short *bottomclip)
diff --git a/src/swrenderer/scene/r_portal.h b/src/swrenderer/scene/r_portal.h
index 00322e2b6..b4382c68a 100644
--- a/src/swrenderer/scene/r_portal.h
+++ b/src/swrenderer/scene/r_portal.h
@@ -14,6 +14,7 @@
 #pragma once
 
 #include "swrenderer/segments/r_portalsegment.h"
+#include <set>
 
 namespace swrenderer
 {
@@ -54,6 +55,10 @@ namespace swrenderer
 		
 		int numskyboxes = 0; // For ADD_STAT(skyboxes)
 
+		void SetInSkyBox(FSectorPortal *portal) { SectorPortalsInSkyBox.insert(portal); }
+		void ClearInSkyBox(FSectorPortal *portal) { SectorPortalsInSkyBox.erase(portal); }
+		bool InSkyBox(FSectorPortal *portal) const { return SectorPortalsInSkyBox.find(portal) != SectorPortalsInSkyBox.end(); }
+
 	private:
 		void RenderLinePortal(PortalDrawseg* pds, int depth);
 		void RenderLinePortalHighlight(PortalDrawseg* pds);
@@ -61,5 +66,7 @@ namespace swrenderer
 		TArray<DVector3> viewposStack;
 		TArray<VisiblePlane *> visplaneStack;
 		TArray<PortalDrawseg *> WallPortals;
+
+		std::set<FSectorPortal *> SectorPortalsInSkyBox; // Instead of portal->mFlags & PORTSF_INSKYBOX
 	};
 }