From 2194c87956d996158a50f84294d9623de97a77cd Mon Sep 17 00:00:00 2001
From: PMheart <17109513+PMheart@users.noreply.github.com>
Date: Sun, 10 Jul 2022 11:10:42 +0200
Subject: [PATCH] Remove profile 2 from IOGVAHEVCEncodeCapabilities

---
 WhateverGreen/kern_igfx.cpp | 74 ++++++++++++++++++++-----------------
 1 file changed, 41 insertions(+), 33 deletions(-)

diff --git a/WhateverGreen/kern_igfx.cpp b/WhateverGreen/kern_igfx.cpp
index a130e40d..71abd584 100644
--- a/WhateverGreen/kern_igfx.cpp
+++ b/WhateverGreen/kern_igfx.cpp
@@ -1058,47 +1058,55 @@ bool IGFX::applySklAsKblPatches(IOService *that) {
 	DBGLOG("igfx", "disabling VP9 hw decode support on Skylake with KBL kexts");
 	that->removeProperty("IOGVAXDecode");
 
-	auto hevcDecodeCap = OSDynamicCast(OSDictionary, that->getProperty("IOGVAHEVCDecodeCapabilities"));
-	if (!hevcDecodeCap)
-		return false;
+	// SKL does not support 10-bit hardware encoding/decoding, and this causes freezing when attempting to do so.
+	// The removal of profile 2 under VTSupportedProfileArray allows fallback to software encoding/decoding.
+	// Thanks dhinakg and aben for finding this.
+	const char *hevcCapProps[] = { "IOGVAHEVCDecodeCapabilities", "IOGVAHEVCEncodeCapabilities" };
+	for (auto prop : hevcCapProps) {
+		auto hevcCap = OSDynamicCast(OSDictionary, that->getProperty(prop));
+		if (!hevcCap)
+			return false;
 
-	auto newHevcDecodeCap = OSDictionary::withDictionary(hevcDecodeCap);
-	if (!newHevcDecodeCap)
-		return false;
+		auto newHevcCap = OSDictionary::withDictionary(hevcCap);
+		if (!newHevcCap)
+			return false;
 
-	auto vtSuppProf = OSDynamicCast(OSArray, newHevcDecodeCap->getObject("VTSupportedProfileArray"));
-	if (!vtSuppProf) {
-		newHevcDecodeCap->release();
-		return false;
-	}
+		auto vtSuppProf = OSDynamicCast(OSArray, newHevcCap->getObject("VTSupportedProfileArray"));
+		if (!vtSuppProf) {
+			newHevcCap->release();
+			return false;
+		}
 
-	auto newVtSuppProf = OSArray::withArray(vtSuppProf);
-	if (!newVtSuppProf) {
-		newHevcDecodeCap->release();
-		return false;
-	}
+		auto newVtSuppProf = OSArray::withArray(vtSuppProf);
+		if (!newVtSuppProf) {
+			newHevcCap->release();
+			return false;
+		}
 
-	unsigned int count = newVtSuppProf->getCount();
-	bool found = false;
-	for (unsigned int i = 0; i < count; i++) {
-		auto num = OSDynamicCast(OSNumber, newVtSuppProf->getObject(i));
-		if (!num) break;
+		unsigned int count = newVtSuppProf->getCount();
+		for (unsigned int i = 0; i < count; i++) {
+			auto num = OSDynamicCast(OSNumber, newVtSuppProf->getObject(i));
+			if (!num) {
+				newVtSuppProf->release();
+				newHevcCap->release();
+				return false;
+			}
 
-		if (num->unsigned8BitValue() == 2) {
-			DBGLOG("igfx", "removing profile 2 from VTSupportedProfileArray/IOGVAHEVCDecodeCapabilities index %u on Skylake with KBL kexts", i);
-			newVtSuppProf->removeObject(i);
-			found = true;
-			break;
+			if (num->unsigned8BitValue() == 2) {
+				DBGLOG("igfx", "removing profile 2 from VTSupportedProfileArray/%s index %u on Skylake with KBL kexts", prop, i);
+				newVtSuppProf->removeObject(i);
+				break;
+			}
 		}
-	}
 
-	newHevcDecodeCap->setObject("VTSupportedProfileArray", newVtSuppProf);
-	newVtSuppProf->release();
+		newHevcCap->setObject("VTSupportedProfileArray", newVtSuppProf);
+		newVtSuppProf->release();
 
-	that->setProperty("IOGVAHEVCDecodeCapabilities", newHevcDecodeCap);
-	newHevcDecodeCap->release();
+		that->setProperty(prop, newHevcCap);
+		newHevcCap->release();
+	}
 
-	return found;
+	return true;
 }
 
 bool IGFX::wrapAcceleratorStart(IOService *that, IOService *provider) {
@@ -1124,7 +1132,7 @@ bool IGFX::wrapAcceleratorStart(IOService *that, IOService *provider) {
 		that->setName("IntelAccelerator");
 	
 	if (callbackIGFX->forceSKLAsKBL && !applySklAsKblPatches(that)) {
-		SYSLOG("igfx", "failed to remove profile 2 from VTSupportedProfileArray/IOGVAHEVCDecodeCapabilities");
+		SYSLOG("igfx", "failed to apply patches for Skylake with KBL kexts");
 	}
 
 	bool ret = FunctionCast(wrapAcceleratorStart, callbackIGFX->orgAcceleratorStart)(that, provider);