Merge pull request #89894 from BastiaanOlij/improve_foveated_rendering

Improvements to VRS/Foveated rendering

doc/classes/RenderingServer.xml

@@ -4036,6 +4036,15 @@
 				The texture to use when the VRS mode is set to [constant RenderingServer.VIEWPORT_VRS_TEXTURE]. Equivalent to [member ProjectSettings.rendering/vrs/texture].
 			</description>
 		</method>
+		<method name="viewport_set_vrs_update_mode">
+			<return type="void" />
+			<param index="0" name="viewport" type="RID" />
+			<param index="1" name="mode" type="int" enum="RenderingServer.ViewportVRSUpdateMode" />
+			<description>
+				Sets the update mode for Variable Rate Shading (VRS) for the viewport. VRS requires the input texture to be converted to the format usable by the VRS method supported by the hardware. The update mode defines how often this happens. If the GPU does not support VRS, or VRS is not enabled, this property is ignored.
+				If set to [constant RenderingServer.VIEWPORT_VRS_UPDATE_ONCE], the input texture is copied once and the mode is changed to [constant RenderingServer.VIEWPORT_VRS_UPDATE_DISABLED].
+			</description>
+		</method>
 		<method name="visibility_notifier_create">
 			<return type="RID" />
 			<description>
@@ -4976,11 +4985,23 @@
 			Variable rate shading uses a texture. Note, for stereoscopic use a texture atlas with a texture for each view.
 		</constant>
 		<constant name="VIEWPORT_VRS_XR" value="2" enum="ViewportVRSMode">
-			Variable rate shading texture is supplied by the primary [XRInterface].
+			Variable rate shading texture is supplied by the primary [XRInterface]. Note that this may override the update mode.
 		</constant>
 		<constant name="VIEWPORT_VRS_MAX" value="3" enum="ViewportVRSMode">
 			Represents the size of the [enum ViewportVRSMode] enum.
 		</constant>
+		<constant name="VIEWPORT_VRS_UPDATE_DISABLED" value="0" enum="ViewportVRSUpdateMode">
+			The input texture for variable rate shading will not be processed.
+		</constant>
+		<constant name="VIEWPORT_VRS_UPDATE_ONCE" value="1" enum="ViewportVRSUpdateMode">
+			The input texture for variable rate shading will be processed once.
+		</constant>
+		<constant name="VIEWPORT_VRS_UPDATE_ALWAYS" value="2" enum="ViewportVRSUpdateMode">
+			The input texture for variable rate shading will be processed each frame.
+		</constant>
+		<constant name="VIEWPORT_VRS_UPDATE_MAX" value="3" enum="ViewportVRSUpdateMode">
+			Represents the size of the [enum ViewportVRSUpdateMode] enum.
+		</constant>
 		<constant name="SKY_MODE_AUTOMATIC" value="0" enum="SkyMode">
 			Automatically selects the appropriate process mode based on your sky shader. If your shader uses [code]TIME[/code] or [code]POSITION[/code], this will use [constant SKY_MODE_REALTIME]. If your shader uses any of the [code]LIGHT_*[/code] variables or any custom uniforms, this uses [constant SKY_MODE_INCREMENTAL]. Otherwise, this defaults to [constant SKY_MODE_QUALITY].
 		</constant>

doc/classes/Viewport.xml

@@ -407,6 +407,9 @@
 			- 8×8 = rgb(255, 255, 0) - #ffff00 - Not supported on most hardware
 			[/codeblock]
 		</member>
+		<member name="vrs_update_mode" type="int" setter="set_vrs_update_mode" getter="get_vrs_update_mode" enum="Viewport.VRSUpdateMode" default="1">
+			Sets the update mode for Variable Rate Shading (VRS) for the viewport. VRS requires the input texture to be converted to the format usable by the VRS method supported by the hardware. The update mode defines how often this happens. If the GPU does not support VRS, or VRS is not enabled, this property is ignored.
+		</member>
 		<member name="world_2d" type="World2D" setter="set_world_2d" getter="get_world_2d">
 			The custom [World2D] which can be used as 2D environment source.
 		</member>
@@ -664,5 +667,17 @@
 		<constant name="VRS_MAX" value="3" enum="VRSMode">
 			Represents the size of the [enum VRSMode] enum.
 		</constant>
+		<constant name="VRS_UPDATE_DISABLED" value="0" enum="VRSUpdateMode">
+			The input texture for variable rate shading will not be processed.
+		</constant>
+		<constant name="VRS_UPDATE_ONCE" value="1" enum="VRSUpdateMode">
+			The input texture for variable rate shading will be processed once.
+		</constant>
+		<constant name="VRS_UPDATE_ALWAYS" value="2" enum="VRSUpdateMode">
+			The input texture for variable rate shading will be processed each frame.
+		</constant>
+		<constant name="VRS_UPDATE_MAX" value="3" enum="VRSUpdateMode">
+			Represents the size of the [enum VRSUpdateMode] enum.
+		</constant>
 	</constants>
 </class>

doc/classes/XRVRS.xml

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<class name="XRVRS" inherits="Object" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../class.xsd">
+	<brief_description>
+		Helper class for XR interfaces that generates VRS images.
+	</brief_description>
+	<description>
+		This class is used by various XR interfaces to generate VRS textures that can be used to speed up rendering.
+	</description>
+	<tutorials>
+	</tutorials>
+	<methods>
+		<method name="make_vrs_texture">
+			<return type="RID" />
+			<param index="0" name="target_size" type="Vector2" />
+			<param index="1" name="eye_foci" type="PackedVector2Array" />
+			<description>
+				Generates the VRS texture based on a render [param target_size] adjusted by our VRS tile size. For each eyes focal point passed in [param eye_foci] a layer is created. Focal point should be in NDC.
+				The result will be cached, requesting a VRS texture with unchanged parameters and settings will return the cached RID.
+			</description>
+		</method>
+	</methods>
+	<members>
+		<member name="vrs_min_radius" type="float" setter="set_vrs_min_radius" getter="get_vrs_min_radius" default="20.0">
+			The minimum radius around the focal point where full quality is guaranteed if VRS is used as a percentage of screen size.
+		</member>
+		<member name="vrs_strength" type="float" setter="set_vrs_strength" getter="get_vrs_strength" default="1.0">
+			The strength used to calculate the VRS density map. The greater this value, the more noticeable VRS is.
+		</member>
+	</members>
+</class>

drivers/d3d12/rendering_device_driver_d3d12.cpp

@@ -5912,6 +5912,9 @@ uint64_t RenderingDeviceDriverD3D12::limit_get(Limit p_limit) {
 		case LIMIT_VRS_TEXEL_WIDTH:
 		case LIMIT_VRS_TEXEL_HEIGHT:
 			return vrs_capabilities.ss_image_tile_size;
+		case LIMIT_VRS_MAX_FRAGMENT_WIDTH:
+		case LIMIT_VRS_MAX_FRAGMENT_HEIGHT:
+			return vrs_capabilities.ss_max_fragment_size;
 		default: {
 #ifdef DEV_ENABLED
 			WARN_PRINT("Returning maximum value for unknown limit " + itos(p_limit) + ".");
@@ -6218,6 +6221,7 @@ Error RenderingDeviceDriverD3D12::_check_capabilities() {
 				vrs_capabilities.primitive_in_multiviewport = options6.PerPrimitiveShadingRateSupportedWithViewportIndexing;
 				vrs_capabilities.ss_image_supported = true;
 				vrs_capabilities.ss_image_tile_size = options6.ShadingRateImageTileSize;
+				vrs_capabilities.ss_max_fragment_size = 8; // TODO figure out if this is supplied and/or needed
 				vrs_capabilities.additional_rates_supported = options6.AdditionalShadingRatesSupported;
 			}
 		}

drivers/d3d12/rendering_device_driver_d3d12.h

@@ -105,6 +105,7 @@ class RenderingDeviceDriverD3D12 : public RenderingDeviceDriver {
 		bool primitive_in_multiviewport = false;
 		bool ss_image_supported = false; // We can provide a density map attachment on our framebuffer.
 		uint32_t ss_image_tile_size = 0;
+		uint32_t ss_max_fragment_size = 0;
 		bool additional_rates_supported = false;
 	};
 

drivers/gles3/storage/texture_storage.h

@@ -672,6 +672,8 @@ class TextureStorage : public RendererTextureStorage {
 
 	virtual void render_target_set_vrs_mode(RID p_render_target, RS::ViewportVRSMode p_mode) override {}
 	virtual RS::ViewportVRSMode render_target_get_vrs_mode(RID p_render_target) const override { return RS::VIEWPORT_VRS_DISABLED; }
+	virtual void render_target_set_vrs_update_mode(RID p_render_target, RS::ViewportVRSUpdateMode p_mode) override {}
+	virtual RS::ViewportVRSUpdateMode render_target_get_vrs_update_mode(RID p_render_target) const override { return RS::VIEWPORT_VRS_UPDATE_DISABLED; }
 	virtual void render_target_set_vrs_texture(RID p_render_target, RID p_texture) override {}
 	virtual RID render_target_get_vrs_texture(RID p_render_target) const override { return RID(); }
 

drivers/vulkan/rendering_device_driver_vulkan.cpp

@@ -758,11 +758,13 @@ Error RenderingDeviceDriverVulkan::_check_device_capabilities() {
 				vrs_capabilities.min_texel_size.y = vrs_properties.minFragmentShadingRateAttachmentTexelSize.height;
 				vrs_capabilities.max_texel_size.x = vrs_properties.maxFragmentShadingRateAttachmentTexelSize.width;
 				vrs_capabilities.max_texel_size.y = vrs_properties.maxFragmentShadingRateAttachmentTexelSize.height;
+				vrs_capabilities.max_fragment_size.x = vrs_properties.maxFragmentSize.width; // either 4 or 8
+				vrs_capabilities.max_fragment_size.y = vrs_properties.maxFragmentSize.height; // generally the same as width
 
 				// We'll attempt to default to a texel size of 16x16.
 				vrs_capabilities.texel_size = Vector2i(16, 16).clamp(vrs_capabilities.min_texel_size, vrs_capabilities.max_texel_size);
 
-				print_verbose(String("  Attachment fragment shading rate") + String(", min texel size: (") + itos(vrs_capabilities.min_texel_size.x) + String(", ") + itos(vrs_capabilities.min_texel_size.y) + String(")") + String(", max texel size: (") + itos(vrs_capabilities.max_texel_size.x) + String(", ") + itos(vrs_capabilities.max_texel_size.y) + String(")"));
+				print_verbose(String("  Attachment fragment shading rate") + String(", min texel size: (") + itos(vrs_capabilities.min_texel_size.x) + String(", ") + itos(vrs_capabilities.min_texel_size.y) + String(")") + String(", max texel size: (") + itos(vrs_capabilities.max_texel_size.x) + String(", ") + itos(vrs_capabilities.max_texel_size.y) + String(")") + String(", max fragment size: (") + itos(vrs_capabilities.max_fragment_size.x) + String(", ") + itos(vrs_capabilities.max_fragment_size.y) + String(")"));
 			}
 
 		} else {
@@ -4887,6 +4889,10 @@ uint64_t RenderingDeviceDriverVulkan::limit_get(Limit p_limit) {
 			return vrs_capabilities.texel_size.x;
 		case LIMIT_VRS_TEXEL_HEIGHT:
 			return vrs_capabilities.texel_size.y;
+		case LIMIT_VRS_MAX_FRAGMENT_WIDTH:
+			return vrs_capabilities.max_fragment_size.x;
+		case LIMIT_VRS_MAX_FRAGMENT_HEIGHT:
+			return vrs_capabilities.max_fragment_size.y;
 		default:
 			ERR_FAIL_V(0);
 	}

drivers/vulkan/rendering_device_driver_vulkan.h

@@ -87,6 +87,7 @@ class RenderingDeviceDriverVulkan : public RenderingDeviceDriver {
 
 		Size2i min_texel_size;
 		Size2i max_texel_size;
+		Size2i max_fragment_size;
 
 		Size2i texel_size; // The texel size we'll use
 	};

modules/mobile_vr/doc_classes/MobileVRInterface.xml

@@ -40,6 +40,14 @@
 		<member name="oversample" type="float" setter="set_oversample" getter="get_oversample" default="1.5">
 			The oversample setting. Because of the lens distortion we have to render our buffers at a higher resolution then the screen can natively handle. A value between 1.5 and 2.0 often provides good results but at the cost of performance.
 		</member>
+		<member name="vrs_min_radius" type="float" setter="set_vrs_min_radius" getter="get_vrs_min_radius" default="20.0">
+			The minimum radius around the focal point where full quality is guaranteed if VRS is used as a percentage of screen size.
+			[b]Note:[/b] Mobile and Forward+ renderers only. Requires [member Viewport.vrs_mode] to be set to [constant Viewport.VRS_XR].
+		</member>
+		<member name="vrs_strength" type="float" setter="set_vrs_strength" getter="get_vrs_strength" default="1.0">
+			The strength used to calculate the VRS density map. The greater this value, the more noticeable VRS is. This improves performance at the cost of quality.
+			[b]Note:[/b] Mobile and Forward+ renderers only. Requires [member Viewport.vrs_mode] to be set to [constant Viewport.VRS_XR].
+		</member>
 		<member name="xr_play_area_mode" type="int" setter="set_play_area_mode" getter="get_play_area_mode" overrides="XRInterface" enum="XRInterface.PlayAreaMode" default="1" />
 	</members>
 </class>

modules/mobile_vr/mobile_vr_interface.cpp

@@ -250,6 +250,16 @@ void MobileVRInterface::_bind_methods() {
 	ADD_PROPERTY(PropertyInfo(Variant::FLOAT, "oversample", PROPERTY_HINT_RANGE, "1.0,2.0,0.1"), "set_oversample", "get_oversample");
 	ADD_PROPERTY(PropertyInfo(Variant::FLOAT, "k1", PROPERTY_HINT_RANGE, "0.1,10.0,0.0001"), "set_k1", "get_k1");
 	ADD_PROPERTY(PropertyInfo(Variant::FLOAT, "k2", PROPERTY_HINT_RANGE, "0.1,10.0,0.0001"), "set_k2", "get_k2");
+
+	ClassDB::bind_method(D_METHOD("get_vrs_min_radius"), &MobileVRInterface::get_vrs_min_radius);
+	ClassDB::bind_method(D_METHOD("set_vrs_min_radius", "radius"), &MobileVRInterface::set_vrs_min_radius);
+
+	ClassDB::bind_method(D_METHOD("get_vrs_strength"), &MobileVRInterface::get_vrs_strength);
+	ClassDB::bind_method(D_METHOD("set_vrs_strength", "strength"), &MobileVRInterface::set_vrs_strength);
+
+	ADD_GROUP("Vulkan VRS", "vrs_");
+	ADD_PROPERTY(PropertyInfo(Variant::FLOAT, "vrs_min_radius", PROPERTY_HINT_RANGE, "1.0,100.0,1.0"), "set_vrs_min_radius", "get_vrs_min_radius");
+	ADD_PROPERTY(PropertyInfo(Variant::FLOAT, "vrs_strength", PROPERTY_HINT_RANGE, "0.1,10.0,0.1"), "set_vrs_strength", "get_vrs_strength");
 }
 
 void MobileVRInterface::set_eye_height(const double p_eye_height) {
@@ -316,6 +326,22 @@ double MobileVRInterface::get_k2() const {
 	return k2;
 };
 
+float MobileVRInterface::get_vrs_min_radius() const {
+	return xr_vrs.get_vrs_min_radius();
+}
+
+void MobileVRInterface::set_vrs_min_radius(float p_vrs_min_radius) {
+	xr_vrs.set_vrs_min_radius(p_vrs_min_radius);
+}
+
+float MobileVRInterface::get_vrs_strength() const {
+	return xr_vrs.get_vrs_strength();
+}
+
+void MobileVRInterface::set_vrs_strength(float p_vrs_strength) {
+	xr_vrs.set_vrs_strength(p_vrs_strength);
+}
+
 uint32_t MobileVRInterface::get_view_count() {
 	// needs stereo...
 	return 2;
@@ -489,11 +515,16 @@ Vector<BlitToScreen> MobileVRInterface::post_draw_viewport(RID p_render_target,
 
 	Vector<BlitToScreen> blit_to_screen;
 
-	// We must have a valid render target
+	// We must have a valid render target.
 	ERR_FAIL_COND_V(!p_render_target.is_valid(), blit_to_screen);
 
-	// Because we are rendering to our device we must use our main viewport!
-	ERR_FAIL_COND_V(p_screen_rect == Rect2(), blit_to_screen);
+	// We will only output to screen if this is our main viewport.
+	if (p_screen_rect == Rect2()) {
+		// Warn the developer once, it's up to the developer to output to screen.
+		WARN_PRINT_ONCE("SubViewport used with MobileVRInterface, no output to screen");
+
+		return blit_to_screen;
+	}
 
 	Rect2 modified_screen_rect = Rect2(p_screen_rect.position + offset_rect.position * p_screen_rect.size, p_screen_rect.size * offset_rect.size);
 
@@ -542,6 +573,23 @@ void MobileVRInterface::process() {
 	};
 };
 
+RID MobileVRInterface::get_vrs_texture() {
+	PackedVector2Array eye_foci;
+
+	Size2 target_size = get_render_target_size();
+	real_t aspect_ratio = target_size.x / target_size.y;
+	uint32_t view_count = get_view_count();
+
+	for (uint32_t v = 0; v < view_count; v++) {
+		Projection cm = get_projection_for_view(v, aspect_ratio, 0.1, 1000.0);
+		Vector3 center = cm.xform(Vector3(0.0, 0.0, 999.0));
+
+		eye_foci.push_back(Vector2(center.x, center.y));
+	}
+
+	return xr_vrs.make_vrs_texture(target_size, eye_foci);
+}
+
 MobileVRInterface::MobileVRInterface() {}
 
 MobileVRInterface::~MobileVRInterface() {

modules/mobile_vr/mobile_vr_interface.h

@@ -33,6 +33,7 @@
 
 #include "servers/xr/xr_interface.h"
 #include "servers/xr/xr_positional_tracker.h"
+#include "servers/xr/xr_vrs.h"
 
 /**
 	The mobile interface is a native VR interface that can be used on Android and iOS phones.
@@ -72,6 +73,8 @@ class MobileVRInterface : public XRInterface {
 	Ref<XRPositionalTracker> head;
 	Transform3D head_transform;
 
+	XRVRS xr_vrs;
+
 	/*
 		logic for processing our sensor data, this was originally in our positional tracker logic but I think
 		that doesn't make sense in hindsight. It only makes marginally more sense to park it here for now,
@@ -138,6 +141,12 @@ class MobileVRInterface : public XRInterface {
 	void set_k2(const double p_k2);
 	double get_k2() const;
 
+	float get_vrs_min_radius() const;
+	void set_vrs_min_radius(float p_vrs_min_radius);
+
+	float get_vrs_strength() const;
+	void set_vrs_strength(float p_vrs_strength);
+
 	virtual StringName get_name() const override;
 	virtual uint32_t get_capabilities() const override;
 
@@ -161,6 +170,8 @@ class MobileVRInterface : public XRInterface {
 
 	virtual void process() override;
 
+	virtual RID get_vrs_texture() override;
+
 	MobileVRInterface();
 	~MobileVRInterface();
 };

modules/openxr/doc_classes/OpenXRInterface.xml

@@ -152,6 +152,14 @@
 		<member name="render_target_size_multiplier" type="float" setter="set_render_target_size_multiplier" getter="get_render_target_size_multiplier" default="1.0">
 			The render size multiplier for the current HMD. Must be set before the interface has been initialized.
 		</member>
+		<member name="vrs_min_radius" type="float" setter="set_vrs_min_radius" getter="get_vrs_min_radius" default="20.0">
+			The minimum radius around the focal point where full quality is guaranteed if VRS is used as a percentage of screen size.
+			[b]Note:[/b] Mobile and Forward+ renderers only. Requires [member Viewport.vrs_mode] to be set to [constant Viewport.VRS_XR].
+		</member>
+		<member name="vrs_strength" type="float" setter="set_vrs_strength" getter="get_vrs_strength" default="1.0">
+			The strength used to calculate the VRS density map. The greater this value, the more noticeable VRS is. This improves performance at the cost of quality.
+			[b]Note:[/b] Mobile and Forward+ renderers only. Requires [member Viewport.vrs_mode] to be set to [constant Viewport.VRS_XR].
+		</member>
 	</members>
 	<signals>
 		<signal name="instance_exiting">

modules/openxr/extensions/openxr_eye_gaze_interaction.cpp

@@ -34,6 +34,7 @@
 #include "core/os/os.h"
 
 #include "../action_map/openxr_interaction_profile_metadata.h"
+#include "../openxr_api.h"
 
 OpenXREyeGazeInteractionExtension *OpenXREyeGazeInteractionExtension::singleton = nullptr;
 
@@ -106,3 +107,38 @@ void OpenXREyeGazeInteractionExtension::on_register_metadata() {
 	metadata->register_interaction_profile("Eye gaze", "/interaction_profiles/ext/eye_gaze_interaction", XR_EXT_EYE_GAZE_INTERACTION_EXTENSION_NAME);
 	metadata->register_io_path("/interaction_profiles/ext/eye_gaze_interaction", "Gaze pose", "/user/eyes_ext", "/user/eyes_ext/input/gaze_ext/pose", "", OpenXRAction::OPENXR_ACTION_POSE);
 }
+
+bool OpenXREyeGazeInteractionExtension::get_eye_gaze_pose(double p_dist, Vector3 &r_eye_pose) {
+	OpenXRAPI *openxr_api = OpenXRAPI::get_singleton();
+	ERR_FAIL_NULL_V(openxr_api, false);
+
+	if (!init_eye_gaze_pose) {
+		init_eye_gaze_pose = true;
+
+		eye_tracker = openxr_api->find_tracker("/user/eyes_ext");
+		if (eye_tracker.is_null()) {
+			WARN_PRINT("Couldn't obtain eye tracker");
+		}
+
+		eye_action = openxr_api->find_action("eye_gaze_pose");
+		if (eye_action.is_null()) {
+			WARN_PRINT("Couldn't obtain pose action for `eye_gaze_pose`, make sure to add this to your action map.");
+		}
+	}
+
+	if (eye_tracker.is_null() || eye_action.is_null()) {
+		return false;
+	}
+
+	Transform3D eye_transform;
+	Vector3 linear_velocity;
+	Vector3 angular_velocity;
+	XRPose::TrackingConfidence confidence = openxr_api->get_action_pose(eye_action, eye_tracker, eye_transform, linear_velocity, angular_velocity);
+	if (confidence == XRPose::XR_TRACKING_CONFIDENCE_NONE) {
+		return false;
+	}
+
+	r_eye_pose = eye_transform.origin + eye_transform.basis[2] * p_dist;
+
+	return true;
+}

modules/openxr/extensions/openxr_eye_gaze_interaction.h

@@ -50,11 +50,17 @@ class OpenXREyeGazeInteractionExtension : public OpenXRExtensionWrapper {
 
 	virtual void on_register_metadata() override;
 
+	bool get_eye_gaze_pose(double p_dist, Vector3 &r_eye_pose);
+
 private:
 	static OpenXREyeGazeInteractionExtension *singleton;
 
 	bool available = false;
 	XrSystemEyeGazeInteractionPropertiesEXT properties;
+
+	bool init_eye_gaze_pose = false;
+	RID eye_tracker;
+	RID eye_action;
 };
 
 #endif // OPENXR_EYE_GAZE_INTERACTION_H