Opt-in raw vulkan initialization with hooks (#20565)

# Objective

Currently registering additional required Vulkan features requires
either hard-coding them into `bevy_render` (see the current [DLSS
proposal](#19864)), or forcing
the plugin to take full manual control over wgpu initialization. Neither
is an acceptable or scalable option for a modular engine like Bevy.

## Solution

* Add a new `raw_vulkan_init` Cargo feature, that when enabled switches
to wgpu's raw Vulkan init path, which accepts callbacks that allow
checking and requiring additional Vulkan features.
* Add a new `WgpuRawVulkanInitSettings` resource, which provides wgpu
Vulkan Instance and Device init callbacks, which can be used to detect
and register vulkan features.
* These callbacks can register arbitrary features in the new
`AdditionalVulkanFeatures` resource, which is inserted into the
RenderApp at the same time that the RenderDevice is.

This enables plugins to register initialization callbacks, which must
happen _before_ RenderPlugin. They can then feed off of
`AdditionalVulkanFeatures`, after the renderer is initialized, to detect
if a given feature is supported.

Due to the current lifecycles, this is best accomplished with either:
* A separate "init plugin" that is registered before RenderPlugin, and a
"logic plugin" that does everything else. This should be used if the
plugin logic needs `Plugin::build()` access to render device state,
which needs to be registered _after_ RenderPlugin to have access. The
proposed DLSS feature needs this pattern.
* A single "init plugin" that is registered before RenderPlugin and does
everything. Use this pattern if you can as it is simpler.

With deferred plugin init, we could remove the need for this split.

---------

Co-authored-by: JMS55 <[email protected]>

Author: cart

Cargo.toml

@@ -327,6 +327,9 @@ spirv_shader_passthrough = ["bevy_internal/spirv_shader_passthrough"]
 # Statically linked DXC shader compiler for DirectX 12
 statically-linked-dxc = ["bevy_internal/statically-linked-dxc"]
 
+# Forces the wgpu instance to be initialized using the raw Vulkan HAL, enabling additional configuration
+raw_vulkan_init = ["bevy_internal/raw_vulkan_init"]
+
 # Tracing support, saving a file in Chrome Tracing format
 trace_chrome = ["trace", "bevy_internal/trace_chrome"]
 

crates/bevy_internal/Cargo.toml

@@ -62,6 +62,9 @@ spirv_shader_passthrough = ["bevy_render/spirv_shader_passthrough"]
 # TODO: When wgpu switches to DirectX 12 instead of Vulkan by default on windows, make this a default feature
 statically-linked-dxc = ["bevy_render/statically-linked-dxc"]
 
+# Forces the wgpu instance to be initialized using the raw Vulkan HAL, enabling additional configuration
+raw_vulkan_init = ["bevy_render/raw_vulkan_init"]
+
 # Include tonemapping LUT KTX2 files.
 tonemapping_luts = ["bevy_core_pipeline/tonemapping_luts"]
 # Include Bluenoise texture for environment map generation.

crates/bevy_render/Cargo.toml

@@ -32,6 +32,9 @@ spirv_shader_passthrough = ["wgpu/spirv"]
 # TODO: When wgpu switches to DirectX 12 instead of Vulkan by default on windows, make this a default feature
 statically-linked-dxc = ["wgpu/static-dxc"]
 
+# Forces the wgpu instance to be initialized using the raw Vulkan HAL, enabling additional configuration
+raw_vulkan_init = ["wgpu/vulkan"]
+
 trace = ["profiling"]
 tracing-tracy = ["dep:tracy-client"]
 ci_limits = []

crates/bevy_render/src/lib.rs

@@ -85,7 +85,7 @@ use crate::{
     mesh::{MeshPlugin, MorphPlugin, RenderMesh},
     render_asset::prepare_assets,
     render_resource::{init_empty_bind_group_layout, PipelineCache},
-    renderer::{render_system, RenderAdapterInfo, RenderInstance},
+    renderer::{render_system, RenderAdapterInfo},
     settings::RenderCreation,
     storage::StoragePlugin,
     texture::TexturePlugin,
@@ -114,7 +114,6 @@ use render_asset::{
 use settings::RenderResources;
 use std::sync::Mutex;
 use sync_world::{despawn_temporary_render_entities, entity_sync_system, SyncWorldPlugin};
-use tracing::debug;
 pub use wgpu_wrapper::WgpuWrapper;
 
 /// Contains the default Bevy rendering backend based on wgpu.
@@ -329,76 +328,29 @@ impl Plugin for RenderPlugin {
                         .single(app.world())
                         .ok()
                         .cloned();
+
                     let settings = render_creation.clone();
+
+                    #[cfg(feature = "raw_vulkan_init")]
+                    let raw_vulkan_init_settings = app
+                        .world_mut()
+                        .get_resource::<renderer::raw_vulkan_init::RawVulkanInitSettings>()
+                        .cloned()
+                        .unwrap_or_default();
+
                     let async_renderer = async move {
-                        let instance = wgpu::Instance::new(&wgpu::InstanceDescriptor {
+                        let render_resources = renderer::initialize_renderer(
                             backends,
-                            flags: settings.instance_flags,
-                            memory_budget_thresholds: settings.instance_memory_budget_thresholds,
-                            backend_options: wgpu::BackendOptions {
-                                gl: wgpu::GlBackendOptions {
-                                    gles_minor_version: settings.gles3_minor_version,
-                                    fence_behavior: wgpu::GlFenceBehavior::Normal,
-                                },
-                                dx12: wgpu::Dx12BackendOptions {
-                                    shader_compiler: settings.dx12_shader_compiler.clone(),
-                                },
-                                noop: wgpu::NoopBackendOptions { enable: false },
-                            },
-                        });
-
-                        let surface = primary_window.and_then(|wrapper| {
-                            let maybe_handle = wrapper.0.lock().expect(
-                                "Couldn't get the window handle in time for renderer initialization",
-                            );
-                            if let Some(wrapper) = maybe_handle.as_ref() {
-                                // SAFETY: Plugins should be set up on the main thread.
-                                let handle = unsafe { wrapper.get_handle() };
-                                Some(
-                                    instance
-                                        .create_surface(handle)
-                                        .expect("Failed to create wgpu surface"),
-                                )
-                            } else {
-                                None
-                            }
-                        });
-
-                        let force_fallback_adapter = std::env::var("WGPU_FORCE_FALLBACK_ADAPTER")
-                            .map_or(settings.force_fallback_adapter, |v| {
-                                !(v.is_empty() || v == "0" || v == "false")
-                            });
-
-                        let desired_adapter_name = std::env::var("WGPU_ADAPTER_NAME")
-                            .as_deref()
-                            .map_or(settings.adapter_name.clone(), |x| Some(x.to_lowercase()));
-
-                        let request_adapter_options = wgpu::RequestAdapterOptions {
-                            power_preference: settings.power_preference,
-                            compatible_surface: surface.as_ref(),
-                            force_fallback_adapter,
-                        };
-
-                        let (device, queue, adapter_info, render_adapter) =
-                            renderer::initialize_renderer(
-                                &instance,
-                                &settings,
-                                &request_adapter_options,
-                                desired_adapter_name,
-                            )
-                            .await;
-                        debug!("Configured wgpu adapter Limits: {:#?}", device.limits());
-                        debug!("Configured wgpu adapter Features: {:#?}", device.features());
-                        let mut future_render_resources_inner =
-                            future_render_resources_wrapper.lock().unwrap();
-                        *future_render_resources_inner = Some(RenderResources(
-                            device,
-                            queue,
-                            adapter_info,
-                            render_adapter,
-                            RenderInstance(Arc::new(WgpuWrapper::new(instance))),
-                        ));
+                            primary_window,
+                            &settings,
+                            #[cfg(feature = "raw_vulkan_init")]
+                            raw_vulkan_init_settings,
+                        )
+                        .await;
+
+                        *future_render_resources_wrapper.lock().unwrap() = Some(render_resources);
                     };
+
                     // In wasm, spawn a task and detach it for execution
                     #[cfg(target_arch = "wasm32")]
                     bevy_tasks::IoTaskPool::get()
@@ -461,8 +413,9 @@ impl Plugin for RenderPlugin {
         if let Some(future_render_resources) =
             app.world_mut().remove_resource::<FutureRenderResources>()
         {
-            let RenderResources(device, queue, adapter_info, render_adapter, instance) =
-                future_render_resources.0.lock().unwrap().take().unwrap();
+            let render_resources = future_render_resources.0.lock().unwrap().take().unwrap();
+            let RenderResources(device, queue, adapter_info, render_adapter, instance, ..) =
+                render_resources;
 
             let compressed_image_format_support = CompressedImageFormatSupport(
                 CompressedImageFormats::from_features(device.features()),
@@ -476,6 +429,13 @@ impl Plugin for RenderPlugin {
 
             let render_app = app.sub_app_mut(RenderApp);
 
+            #[cfg(feature = "raw_vulkan_init")]
+            {
+                let additional_vulkan_features: renderer::raw_vulkan_init::AdditionalVulkanFeatures =
+                    render_resources.5;
+                render_app.insert_resource(additional_vulkan_features);
+            }
+
             render_app
                 .insert_resource(instance)
                 .insert_resource(PipelineCache::new(

crates/bevy_render/src/renderer/mod.rs

@@ -1,28 +1,29 @@
 mod graph_runner;
+#[cfg(feature = "raw_vulkan_init")]
+pub mod raw_vulkan_init;
 mod render_device;
 
-use crate::WgpuWrapper;
-use bevy_derive::{Deref, DerefMut};
-#[cfg(not(all(target_arch = "wasm32", target_feature = "atomics")))]
-use bevy_tasks::ComputeTaskPool;
 pub use graph_runner::*;
 pub use render_device::*;
-use tracing::{debug, error, info, info_span, warn};
 
 use crate::{
     diagnostic::{internal::DiagnosticsRecorder, RecordDiagnostics},
     render_graph::RenderGraph,
     render_phase::TrackedRenderPass,
     render_resource::RenderPassDescriptor,
-    settings::{WgpuSettings, WgpuSettingsPriority},
+    settings::{RenderResources, WgpuSettings, WgpuSettingsPriority},
     view::{ExtractedWindows, ViewTarget},
+    WgpuWrapper,
 };
 use alloc::sync::Arc;
+use bevy_derive::{Deref, DerefMut};
 use bevy_ecs::{prelude::*, system::SystemState};
 use bevy_platform::time::Instant;
 use bevy_time::TimeSender;
+use bevy_window::RawHandleWrapperHolder;
+use tracing::{debug, error, info, info_span, warn};
 use wgpu::{
-    Adapter, AdapterInfo, CommandBuffer, CommandEncoder, DeviceType, Instance, Queue,
+    Adapter, AdapterInfo, Backends, CommandBuffer, CommandEncoder, DeviceType, Instance, Queue,
     RequestAdapterOptions, Trace,
 };
 
@@ -171,15 +172,76 @@ fn find_adapter_by_name(
 /// Initializes the renderer by retrieving and preparing the GPU instance, device and queue
 /// for the specified backend.
 pub async fn initialize_renderer(
-    instance: &Instance,
+    backends: Backends,
+    primary_window: Option<RawHandleWrapperHolder>,
     options: &WgpuSettings,
-    request_adapter_options: &RequestAdapterOptions<'_, '_>,
-    desired_adapter_name: Option<String>,
-) -> (RenderDevice, RenderQueue, RenderAdapterInfo, RenderAdapter) {
+    #[cfg(feature = "raw_vulkan_init")]
+    raw_vulkan_init_settings: raw_vulkan_init::RawVulkanInitSettings,
+) -> RenderResources {
+    let instance_descriptor = wgpu::InstanceDescriptor {
+        backends,
+        flags: options.instance_flags,
+        memory_budget_thresholds: options.instance_memory_budget_thresholds,
+        backend_options: wgpu::BackendOptions {
+            gl: wgpu::GlBackendOptions {
+                gles_minor_version: options.gles3_minor_version,
+                fence_behavior: wgpu::GlFenceBehavior::Normal,
+            },
+            dx12: wgpu::Dx12BackendOptions {
+                shader_compiler: options.dx12_shader_compiler.clone(),
+            },
+            noop: wgpu::NoopBackendOptions { enable: false },
+        },
+    };
+
+    #[cfg(not(feature = "raw_vulkan_init"))]
+    let instance = Instance::new(&instance_descriptor);
+    #[cfg(feature = "raw_vulkan_init")]
+    let mut additional_vulkan_features = raw_vulkan_init::AdditionalVulkanFeatures::default();
+    #[cfg(feature = "raw_vulkan_init")]
+    let instance = raw_vulkan_init::create_raw_vulkan_instance(
+        &instance_descriptor,
+        &raw_vulkan_init_settings,
+        &mut additional_vulkan_features,
+    );
+
+    let surface = primary_window.and_then(|wrapper| {
+        let maybe_handle = wrapper
+            .0
+            .lock()
+            .expect("Couldn't get the window handle in time for renderer initialization");
+        if let Some(wrapper) = maybe_handle.as_ref() {
+            // SAFETY: Plugins should be set up on the main thread.
+            let handle = unsafe { wrapper.get_handle() };
+            Some(
+                instance
+                    .create_surface(handle)
+                    .expect("Failed to create wgpu surface"),
+            )
+        } else {
+            None
+        }
+    });
+
+    let force_fallback_adapter = std::env::var("WGPU_FORCE_FALLBACK_ADAPTER")
+        .map_or(options.force_fallback_adapter, |v| {
+            !(v.is_empty() || v == "0" || v == "false")
+        });
+
+    let desired_adapter_name = std::env::var("WGPU_ADAPTER_NAME")
+        .as_deref()
+        .map_or(options.adapter_name.clone(), |x| Some(x.to_lowercase()));
+
+    let request_adapter_options = RequestAdapterOptions {
+        power_preference: options.power_preference,
+        compatible_surface: surface.as_ref(),
+        force_fallback_adapter,
+    };
+
     #[cfg(not(target_family = "wasm"))]
     let mut selected_adapter = desired_adapter_name.and_then(|adapter_name| {
         find_adapter_by_name(
-            instance,
+            &instance,
             options,
             request_adapter_options.compatible_surface,
             &adapter_name,
@@ -198,7 +260,10 @@ pub async fn initialize_renderer(
             "Searching for adapter with options: {:?}",
             request_adapter_options
         );
-        selected_adapter = instance.request_adapter(request_adapter_options).await.ok();
+        selected_adapter = instance
+            .request_adapter(&request_adapter_options)
+            .await
+            .ok();
     }
 
     let adapter = selected_adapter.expect(GPU_NOT_FOUND_ERROR_MESSAGE);
@@ -364,24 +429,39 @@ pub async fn initialize_renderer(
         };
     }
 
-    let (device, queue) = adapter
-        .request_device(&wgpu::DeviceDescriptor {
-            label: options.device_label.as_ref().map(AsRef::as_ref),
-            required_features: features,
-            required_limits: limits,
-            memory_hints: options.memory_hints.clone(),
-            // See https://github.com/gfx-rs/wgpu/issues/5974
-            trace: Trace::Off,
-        })
-        .await
-        .unwrap();
-    let queue = Arc::new(WgpuWrapper::new(queue));
-    let adapter = Arc::new(WgpuWrapper::new(adapter));
-    (
+    let device_descriptor = wgpu::DeviceDescriptor {
+        label: options.device_label.as_ref().map(AsRef::as_ref),
+        required_features: features,
+        required_limits: limits,
+        memory_hints: options.memory_hints.clone(),
+        // See https://github.com/gfx-rs/wgpu/issues/5974
+        trace: Trace::Off,
+    };
+
+    #[cfg(not(feature = "raw_vulkan_init"))]
+    let (device, queue) = adapter.request_device(&device_descriptor).await.unwrap();
+
+    #[cfg(feature = "raw_vulkan_init")]
+    let (device, queue) = raw_vulkan_init::create_raw_device(
+        &adapter,
+        &device_descriptor,
+        &raw_vulkan_init_settings,
+        &mut additional_vulkan_features,
+    )
+    .await
+    .unwrap();
+
+    debug!("Configured wgpu adapter Limits: {:#?}", device.limits());
+    debug!("Configured wgpu adapter Features: {:#?}", device.features());
+
+    RenderResources(
         RenderDevice::from(device),
-        RenderQueue(queue),
+        RenderQueue(Arc::new(WgpuWrapper::new(queue))),
         RenderAdapterInfo(WgpuWrapper::new(adapter_info)),
-        RenderAdapter(adapter),
+        RenderAdapter(Arc::new(WgpuWrapper::new(adapter))),
+        RenderInstance(Arc::new(WgpuWrapper::new(instance))),
+        #[cfg(feature = "raw_vulkan_init")]
+        additional_vulkan_features,
     )
 }
 
@@ -499,22 +579,24 @@ impl<'w> RenderContext<'w> {
 
         #[cfg(not(all(target_arch = "wasm32", target_feature = "atomics")))]
         {
-            let mut task_based_command_buffers = ComputeTaskPool::get().scope(|task_pool| {
-                for (i, queued_command_buffer) in self.command_buffer_queue.into_iter().enumerate()
-                {
-                    match queued_command_buffer {
-                        QueuedCommandBuffer::Ready(command_buffer) => {
-                            command_buffers.push((i, command_buffer));
-                        }
-                        QueuedCommandBuffer::Task(command_buffer_generation_task) => {
-                            let render_device = self.render_device.clone();
-                            task_pool.spawn(async move {
-                                (i, command_buffer_generation_task(render_device))
-                            });
+            let mut task_based_command_buffers =
+                bevy_tasks::ComputeTaskPool::get().scope(|task_pool| {
+                    for (i, queued_command_buffer) in
+                        self.command_buffer_queue.into_iter().enumerate()
+                    {
+                        match queued_command_buffer {
+                            QueuedCommandBuffer::Ready(command_buffer) => {
+                                command_buffers.push((i, command_buffer));
+                            }
+                            QueuedCommandBuffer::Task(command_buffer_generation_task) => {
+                                let render_device = self.render_device.clone();
+                                task_pool.spawn(async move {
+                                    (i, command_buffer_generation_task(render_device))
+                                });
+                            }
                         }
                     }
-                }
-            });
+                });
             command_buffers.append(&mut task_based_command_buffers);
         }