Make PipelineCache internally mutable. (#7205)

# Objective

- Allow rendering queue systems to use a `Res<PipelineCache>` even for queueing up new rendering pipelines. This is part of unblocking parallel execution queue systems.

## Solution

- Make `PipelineCache` internally mutable w.r.t to queueing new pipelines. Pipelines are no longer immediately updated into the cache state, but rather queued into a Vec. The Vec of pending new pipelines is then later processed at the same time we actually create the queued pipelines on the GPU device.

---

## Changelog

`PipelineCache` no longer requires mutable access in order to queue render / compute pipelines.

## Migration Guide

* Most usages of `resource_mut::<PipelineCache>` and `ResMut<PipelineCache>` can be changed to `resource::<PipelineCache>` and `Res<PipelineCache>` as long as they don't use any methods requiring mutability - the only public method requiring it is `process_queue`.

Author: danchia

crates/bevy_core_pipeline/src/bloom/mod.rs

@@ -434,7 +434,7 @@ impl FromWorld for BloomPipelines {
                 ],
             });
 
-        let mut pipeline_cache = world.resource_mut::<PipelineCache>();
+        let pipeline_cache = world.resource::<PipelineCache>();
 
         let downsampling_prefilter_pipeline =
             pipeline_cache.queue_render_pipeline(RenderPipelineDescriptor {

crates/bevy_core_pipeline/src/fxaa/mod.rs

@@ -223,7 +223,7 @@ impl SpecializedRenderPipeline for FxaaPipeline {
 
 pub fn prepare_fxaa_pipelines(
     mut commands: Commands,
-    mut pipeline_cache: ResMut<PipelineCache>,
+    pipeline_cache: Res<PipelineCache>,
     mut pipelines: ResMut<SpecializedRenderPipelines<FxaaPipeline>>,
     fxaa_pipeline: Res<FxaaPipeline>,
     views: Query<(Entity, &ExtractedView, &Fxaa)>,
@@ -233,7 +233,7 @@ pub fn prepare_fxaa_pipelines(
             continue;
         }
         let pipeline_id = pipelines.specialize(
-            &mut pipeline_cache,
+            &pipeline_cache,
             &fxaa_pipeline,
             FxaaPipelineKey {
                 edge_threshold: fxaa.edge_threshold,

crates/bevy_core_pipeline/src/tonemapping/mod.rs

@@ -126,7 +126,7 @@ pub struct ViewTonemappingPipeline(CachedRenderPipelineId);
 
 pub fn queue_view_tonemapping_pipelines(
     mut commands: Commands,
-    mut pipeline_cache: ResMut<PipelineCache>,
+    pipeline_cache: Res<PipelineCache>,
     mut pipelines: ResMut<SpecializedRenderPipelines<TonemappingPipeline>>,
     upscaling_pipeline: Res<TonemappingPipeline>,
     view_targets: Query<(Entity, &Tonemapping)>,
@@ -136,7 +136,7 @@ pub fn queue_view_tonemapping_pipelines(
             let key = TonemappingPipelineKey {
                 deband_dither: *deband_dither,
             };
-            let pipeline = pipelines.specialize(&mut pipeline_cache, &upscaling_pipeline, key);
+            let pipeline = pipelines.specialize(&pipeline_cache, &upscaling_pipeline, key);
 
             commands
                 .entity(entity)

crates/bevy_core_pipeline/src/upscaling/mod.rs

@@ -112,7 +112,7 @@ pub struct ViewUpscalingPipeline(CachedRenderPipelineId);
 
 fn queue_view_upscaling_pipelines(
     mut commands: Commands,
-    mut pipeline_cache: ResMut<PipelineCache>,
+    pipeline_cache: Res<PipelineCache>,
     mut pipelines: ResMut<SpecializedRenderPipelines<UpscalingPipeline>>,
     upscaling_pipeline: Res<UpscalingPipeline>,
     view_targets: Query<(Entity, &ViewTarget)>,
@@ -122,7 +122,7 @@ fn queue_view_upscaling_pipelines(
             upscaling_mode: UpscalingMode::Filtering,
             texture_format: view_target.out_texture_format(),
         };
-        let pipeline = pipelines.specialize(&mut pipeline_cache, &upscaling_pipeline, key);
+        let pipeline = pipelines.specialize(&pipeline_cache, &upscaling_pipeline, key);
 
         commands
             .entity(entity)

crates/bevy_pbr/src/material.rs

@@ -333,7 +333,7 @@ pub fn queue_material_meshes<M: Material>(
     transparent_draw_functions: Res<DrawFunctions<Transparent3d>>,
     material_pipeline: Res<MaterialPipeline<M>>,
     mut pipelines: ResMut<SpecializedMeshPipelines<MaterialPipeline<M>>>,
-    mut pipeline_cache: ResMut<PipelineCache>,
+    pipeline_cache: Res<PipelineCache>,
     msaa: Res<Msaa>,
     render_meshes: Res<RenderAssets<Mesh>>,
     render_materials: Res<RenderMaterials<M>>,
@@ -391,7 +391,7 @@ pub fn queue_material_meshes<M: Material>(
                         }
 
                         let pipeline_id = pipelines.specialize(
-                            &mut pipeline_cache,
+                            &pipeline_cache,
                             &material_pipeline,
                             MaterialPipelineKey {
                                 mesh_key,

crates/bevy_pbr/src/render/light.rs

@@ -1626,7 +1626,7 @@ pub fn queue_shadows(
     casting_meshes: Query<&Handle<Mesh>, Without<NotShadowCaster>>,
     render_meshes: Res<RenderAssets<Mesh>>,
     mut pipelines: ResMut<SpecializedMeshPipelines<ShadowPipeline>>,
-    mut pipeline_cache: ResMut<PipelineCache>,
+    pipeline_cache: Res<PipelineCache>,
     view_lights: Query<&ViewLightEntities>,
     mut view_light_shadow_phases: Query<(&LightEntity, &mut RenderPhase<Shadow>)>,
     point_light_entities: Query<&CubemapVisibleEntities, With<ExtractedPointLight>>,
@@ -1661,7 +1661,7 @@ pub fn queue_shadows(
                         let key =
                             ShadowPipelineKey::from_primitive_topology(mesh.primitive_topology);
                         let pipeline_id = pipelines.specialize(
-                            &mut pipeline_cache,
+                            &pipeline_cache,
                             &shadow_pipeline,
                             key,
                             &mesh.layout,

crates/bevy_pbr/src/wireframe.rs

@@ -108,7 +108,7 @@ fn queue_wireframes(
     wireframe_config: Res<WireframeConfig>,
     wireframe_pipeline: Res<WireframePipeline>,
     mut pipelines: ResMut<SpecializedMeshPipelines<WireframePipeline>>,
-    mut pipeline_cache: ResMut<PipelineCache>,
+    pipeline_cache: Res<PipelineCache>,
     msaa: Res<Msaa>,
     mut material_meshes: ParamSet<(
         Query<(Entity, &Handle<Mesh>, &MeshUniform)>,
@@ -128,7 +128,7 @@ fn queue_wireframes(
                     let key = view_key
                         | MeshPipelineKey::from_primitive_topology(mesh.primitive_topology);
                     let pipeline_id = pipelines.specialize(
-                        &mut pipeline_cache,
+                        &pipeline_cache,
                         &wireframe_pipeline,
                         key,
                         &mesh.layout,

crates/bevy_render/src/lib.rs

@@ -198,6 +198,8 @@ impl Plugin for RenderPlugin {
                 .add_stage(
                     RenderStage::Render,
                     SystemStage::parallel()
+                        // Note: Must run before `render_system` in order to
+                        // processed newly queued pipelines.
                         .with_system(PipelineCache::process_pipeline_queue_system)
                         .with_system(render_system.at_end()),
                 )

crates/bevy_render/src/render_resource/pipeline_cache.rs

@@ -17,6 +17,7 @@ use bevy_utils::{
     tracing::{debug, error},
     Entry, HashMap, HashSet,
 };
+use parking_lot::Mutex;
 use std::{hash::Hash, iter::FusedIterator, mem, ops::Deref};
 use thiserror::Error;
 use wgpu::{PipelineLayoutDescriptor, VertexBufferLayout as RawVertexBufferLayout};
@@ -343,6 +344,7 @@ pub struct PipelineCache {
     device: RenderDevice,
     pipelines: Vec<CachedPipeline>,
     waiting_pipelines: HashSet<CachedPipelineId>,
+    new_pipelines: Mutex<Vec<CachedPipeline>>,
 }
 
 impl PipelineCache {
@@ -357,6 +359,7 @@ impl PipelineCache {
             layout_cache: default(),
             shader_cache: default(),
             waiting_pipelines: default(),
+            new_pipelines: default(),
             pipelines: default(),
         }
     }
@@ -455,15 +458,15 @@ impl PipelineCache {
     /// [`get_render_pipeline_state()`]: PipelineCache::get_render_pipeline_state
     /// [`get_render_pipeline()`]: PipelineCache::get_render_pipeline
     pub fn queue_render_pipeline(
-        &mut self,
+        &self,
         descriptor: RenderPipelineDescriptor,
     ) -> CachedRenderPipelineId {
-        let id = CachedRenderPipelineId(self.pipelines.len());
-        self.pipelines.push(CachedPipeline {
+        let mut new_pipelines = self.new_pipelines.lock();
+        let id = CachedRenderPipelineId(self.pipelines.len() + new_pipelines.len());
+        new_pipelines.push(CachedPipeline {
             descriptor: PipelineDescriptor::RenderPipelineDescriptor(Box::new(descriptor)),
             state: CachedPipelineState::Queued,
         });
-        self.waiting_pipelines.insert(id.0);
         id
     }
 
@@ -481,15 +484,15 @@ impl PipelineCache {
     /// [`get_compute_pipeline_state()`]: PipelineCache::get_compute_pipeline_state
     /// [`get_compute_pipeline()`]: PipelineCache::get_compute_pipeline
     pub fn queue_compute_pipeline(
-        &mut self,
+        &self,
         descriptor: ComputePipelineDescriptor,
     ) -> CachedComputePipelineId {
-        let id = CachedComputePipelineId(self.pipelines.len());
-        self.pipelines.push(CachedPipeline {
+        let mut new_pipelines = self.new_pipelines.lock();
+        let id = CachedComputePipelineId(self.pipelines.len() + new_pipelines.len());
+        new_pipelines.push(CachedPipeline {
             descriptor: PipelineDescriptor::ComputePipelineDescriptor(Box::new(descriptor)),
             state: CachedPipelineState::Queued,
         });
-        self.waiting_pipelines.insert(id.0);
         id
     }
 
@@ -632,9 +635,18 @@ impl PipelineCache {
     ///
     /// [`RenderStage::Render`]: crate::RenderStage::Render
     pub fn process_queue(&mut self) {
-        let waiting_pipelines = mem::take(&mut self.waiting_pipelines);
+        let mut waiting_pipelines = mem::take(&mut self.waiting_pipelines);
         let mut pipelines = mem::take(&mut self.pipelines);
 
+        {
+            let mut new_pipelines = self.new_pipelines.lock();
+            for new_pipeline in new_pipelines.drain(..) {
+                let id = pipelines.len();
+                pipelines.push(new_pipeline);
+                waiting_pipelines.insert(id);
+            }
+        }
+
         for id in waiting_pipelines {
             let pipeline = &mut pipelines[id];
             if matches!(pipeline.state, CachedPipelineState::Ok(_)) {

crates/bevy_render/src/render_resource/pipeline_specializer.rs

@@ -33,7 +33,7 @@ impl<S: SpecializedRenderPipeline> Default for SpecializedRenderPipelines<S> {
 impl<S: SpecializedRenderPipeline> SpecializedRenderPipelines<S> {
     pub fn specialize(
         &mut self,
-        cache: &mut PipelineCache,
+        cache: &PipelineCache,
         specialize_pipeline: &S,
         key: S::Key,
     ) -> CachedRenderPipelineId {
@@ -103,7 +103,7 @@ impl<S: SpecializedMeshPipeline> SpecializedMeshPipelines<S> {
     #[inline]
     pub fn specialize(
         &mut self,
-        cache: &mut PipelineCache,
+        cache: &PipelineCache,
         specialize_pipeline: &S,
         key: S::Key,
         layout: &MeshVertexBufferLayout,