Load and compose components once at start of validation

Signed-off-by: itowlson <ivan.towlson@fermyon.com>

Author: itowlson

crates/environments/src/lib.rs

@@ -5,15 +5,14 @@ mod loader;
 
 use environment_definition::{TargetEnvironment, TargetWorld, TriggerType};
 pub use loader::ResolutionContext;
-use loader::{component_source, ComponentSourceLoader, ComponentToValidate};
+use loader::{load_and_resolve_all, ComponentToValidate};
 
 pub async fn validate_application_against_environment_ids(
     env_ids: impl Iterator<Item = &str>,
     app: &spin_manifest::schema::v2::AppManifest,
     resolution_context: &ResolutionContext,
 ) -> anyhow::Result<()> {
-    let envs = futures::future::join_all(env_ids.map(resolve_environment_id)).await;
-    let envs: Vec<_> = envs.into_iter().collect::<Result<_, _>>()?;
+    let envs = join_all_result(env_ids.map(resolve_environment_id)).await?;
     validate_application_against_environments(&envs, app, resolution_context).await
 }
 
@@ -54,6 +53,8 @@ pub async fn validate_application_against_environments(
     app: &spin_manifest::schema::v2::AppManifest,
     resolution_context: &ResolutionContext,
 ) -> anyhow::Result<()> {
+    use futures::FutureExt;
+
     for trigger_type in app.triggers.keys() {
         if let Some(env) = envs
             .iter()
@@ -66,26 +67,15 @@ pub async fn validate_application_against_environments(
         }
     }
 
-    let components_by_trigger_type = app
-        .triggers
-        .iter()
-        .map(|(ty, ts)| {
-            ts.iter()
-                .map(|t| component_source(app, t))
-                .collect::<Result<Vec<_>, _>>()
-                .map(|css| (ty, css))
-        })
-        .collect::<Result<Vec<_>, _>>()?;
+    let components_by_trigger_type_futs = app.triggers.iter().map(|(ty, ts)| {
+        load_and_resolve_all(app, ts, resolution_context)
+            .map(|css| css.map(|css| (ty.to_owned(), css)))
+    });
+    let components_by_trigger_type = join_all_result(components_by_trigger_type_futs).await?;
 
     for (trigger_type, component) in components_by_trigger_type {
         for component in &component {
-            validate_component_against_environments(
-                envs,
-                trigger_type,
-                component,
-                resolution_context,
-            )
-            .await?;
+            validate_component_against_environments(envs, &trigger_type, component).await?;
         }
     }
 
@@ -96,7 +86,6 @@ async fn validate_component_against_environments(
     envs: &[TargetEnvironment],
     trigger_type: &TriggerType,
     component: &ComponentToValidate<'_>,
-    resolution_context: &ResolutionContext,
 ) -> anyhow::Result<()> {
     let worlds = envs
         .iter()
@@ -110,21 +99,18 @@ async fn validate_component_against_environments(
                 .map(|w| (e.name.as_str(), w))
         })
         .collect::<Result<std::collections::HashSet<_>, _>>()?;
-    validate_component_against_worlds(worlds.into_iter(), component, resolution_context).await?;
+    validate_component_against_worlds(worlds.into_iter(), component).await?;
     Ok(())
 }
 
 async fn validate_component_against_worlds(
     target_worlds: impl Iterator<Item = (&str, &TargetWorld)>,
     component: &ComponentToValidate<'_>,
-    resolution_context: &ResolutionContext,
 ) -> anyhow::Result<()> {
-    let loader = ComponentSourceLoader::new(resolution_context.wasm_loader());
-    let wasm_bytes = spin_compose::compose(&loader, component).await?;
+    let wasm_bytes = component.wasm_bytes();
 
     for (env_name, target_world) in target_worlds {
-        validate_wasm_against_any_world(env_name, target_world, component, wasm_bytes.as_ref())
-            .await?;
+        validate_wasm_against_any_world(env_name, target_world, component, wasm_bytes).await?;
     }
 
     tracing::info!(
@@ -223,3 +209,15 @@ async fn validate_wasm_against_world(
         },
     }
 }
+
+/// Equivalent to futures::future::join_all, but specialised for iterators of
+/// fallible futures. It returns a Result<Vec<...>> instead of a Vec<Result<...>> -
+/// this just moves the transposition boilerplate out of the main flow.
+async fn join_all_result<T, I>(iter: I) -> anyhow::Result<Vec<T>>
+where
+    I: IntoIterator,
+    I::Item: std::future::Future<Output = anyhow::Result<T>>,
+{
+    let vec_result = futures::future::join_all(iter).await;
+    vec_result.into_iter().collect()
+}

crates/environments/src/loader.rs

@@ -4,6 +4,12 @@ use anyhow::anyhow;
 use spin_common::ui::quoted_path;
 
 pub(crate) struct ComponentToValidate<'a> {
+    id: &'a str,
+    source_description: String,
+    wasm: Vec<u8>,
+}
+
+struct ComponentSource<'a> {
     id: &'a str,
     source: &'a spin_manifest::schema::v2::ComponentSource,
     dependencies: WrappedComponentDependencies,
@@ -14,22 +20,30 @@ impl<'a> ComponentToValidate<'a> {
         self.id
     }
 
-    pub fn source_description(&self) -> String {
-        match self.source {
-            spin_manifest::schema::v2::ComponentSource::Local(path) => {
-                format!("file {}", quoted_path(path))
-            }
-            spin_manifest::schema::v2::ComponentSource::Remote { url, .. } => format!("URL {url}"),
-            spin_manifest::schema::v2::ComponentSource::Registry { package, .. } => {
-                format!("package {package}")
-            }
-        }
+    pub fn source_description(&self) -> &str {
+        &self.source_description
+    }
+
+    pub fn wasm_bytes(&self) -> &[u8] {
+        &self.wasm
     }
 }
 
-pub fn component_source<'a>(
+pub async fn load_and_resolve_all<'a>(
+    app: &'a spin_manifest::schema::v2::AppManifest,
+    triggers: &'a [spin_manifest::schema::v2::Trigger],
+    resolution_context: &'a ResolutionContext,
+) -> anyhow::Result<Vec<ComponentToValidate<'a>>> {
+    let component_futures = triggers
+        .iter()
+        .map(|t| load_and_resolve_one(app, t, resolution_context));
+    crate::join_all_result(component_futures).await
+}
+
+async fn load_and_resolve_one<'a>(
     app: &'a spin_manifest::schema::v2::AppManifest,
     trigger: &'a spin_manifest::schema::v2::Trigger,
+    resolution_context: &'a ResolutionContext,
 ) -> anyhow::Result<ComponentToValidate<'a>> {
     let component_spec = trigger
         .component
@@ -50,10 +64,21 @@ pub fn component_source<'a>(
             (id, &component.source, &component.dependencies)
         }
     };
-    Ok(ComponentToValidate {
+
+    let component = ComponentSource {
         id,
         source,
         dependencies: WrappedComponentDependencies::new(dependencies),
+    };
+
+    let loader = ComponentSourceLoader::new(resolution_context.wasm_loader());
+
+    let wasm = spin_compose::compose(&loader, &component).await?;
+
+    Ok(ComponentToValidate {
+        id,
+        source_description: source_description(component.source),
+        wasm,
     })
 }
 
@@ -68,33 +93,29 @@ impl ResolutionContext {
         Ok(Self { wasm_loader })
     }
 
-    pub(crate) fn wasm_loader(&self) -> &spin_loader::WasmLoader {
+    fn wasm_loader(&self) -> &spin_loader::WasmLoader {
         &self.wasm_loader
     }
 }
 
-pub(crate) struct ComponentSourceLoader<'a> {
+struct ComponentSourceLoader<'a> {
     wasm_loader: &'a spin_loader::WasmLoader,
-    _phantom: std::marker::PhantomData<&'a usize>,
 }
 
 impl<'a> ComponentSourceLoader<'a> {
     pub fn new(wasm_loader: &'a spin_loader::WasmLoader) -> Self {
-        Self {
-            wasm_loader,
-            _phantom: std::marker::PhantomData,
-        }
+        Self { wasm_loader }
     }
 }
 
 #[async_trait::async_trait]
 impl<'a> spin_compose::ComponentSourceLoader for ComponentSourceLoader<'a> {
-    type Component = ComponentToValidate<'a>;
+    type Component = ComponentSource<'a>;
     type Dependency = WrappedComponentDependency;
     async fn load_component_source(&self, source: &Self::Component) -> anyhow::Result<Vec<u8>> {
         let path = self
             .wasm_loader
-            .load_component_source(source.id(), source.source)
+            .load_component_source(source.id, source.source)
             .await?;
         let bytes = tokio::fs::read(&path).await?;
         let component = spin_componentize::componentize_if_necessary(&bytes)?;
@@ -144,7 +165,7 @@ impl WrappedComponentDependencies {
 }
 
 #[async_trait::async_trait]
-impl<'a> spin_compose::ComponentLike for ComponentToValidate<'a> {
+impl<'a> spin_compose::ComponentLike for ComponentSource<'a> {
     type Dependency = WrappedComponentDependency;
 
     fn dependencies(
@@ -176,3 +197,15 @@ impl spin_compose::DependencyLike for WrappedComponentDependency {
         }
     }
 }
+
+fn source_description(source: &spin_manifest::schema::v2::ComponentSource) -> String {
+    match source {
+        spin_manifest::schema::v2::ComponentSource::Local(path) => {
+            format!("file {}", quoted_path(path))
+        }
+        spin_manifest::schema::v2::ComponentSource::Remote { url, .. } => format!("URL {url}"),
+        spin_manifest::schema::v2::ComponentSource::Registry { package, .. } => {
+            format!("package {package}")
+        }
+    }
+}