From be7505525e4f871ff80982784f332669294c38f6 Mon Sep 17 00:00:00 2001 From: Joseph Marrero Corchado Date: Tue, 9 Sep 2025 11:53:31 -0400 Subject: [PATCH] WIP: Use own container-storage for host images if the refspec On install add flag to enable unified-storage Signed-off-by: Joseph Marrero Corchado --- crates/lib/src/cli.rs | 32 ++++++++- crates/lib/src/deploy.rs | 133 ++++++++++++++++++++++++++++++++++---- crates/lib/src/install.rs | 47 +++++++++++++- 3 files changed, 194 insertions(+), 18 deletions(-) diff --git a/crates/lib/src/cli.rs b/crates/lib/src/cli.rs index 3412a6b03..851428f25 100644 --- a/crates/lib/src/cli.rs +++ b/crates/lib/src/cli.rs @@ -928,7 +928,19 @@ async fn upgrade(opts: UpgradeOpts) -> Result<()> { } } } else { - let fetched = crate::deploy::pull(repo, imgref, None, opts.quiet, prog.clone()).await?; + // Check if image exists in bootc storage (/usr/lib/bootc/storage) + let imgstore = sysroot.get_ensure_imgstore()?; + let use_unified = imgstore + .exists(&format!("{imgref:#}")) + .await + .unwrap_or(false); + + let fetched = if use_unified { + crate::deploy::pull_unified(repo, imgref, None, opts.quiet, prog.clone(), sysroot) + .await? + } else { + crate::deploy::pull(repo, imgref, None, opts.quiet, prog.clone()).await? + }; let staged_digest = staged_image.map(|s| s.digest().expect("valid digest in status")); let fetched_digest = &fetched.manifest_digest; tracing::debug!("staged: {staged_digest:?}"); @@ -1056,7 +1068,18 @@ async fn switch(opts: SwitchOpts) -> Result<()> { let new_spec = RequiredHostSpec::from_spec(&new_spec)?; - let fetched = crate::deploy::pull(repo, &target, None, opts.quiet, prog.clone()).await?; + // Check if image exists in bootc storage (/usr/lib/bootc/storage) + let imgstore = sysroot.get_ensure_imgstore()?; + let use_unified = imgstore + .exists(&format!("{target:#}")) + .await + .unwrap_or(false); + + let fetched = if use_unified { + crate::deploy::pull_unified(repo, &target, None, opts.quiet, prog.clone(), sysroot).await? + } else { + crate::deploy::pull(repo, &target, None, opts.quiet, prog.clone()).await? + }; if !opts.retain { // By default, we prune the previous ostree ref so it will go away after later upgrades @@ -1422,7 +1445,10 @@ async fn run_from_opt(opt: Opt) -> Result<()> { let mut w = SplitStreamWriter::new(&cfs, None, Some(testdata_digest)); w.write_inline(testdata); let object = cfs.write_stream(w, Some("testobject"))?.to_hex(); - assert_eq!(object, "5d94ceb0b2bb3a78237e0a74bc030a262239ab5f47754a5eb2e42941056b64cb21035d64a8f7c2f156e34b820802fa51884de2b1f7dc3a41b9878fc543cd9b07"); + assert_eq!( + object, + "5d94ceb0b2bb3a78237e0a74bc030a262239ab5f47754a5eb2e42941056b64cb21035d64a8f7c2f156e34b820802fa51884de2b1f7dc3a41b9878fc543cd9b07" + ); Ok(()) } // We don't depend on fsverity-utils today, so re-expose some helpful CLI tools. diff --git a/crates/lib/src/deploy.rs b/crates/lib/src/deploy.rs index 7b7ff59e1..a0c1231e8 100644 --- a/crates/lib/src/deploy.rs +++ b/crates/lib/src/deploy.rs @@ -380,6 +380,112 @@ pub(crate) async fn prepare_for_pull( Ok(PreparedPullResult::Ready(Box::new(prepared_image))) } +/// Unified approach: Use bootc's CStorage to pull the image, then prepare from containers-storage. +/// This reuses the same infrastructure as LBIs. +pub(crate) async fn prepare_for_pull_unified( + repo: &ostree::Repo, + imgref: &ImageReference, + target_imgref: Option<&OstreeImageReference>, + store: &Storage, +) -> Result { + // Get or initialize the bootc container storage (same as used for LBIs) + let imgstore = store.get_ensure_imgstore()?; + + let image_ref_str = format!("{imgref:#}"); + + // Log the original transport being used for the pull + tracing::info!( + "Unified pull: pulling from transport '{}' to bootc storage", + &imgref.transport + ); + + // Pull the image to bootc storage using the same method as LBIs + imgstore + .pull(&image_ref_str, crate::podstorage::PullMode::Always) + .await?; + + // Now create a containers-storage reference to read from bootc storage + tracing::info!("Unified pull: now importing from containers-storage transport"); + let containers_storage_imgref = ImageReference { + transport: "containers-storage".to_string(), + image: imgref.image.clone(), + signature: imgref.signature.clone(), + }; + let ostree_imgref = OstreeImageReference::from(containers_storage_imgref); + + // Use the standard preparation flow but reading from containers-storage + let mut imp = new_importer(repo, &ostree_imgref).await?; + if let Some(target) = target_imgref { + imp.set_target(target); + } + let prep = match imp.prepare().await? { + PrepareResult::AlreadyPresent(c) => { + println!("No changes in {imgref:#} => {}", c.manifest_digest); + return Ok(PreparedPullResult::AlreadyPresent(Box::new((*c).into()))); + } + PrepareResult::Ready(p) => p, + }; + check_bootc_label(&prep.config); + if let Some(warning) = prep.deprecated_warning() { + ostree_ext::cli::print_deprecated_warning(warning).await; + } + ostree_ext::cli::print_layer_status(&prep); + let layers_to_fetch = prep.layers_to_fetch().collect::>>()?; + + // Log that we're importing a new image from containers-storage + const PULLING_NEW_IMAGE_ID: &str = "6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f1a0"; + tracing::info!( + message_id = PULLING_NEW_IMAGE_ID, + bootc.image.reference = &imgref.image, + bootc.image.transport = "containers-storage", + bootc.original_transport = &imgref.transport, + bootc.status = "importing_from_storage", + "Importing image from bootc storage: {}", + ostree_imgref + ); + + let prepared_image = PreparedImportMeta { + imp, + n_layers_to_fetch: layers_to_fetch.len(), + layers_total: prep.all_layers().count(), + bytes_to_fetch: layers_to_fetch.iter().map(|(l, _)| l.layer.size()).sum(), + bytes_total: prep.all_layers().map(|l| l.layer.size()).sum(), + digest: prep.manifest_digest.clone(), + prep, + }; + + Ok(PreparedPullResult::Ready(Box::new(prepared_image))) +} + +/// Unified pull: Use podman to pull to containers-storage, then read from there +pub(crate) async fn pull_unified( + repo: &ostree::Repo, + imgref: &ImageReference, + target_imgref: Option<&OstreeImageReference>, + quiet: bool, + prog: ProgressWriter, + store: &Storage, +) -> Result> { + match prepare_for_pull_unified(repo, imgref, target_imgref, store).await? { + PreparedPullResult::AlreadyPresent(existing) => { + // Log that the image was already present (Debug level since it's not actionable) + const IMAGE_ALREADY_PRESENT_ID: &str = "5c4d3e2f1a0b9c8d7e6f5a4b3c2d1e0f9"; + tracing::debug!( + message_id = IMAGE_ALREADY_PRESENT_ID, + bootc.image.reference = &imgref.image, + bootc.image.transport = &imgref.transport, + bootc.status = "already_present", + "Image already present: {}", + imgref + ); + Ok(existing) + } + PreparedPullResult::Ready(prepared_image_meta) => { + pull_from_prepared(imgref, quiet, prog, *prepared_image_meta).await + } + } +} + #[context("Pulling")] pub(crate) async fn pull_from_prepared( imgref: &ImageReference, @@ -429,18 +535,21 @@ pub(crate) async fn pull_from_prepared( let imgref_canonicalized = imgref.clone().canonicalize()?; tracing::debug!("Canonicalized image reference: {imgref_canonicalized:#}"); - // Log successful import completion - const IMPORT_COMPLETE_JOURNAL_ID: &str = "4d3e2f1a0b9c8d7e6f5a4b3c2d1e0f9a8"; - - tracing::info!( - message_id = IMPORT_COMPLETE_JOURNAL_ID, - bootc.image.reference = &imgref.image, - bootc.image.transport = &imgref.transport, - bootc.manifest_digest = import.manifest_digest.as_ref(), - bootc.ostree_commit = &import.merge_commit, - "Successfully imported image: {}", - imgref - ); + // Log successful import completion (skip if using unified storage to avoid double logging) + let is_unified_path = imgref.transport == "containers-storage"; + if !is_unified_path { + const IMPORT_COMPLETE_JOURNAL_ID: &str = "4d3e2f1a0b9c8d7e6f5a4b3c2d1e0f9a8"; + + tracing::info!( + message_id = IMPORT_COMPLETE_JOURNAL_ID, + bootc.image.reference = &imgref.image, + bootc.image.transport = &imgref.transport, + bootc.manifest_digest = import.manifest_digest.as_ref(), + bootc.ostree_commit = &import.merge_commit, + "Successfully imported image: {}", + imgref + ); + } if let Some(msg) = ostree_container::store::image_filtered_content_warning(&import.filtered_files) diff --git a/crates/lib/src/install.rs b/crates/lib/src/install.rs index defa9d19c..164051be1 100644 --- a/crates/lib/src/install.rs +++ b/crates/lib/src/install.rs @@ -166,6 +166,15 @@ pub(crate) struct InstallTargetOpts { #[clap(long)] #[serde(default)] pub(crate) skip_fetch_check: bool, + + /// Use unified storage path to pull images (experimental) + /// + /// When enabled, this uses bootc's container storage (/usr/lib/bootc/storage) to pull + /// the image first, then imports it from there. This is the same approach used for + /// logically bound images. + #[clap(long)] + #[serde(default)] + pub(crate) unified_storage_exp: bool, } #[derive(clap::Args, Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -426,6 +435,7 @@ pub(crate) struct State { pub(crate) selinux_state: SELinuxFinalState, #[allow(dead_code)] pub(crate) config_opts: InstallConfigOpts, + pub(crate) target_opts: InstallTargetOpts, pub(crate) target_imgref: ostree_container::OstreeImageReference, #[allow(dead_code)] pub(crate) prepareroot_config: HashMap, @@ -787,6 +797,7 @@ async fn install_container( state: &State, root_setup: &RootSetup, sysroot: &ostree::Sysroot, + storage: &Storage, has_ostree: bool, ) -> Result<(ostree::Deployment, InstallAleph)> { let sepolicy = state.load_policy()?; @@ -826,9 +837,38 @@ async fn install_container( let repo = &sysroot.repo(); repo.set_disable_fsync(true); - let pulled_image = match prepare_for_pull(repo, &spec_imgref, Some(&state.target_imgref)) + // Determine whether to use unified storage path + let use_unified = if state.target_opts.unified_storage_exp { + // Explicit flag always uses unified path + true + } else { + // Auto-detect: check if image exists in bootc storage (same as upgrade/switch) + let imgstore = storage.get_ensure_imgstore()?; + imgstore + .exists(&format!("{spec_imgref:#}")) + .await + .unwrap_or_else(|e| { + tracing::warn!( + "Failed to check bootc storage for image: {e}; falling back to standard pull" + ); + false + }) + }; + + let prepared = if use_unified { + tracing::info!("Using unified storage path for installation"); + crate::deploy::prepare_for_pull_unified( + repo, + &spec_imgref, + Some(&state.target_imgref), + storage, + ) .await? - { + } else { + prepare_for_pull(repo, &spec_imgref, Some(&state.target_imgref)).await? + }; + + let pulled_image = match prepared { PreparedPullResult::AlreadyPresent(existing) => existing, PreparedPullResult::Ready(image_meta) => { check_disk_space(root_setup.physical_root.as_fd(), &image_meta, &spec_imgref)?; @@ -1364,6 +1404,7 @@ async fn prepare_install( selinux_state, source, config_opts, + target_opts, target_imgref, install_config, prepareroot_config, @@ -1395,7 +1436,7 @@ async fn install_with_sysroot( // And actually set up the container in that root, returning a deployment and // the aleph state (see below). - let (deployment, aleph) = install_container(state, rootfs, ostree, has_ostree).await?; + let (deployment, aleph) = install_container(state, rootfs, ostree, storage, has_ostree).await?; // Write the aleph data that captures the system state at the time of provisioning for aid in future debugging. aleph.write_to(&rootfs.physical_root)?;