From f2b0708f695ec3f384e6c8651dca530dccf27eca Mon Sep 17 00:00:00 2001 From: Justin Schneck Date: Tue, 23 Dec 2025 15:36:17 -0500 Subject: [PATCH 01/10] bump to 0.19.2-dev --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5b8bb17..82f7bce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -130,7 +130,7 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "avocado-cli" -version = "0.19.1" +version = "0.19.2-dev" dependencies = [ "anyhow", "base64", diff --git a/Cargo.toml b/Cargo.toml index 5fd1114..51f6455 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "avocado-cli" -version = "0.19.1" +version = "0.19.2-dev" edition = "2021" description = "Command line interface for Avocado." authors = ["Avocado"] From 252634ed58d816337512e29413e3ec907c09dfab Mon Sep 17 00:00:00 2001 From: Justin Schneck Date: Tue, 23 Dec 2025 15:36:38 -0500 Subject: [PATCH 02/10] add warning output for config parsing errors --- src/utils/interpolation/mod.rs | 111 ++++++++++++++++++++++++++++----- tests/interpolation.rs | 23 ++++++- 2 files changed, 116 insertions(+), 18 deletions(-) diff --git a/src/utils/interpolation/mod.rs b/src/utils/interpolation/mod.rs index af082d8..19f5d90 100644 --- a/src/utils/interpolation/mod.rs +++ b/src/utils/interpolation/mod.rs @@ -103,7 +103,9 @@ pub fn interpolate_config(yaml_value: &mut Value, cli_target: Option<&str>) -> R let root = yaml_value.clone(); // Create a new resolving stack for each iteration let mut resolving_stack = HashSet::new(); - changed = interpolate_value(yaml_value, &root, cli_target, &mut resolving_stack)?; + // Start with empty path at root level + let path: Vec = Vec::new(); + changed = interpolate_value(yaml_value, &root, cli_target, &mut resolving_stack, &path)?; iteration += 1; } @@ -116,6 +118,31 @@ pub fn interpolate_config(yaml_value: &mut Value, cli_target: Option<&str>) -> R Ok(()) } +/// Represents where in the YAML structure a template was found. +#[derive(Clone, Debug)] +enum YamlLocation { + /// Template found in a mapping key + Key(String), + /// Template found in a value + Value, +} + +/// Format a YAML path for display in error messages. +fn format_yaml_path(path: &[String], location: &YamlLocation) -> String { + if path.is_empty() { + match location { + YamlLocation::Key(key) => format!("key \"{}\"", key), + YamlLocation::Value => "root value".to_string(), + } + } else { + let path_str = path.join("."); + match location { + YamlLocation::Key(key) => format!("{}.\"{}\" (key)", path_str, key), + YamlLocation::Value => path_str, + } + } +} + /// Recursively interpolate a single value. /// /// # Arguments @@ -123,6 +150,7 @@ pub fn interpolate_config(yaml_value: &mut Value, cli_target: Option<&str>) -> R /// * `root` - The root YAML value for config references /// * `cli_target` - Optional CLI target value /// * `resolving_stack` - Set of templates currently being resolved (for cycle detection) +/// * `path` - The current YAML path for error messages /// /// # Returns /// Result with a boolean indicating if any changes were made @@ -131,12 +159,16 @@ fn interpolate_value( root: &Value, cli_target: Option<&str>, resolving_stack: &mut HashSet, + path: &[String], ) -> Result { let mut changed = false; match value { Value::String(s) => { - if let Some(new_value) = interpolate_string(s, root, cli_target, resolving_stack)? { + let location = YamlLocation::Value; + if let Some(new_value) = + interpolate_string(s, root, cli_target, resolving_stack, path, &location)? + { *s = new_value; changed = true; } @@ -148,8 +180,9 @@ fn interpolate_value( for (k, v) in map.iter() { if let Value::String(key_str) = k { + let location = YamlLocation::Key(key_str.clone()); if let Some(new_key) = - interpolate_string(key_str, root, cli_target, resolving_stack)? + interpolate_string(key_str, root, cli_target, resolving_stack, path, &location)? { keys_to_replace.push((k.clone(), Value::String(new_key), v.clone())); } @@ -164,15 +197,23 @@ fn interpolate_value( } // Then interpolate all values (including newly inserted ones) - for (_, v) in map.iter_mut() { - if interpolate_value(v, root, cli_target, resolving_stack)? { + for (k, v) in map.iter_mut() { + let key_str = match k { + Value::String(s) => s.clone(), + _ => format!("{:?}", k), + }; + let mut child_path = path.to_vec(); + child_path.push(key_str); + if interpolate_value(v, root, cli_target, resolving_stack, &child_path)? { changed = true; } } } Value::Sequence(seq) => { - for item in seq.iter_mut() { - if interpolate_value(item, root, cli_target, resolving_stack)? { + for (idx, item) in seq.iter_mut().enumerate() { + let mut child_path = path.to_vec(); + child_path.push(format!("[{}]", idx)); + if interpolate_value(item, root, cli_target, resolving_stack, &child_path)? { changed = true; } } @@ -192,6 +233,8 @@ fn interpolate_value( /// * `root` - The root YAML value for config references /// * `cli_target` - Optional CLI target value /// * `resolving_stack` - Set of templates currently being resolved (for cycle detection) +/// * `path` - The current YAML path for error messages +/// * `location` - Whether this is a key or value /// /// # Returns /// Result with Option - Some(new_string) if changes were made, None if no templates found @@ -200,6 +243,8 @@ fn interpolate_string( root: &Value, cli_target: Option<&str>, resolving_stack: &mut HashSet, + path: &[String], + location: &YamlLocation, ) -> Result> { // Regex to match {{ ... }} templates let re = Regex::new(r"\{\{\s*([^}]+)\s*\}\}").unwrap(); @@ -216,9 +261,20 @@ fn interpolate_string( let full_match = capture.get(0).unwrap().as_str(); let template = capture.get(1).unwrap().as_str().trim(); - if let Some(replacement) = resolve_template(template, root, cli_target, resolving_stack)? { - result = result.replace(full_match, &replacement); - any_replaced = true; + match resolve_template(template, root, cli_target, resolving_stack) { + Ok(Some(replacement)) => { + result = result.replace(full_match, &replacement); + any_replaced = true; + } + Ok(None) => {} + Err(e) => { + // Add context about where in the YAML this template was found + let yaml_location = format_yaml_path(path, location); + return Err(e.context(format!( + "in template '{{{{ {} }}}}' at {}", + template, yaml_location + ))); + } } } @@ -385,6 +441,14 @@ reference: "{{ config.nested.deep.value }}" ); } + /// Helper to get the full error chain as a string for assertions. + fn error_chain_string(err: &anyhow::Error) -> String { + err.chain() + .map(|e| e.to_string()) + .collect::>() + .join(": ") + } + #[test] fn test_missing_config_path() { let mut config = parse_yaml( @@ -393,10 +457,22 @@ reference: "{{ config.nonexistent.path }}" "#, ); - // Should return an error + // Should return an error with location context let result = interpolate_config(&mut config, None); assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("not found")); + let err = result.unwrap_err(); + let full_error = error_chain_string(&err); + // Should include both the location and the "not found" message + assert!( + full_error.contains("not found"), + "Expected 'not found' in error, got: {}", + full_error + ); + assert!( + full_error.contains("reference"), + "Expected 'reference' location in error, got: {}", + full_error + ); } #[test] @@ -691,10 +767,13 @@ key: "{{ unknown.value }}" let result = interpolate_config(&mut config, None); assert!(result.is_err()); - assert!(result - .unwrap_err() - .to_string() - .contains("Unknown template context")); + let err = result.unwrap_err(); + let full_error = error_chain_string(&err); + assert!( + full_error.contains("Unknown template context"), + "Expected 'Unknown template context' in error, got: {}", + full_error + ); } #[test] diff --git a/tests/interpolation.rs b/tests/interpolation.rs index 904f5a0..6b075a0 100644 --- a/tests/interpolation.rs +++ b/tests/interpolation.rs @@ -148,6 +148,14 @@ runtime: assert_eq!(target, "{{ avocado.target }}"); } +/// Helper to get the full error chain as a string for assertions. +fn error_chain_string(err: &anyhow::Error) -> String { + err.chain() + .map(|e| e.to_string()) + .collect::>() + .join(": ") +} + #[test] fn test_missing_config_path_error() { let test_yaml = r#" @@ -159,8 +167,19 @@ reference: "{{ config.nonexistent.path }}" let result = avocado_cli::utils::interpolation::interpolate_config(&mut parsed, None); assert!(result.is_err()); - let error_msg = result.unwrap_err().to_string(); - assert!(error_msg.contains("not found")); + let err = result.unwrap_err(); + let full_error = error_chain_string(&err); + // Should contain both the location and the "not found" message + assert!( + full_error.contains("not found"), + "Expected 'not found' in error, got: {}", + full_error + ); + assert!( + full_error.contains("reference"), + "Expected 'reference' location in error, got: {}", + full_error + ); } #[test] From df3836f365a1a1bd1d30afab6ec88e24204316a8 Mon Sep 17 00:00:00 2001 From: Justin Schneck Date: Thu, 25 Dec 2025 16:15:07 -0500 Subject: [PATCH 03/10] preserve file attributes on extension overlay merging. --- src/commands/ext/build.rs | 16 ++++++++-------- src/utils/interpolation/mod.rs | 11 ++++++++--- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/commands/ext/build.rs b/src/commands/ext/build.rs index 6b44375..e2b159a 100644 --- a/src/commands/ext/build.rs +++ b/src/commands/ext/build.rs @@ -616,9 +616,9 @@ fi # Copy overlay directory to extension sysroot (opaque mode) if [ -d "/opt/src/{}" ]; then echo "Copying overlay directory '{}' to extension sysroot (opaque mode)" - # Use cp -r to replace directory contents completely - cp -r /opt/src/{}/* "$AVOCADO_EXT_SYSROOTS/{}/" - # Fix ownership to root:root for copied overlay files only + # Use cp -a to replace directory contents completely while preserving permissions + cp -a /opt/src/{}/* "$AVOCADO_EXT_SYSROOTS/{}/" + # Fix ownership to root:root for copied overlay files only (permissions are preserved) echo "Setting ownership to root:root for overlay files" find "/opt/src/{}" -mindepth 1 | while IFS= read -r srcpath; do relpath="$(echo "$srcpath" | sed "s|^/opt/src/{}||" | sed "s|^/||")" @@ -789,9 +789,9 @@ fi # Copy overlay directory to extension sysroot (opaque mode) if [ -d "/opt/src/{}" ]; then echo "Copying overlay directory '{}' to extension sysroot (opaque mode)" - # Use cp -r to replace directory contents completely - cp -r /opt/src/{}/* "$AVOCADO_EXT_SYSROOTS/{}/" - # Fix ownership to root:root for copied overlay files only + # Use cp -a to replace directory contents completely while preserving permissions + cp -a /opt/src/{}/* "$AVOCADO_EXT_SYSROOTS/{}/" + # Fix ownership to root:root for copied overlay files only (permissions are preserved) echo "Setting ownership to root:root for overlay files" find "/opt/src/{}" -mindepth 1 | while IFS= read -r srcpath; do relpath="$(echo "$srcpath" | sed "s|^/opt/src/{}||" | sed "s|^/||")" @@ -1970,7 +1970,7 @@ mod tests { assert!(script.contains( "echo \"Copying overlay directory 'peridio' to extension sysroot (opaque mode)\"" )); - assert!(script.contains("cp -r /opt/src/peridio/* \"$AVOCADO_EXT_SYSROOTS/opaque-ext/\"")); + assert!(script.contains("cp -a /opt/src/peridio/* \"$AVOCADO_EXT_SYSROOTS/opaque-ext/\"")); assert!(script.contains("echo \"Setting ownership to root:root for overlay files\"")); assert!(script.contains("find \"/opt/src/peridio\" -mindepth 1")); assert!(script.contains("echo \"Error: Overlay directory 'peridio' not found in source\"")); @@ -2011,7 +2011,7 @@ mod tests { assert!(script.contains( "echo \"Copying overlay directory 'peridio' to extension sysroot (opaque mode)\"" )); - assert!(script.contains("cp -r /opt/src/peridio/* \"$AVOCADO_EXT_SYSROOTS/opaque-ext/\"")); + assert!(script.contains("cp -a /opt/src/peridio/* \"$AVOCADO_EXT_SYSROOTS/opaque-ext/\"")); assert!(script.contains("echo \"Setting ownership to root:root for overlay files\"")); assert!(script.contains("find \"/opt/src/peridio\" -mindepth 1")); assert!(script.contains("echo \"Error: Overlay directory 'peridio' not found in source\"")); diff --git a/src/utils/interpolation/mod.rs b/src/utils/interpolation/mod.rs index 19f5d90..2a05c5a 100644 --- a/src/utils/interpolation/mod.rs +++ b/src/utils/interpolation/mod.rs @@ -181,9 +181,14 @@ fn interpolate_value( for (k, v) in map.iter() { if let Value::String(key_str) = k { let location = YamlLocation::Key(key_str.clone()); - if let Some(new_key) = - interpolate_string(key_str, root, cli_target, resolving_stack, path, &location)? - { + if let Some(new_key) = interpolate_string( + key_str, + root, + cli_target, + resolving_stack, + path, + &location, + )? { keys_to_replace.push((k.clone(), Value::String(new_key), v.clone())); } } From 7ab5348ffd423b51888b2d83101b9e56785e612c Mon Sep 17 00:00:00 2001 From: Justin Schneck Date: Thu, 25 Dec 2025 16:50:29 -0500 Subject: [PATCH 04/10] move target-sysroot install to sdk install --- src/commands/sdk/install.rs | 149 ++++++++++++++++++------------------ src/utils/container.rs | 9 --- 2 files changed, 73 insertions(+), 85 deletions(-) diff --git a/src/commands/sdk/install.rs b/src/commands/sdk/install.rs index 1f202e2..db3f7ac 100644 --- a/src/commands/sdk/install.rs +++ b/src/commands/sdk/install.rs @@ -97,9 +97,6 @@ impl SdkInstallCommand { ) .with_context(|| "Failed to parse extension SDK dependencies")?; - // Get compile section dependencies - let compile_dependencies = config.get_compile_dependencies(); - // Get repo_url and repo_release from config let repo_url = config.get_sdk_repo_url(); let repo_release = config.get_sdk_repo_release(); @@ -180,32 +177,50 @@ $DNF_SDK_HOST \ print_success("No dependencies configured.", OutputLevel::Normal); } - // Install compile section dependencies (into target-dev sysroot) + // Install target-sysroot if there are any sdk.compile dependencies + // This aggregates all dependencies from all compile sections (main config + external extensions) + let compile_dependencies = config.get_compile_dependencies(); if !compile_dependencies.is_empty() { - print_info("Installing SDK compile dependencies.", OutputLevel::Normal); - let total = compile_dependencies.len(); - - for (index, (section_name, dependencies)) in compile_dependencies.iter().enumerate() { - let compile_packages = self.build_package_list(dependencies); - - if !compile_packages.is_empty() { - let installroot = "${AVOCADO_SDK_PREFIX}/target-sysroot"; - let yes = if self.force { "-y" } else { "" }; - let dnf_args_str = if let Some(args) = &self.dnf_args { - format!(" {} ", args.join(" ")) - } else { - String::new() - }; - // For compile dependencies (target-dev sysroot), we: - // - Use $DNF_NO_SCRIPTS to skip scriptlet execution (not needed for cross-compilation) - // - Always disable weak dependencies (dev packages don't need them) - // - Skip documentation (not needed in dev sysroot) - let command = format!( - r#" + // Aggregate all compile dependencies into a single list + let mut all_compile_packages: Vec = Vec::new(); + for dependencies in compile_dependencies.values() { + let packages = self.build_package_list(dependencies); + all_compile_packages.extend(packages); + } + + // Deduplicate packages + all_compile_packages.sort(); + all_compile_packages.dedup(); + + print_info( + &format!( + "Installing target-sysroot with {} compile dependencies.", + all_compile_packages.len() + ), + OutputLevel::Normal, + ); + + let yes = if self.force { "-y" } else { "" }; + let dnf_args_str = if let Some(args) = &self.dnf_args { + format!(" {} ", args.join(" ")) + } else { + String::new() + }; + + // Build the target-sysroot package spec with version from distro.version + let target_sysroot_pkg = if let Some(version) = config.get_distro_version() { + format!("avocado-sdk-target-sysroot-{}", version) + } else { + "avocado-sdk-target-sysroot".to_string() + }; + + // Install the target-sysroot with packagegroup-core-standalone-sdk-target plus compile deps + let command = format!( + r#" RPM_ETCCONFIGDIR=$DNF_SDK_TARGET_PREFIX \ $DNF_SDK_HOST $DNF_NO_SCRIPTS \ --setopt=sslcacert=${{SSL_CERT_FILE}} \ - --installroot {} \ + --installroot ${{AVOCADO_SDK_PREFIX}}/target-sysroot \ --setopt=install_weak_deps=0 \ --nodocs \ $DNF_SDK_TARGET_REPO_CONF \ @@ -213,60 +228,42 @@ $DNF_SDK_HOST $DNF_NO_SCRIPTS \ {} \ install \ {} \ + {} \ {} "#, - installroot, - dnf_args_str, - yes, - compile_packages.join(" ") - ); - - print_info( - &format!( - "Installing ({}/{}) compile dependencies for section '{}'", - index + 1, - total, - section_name - ), - OutputLevel::Normal, - ); - - // Use the container helper's run_in_container method with target-dev installroot - let run_config = RunConfig { - container_image: container_image.to_string(), - target: target.clone(), - command, - verbose: self.verbose, - source_environment: true, - interactive: !self.force, - repo_url: repo_url.clone(), - repo_release: repo_release.clone(), - container_args: merged_container_args.clone(), - dnf_args: self.dnf_args.clone(), - disable_weak_dependencies: config.get_sdk_disable_weak_dependencies(), - ..Default::default() - }; - let install_success = container_helper.run_in_container(run_config).await?; - - if !install_success { - return Err(anyhow::anyhow!( - "Failed to install dependencies for compile section '{section_name}'." - )); - } - } else { - print_info( - &format!( - "({}/{}) [{}] No dependencies configured.", - index + 1, - total, - section_name - ), - OutputLevel::Normal, - ); - } - } + dnf_args_str, + yes, + target_sysroot_pkg, + all_compile_packages.join(" ") + ); + + let run_config = RunConfig { + container_image: container_image.to_string(), + target: target.clone(), + command, + verbose: self.verbose, + source_environment: true, + interactive: !self.force, + repo_url: repo_url.clone(), + repo_release: repo_release.clone(), + container_args: merged_container_args.clone(), + dnf_args: self.dnf_args.clone(), + disable_weak_dependencies: config.get_sdk_disable_weak_dependencies(), + ..Default::default() + }; + + let install_success = container_helper.run_in_container(run_config).await?; - print_success("Installed SDK compile dependencies.", OutputLevel::Normal); + if install_success { + print_success( + "Installed target-sysroot with compile dependencies.", + OutputLevel::Normal, + ); + } else { + return Err(anyhow::anyhow!( + "Failed to install target-sysroot with compile dependencies." + )); + } } // Write SDK install stamp (unless --no-stamps) diff --git a/src/utils/container.rs b/src/utils/container.rs index 09e00a6..96aa497 100644 --- a/src/utils/container.rs +++ b/src/utils/container.rs @@ -729,15 +729,6 @@ MACROS_EOF RPM_ETCCONFIGDIR="$DNF_SDK_TARGET_PREFIX" \ $DNF_SDK_HOST $DNF_NO_SCRIPTS $DNF_SDK_TARGET_REPO_CONF \ -y --installroot $AVOCADO_PREFIX/rootfs install avocado-pkg-rootfs - - echo "[INFO] Installing SDK target sysroot." - RPM_ETCCONFIGDIR=$DNF_SDK_TARGET_PREFIX \ - $DNF_SDK_HOST $DNF_NO_SCRIPTS \ - $DNF_SDK_TARGET_REPO_CONF \ - -y \ - --installroot ${AVOCADO_SDK_PREFIX}/target-sysroot \ - install \ - packagegroup-core-standalone-sdk-target fi "#); } From 50db076efa290cc1823a01a73facdc5ac6c3f6a0 Mon Sep 17 00:00:00 2001 From: Justin Schneck Date: Thu, 25 Dec 2025 16:56:09 -0500 Subject: [PATCH 05/10] move rootfs install to sdk install --- src/commands/sdk/install.rs | 54 +++++++++++++++++++++++++++++++++++++ src/utils/container.rs | 5 ---- 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/src/commands/sdk/install.rs b/src/commands/sdk/install.rs index db3f7ac..abc2da3 100644 --- a/src/commands/sdk/install.rs +++ b/src/commands/sdk/install.rs @@ -177,6 +177,60 @@ $DNF_SDK_HOST \ print_success("No dependencies configured.", OutputLevel::Normal); } + // Install rootfs sysroot with version from distro.version + print_info("Installing rootfs sysroot.", OutputLevel::Normal); + + let rootfs_pkg = if let Some(version) = config.get_distro_version() { + format!("avocado-pkg-rootfs-{}", version) + } else { + "avocado-pkg-rootfs".to_string() + }; + + let yes = if self.force { "-y" } else { "" }; + let dnf_args_str = if let Some(args) = &self.dnf_args { + format!(" {} ", args.join(" ")) + } else { + String::new() + }; + + let rootfs_command = format!( + r#" +RPM_ETCCONFIGDIR=$DNF_SDK_TARGET_PREFIX \ +$DNF_SDK_HOST $DNF_NO_SCRIPTS \ + --setopt=sslcacert=${{SSL_CERT_FILE}} \ + --installroot ${{AVOCADO_PREFIX}}/rootfs \ + $DNF_SDK_TARGET_REPO_CONF \ + {} \ + install \ + {} \ + {} +"#, + dnf_args_str, yes, rootfs_pkg + ); + + let run_config = RunConfig { + container_image: container_image.to_string(), + target: target.clone(), + command: rootfs_command, + verbose: self.verbose, + source_environment: true, + interactive: !self.force, + repo_url: repo_url.clone(), + repo_release: repo_release.clone(), + container_args: merged_container_args.clone(), + dnf_args: self.dnf_args.clone(), + disable_weak_dependencies: config.get_sdk_disable_weak_dependencies(), + ..Default::default() + }; + + let rootfs_success = container_helper.run_in_container(run_config).await?; + + if rootfs_success { + print_success("Installed rootfs sysroot.", OutputLevel::Normal); + } else { + return Err(anyhow::anyhow!("Failed to install rootfs sysroot.")); + } + // Install target-sysroot if there are any sdk.compile dependencies // This aggregates all dependencies from all compile sections (main config + external extensions) let compile_dependencies = config.get_compile_dependencies(); diff --git a/src/utils/container.rs b/src/utils/container.rs index 96aa497..1b7c148 100644 --- a/src/utils/container.rs +++ b/src/utils/container.rs @@ -724,11 +724,6 @@ MACROS_EOF RPM_CONFIGDIR="$AVOCADO_SDK_PREFIX/usr/lib/rpm" \ RPM_ETCCONFIGDIR="$AVOCADO_SDK_PREFIX" \ $DNF_SDK_HOST $DNF_NO_SCRIPTS $DNF_SDK_HOST_OPTS $DNF_SDK_REPO_CONF -y install avocado-sdk-bootstrap - - echo "[INFO] Installing rootfs sysroot." - RPM_ETCCONFIGDIR="$DNF_SDK_TARGET_PREFIX" \ - $DNF_SDK_HOST $DNF_NO_SCRIPTS $DNF_SDK_TARGET_REPO_CONF \ - -y --installroot $AVOCADO_PREFIX/rootfs install avocado-pkg-rootfs fi "#); } From 5fd174cb1582463dea45495bce9fc856f177fc37 Mon Sep 17 00:00:00 2001 From: Justin Schneck Date: Thu, 25 Dec 2025 17:47:25 -0500 Subject: [PATCH 06/10] move sdk bootstrap to sdk install --- src/commands/sdk/install.rs | 205 +++++++++++++++++++++++++++++++----- src/utils/container.rs | 11 -- 2 files changed, 181 insertions(+), 35 deletions(-) diff --git a/src/commands/sdk/install.rs b/src/commands/sdk/install.rs index abc2da3..be87ff7 100644 --- a/src/commands/sdk/install.rs +++ b/src/commands/sdk/install.rs @@ -105,6 +105,176 @@ impl SdkInstallCommand { let container_helper = SdkContainer::from_config(&self.config_path, config)?.verbose(self.verbose); + // Install avocado-sdk-{target} with version from distro.version + print_info( + &format!("Installing SDK for target '{}'.", target), + OutputLevel::Normal, + ); + + let sdk_target_pkg = if let Some(version) = config.get_distro_version() { + format!("avocado-sdk-{}-{}", target, version) + } else { + format!("avocado-sdk-{}", target) + }; + + let sdk_target_command = format!( + r#" +RPM_CONFIGDIR=$AVOCADO_SDK_PREFIX/usr/lib/rpm \ +RPM_ETCCONFIGDIR=$AVOCADO_SDK_PREFIX \ +$DNF_SDK_HOST $DNF_NO_SCRIPTS \ + $DNF_SDK_HOST_OPTS \ + $DNF_SDK_HOST_REPO_CONF \ + -y \ + install \ + {} +"#, + sdk_target_pkg + ); + + let run_config = RunConfig { + container_image: container_image.to_string(), + target: target.clone(), + command: sdk_target_command, + verbose: self.verbose, + source_environment: true, + interactive: false, + repo_url: repo_url.clone(), + repo_release: repo_release.clone(), + container_args: merged_container_args.clone(), + dnf_args: self.dnf_args.clone(), + disable_weak_dependencies: config.get_sdk_disable_weak_dependencies(), + ..Default::default() + }; + + let sdk_target_success = container_helper.run_in_container(run_config).await?; + + if sdk_target_success { + print_success( + &format!("Installed SDK for target '{}'.", target), + OutputLevel::Normal, + ); + } else { + return Err(anyhow::anyhow!( + "Failed to install SDK for target '{}'.", + target + )); + } + + // Run check-update to refresh metadata + let check_update_command = r#" +RPM_CONFIGDIR=$AVOCADO_SDK_PREFIX/usr/lib/rpm \ +RPM_ETCCONFIGDIR=$AVOCADO_SDK_PREFIX \ +$DNF_SDK_HOST \ + $DNF_SDK_HOST_OPTS \ + $DNF_SDK_REPO_CONF \ + check-update || true +"#; + + let run_config = RunConfig { + container_image: container_image.to_string(), + target: target.clone(), + command: check_update_command.to_string(), + verbose: self.verbose, + source_environment: true, + interactive: false, + repo_url: repo_url.clone(), + repo_release: repo_release.clone(), + container_args: merged_container_args.clone(), + dnf_args: self.dnf_args.clone(), + disable_weak_dependencies: config.get_sdk_disable_weak_dependencies(), + ..Default::default() + }; + + container_helper.run_in_container(run_config).await?; + + // Install avocado-sdk-bootstrap with version from distro.version + print_info("Installing SDK bootstrap.", OutputLevel::Normal); + + let bootstrap_pkg = if let Some(version) = config.get_distro_version() { + format!("avocado-sdk-bootstrap-{}", version) + } else { + "avocado-sdk-bootstrap".to_string() + }; + + let bootstrap_command = format!( + r#" +RPM_CONFIGDIR=$AVOCADO_SDK_PREFIX/usr/lib/rpm \ +RPM_ETCCONFIGDIR=$AVOCADO_SDK_PREFIX \ +$DNF_SDK_HOST $DNF_NO_SCRIPTS \ + $DNF_SDK_HOST_OPTS \ + $DNF_SDK_REPO_CONF \ + -y \ + install \ + {} +"#, + bootstrap_pkg + ); + + let run_config = RunConfig { + container_image: container_image.to_string(), + target: target.clone(), + command: bootstrap_command, + verbose: self.verbose, + source_environment: true, + interactive: false, + repo_url: repo_url.clone(), + repo_release: repo_release.clone(), + container_args: merged_container_args.clone(), + dnf_args: self.dnf_args.clone(), + disable_weak_dependencies: config.get_sdk_disable_weak_dependencies(), + ..Default::default() + }; + + let bootstrap_success = container_helper.run_in_container(run_config).await?; + + if bootstrap_success { + print_success("Installed SDK bootstrap.", OutputLevel::Normal); + } else { + return Err(anyhow::anyhow!("Failed to install SDK bootstrap.")); + } + + // After bootstrap, source environment-setup and configure SSL certs for subsequent commands + if self.verbose { + print_info( + "Configuring SDK environment after bootstrap.", + OutputLevel::Normal, + ); + } + + let env_setup_command = r#" +# Source the environment setup if it exists +if [ -f "${AVOCADO_SDK_PREFIX}/environment-setup" ]; then + source "${AVOCADO_SDK_PREFIX}/environment-setup" + echo "[INFO] Sourced SDK environment setup." +fi + +# Add SSL certificate path to DNF options and CURL if it exists +if [ -f "${AVOCADO_SDK_PREFIX}/etc/ssl/certs/ca-certificates.crt" ]; then + export DNF_SDK_HOST_OPTS="${DNF_SDK_HOST_OPTS} \ + --setopt=sslcacert=${SSL_CERT_FILE} \ +" + export CURL_CA_BUNDLE=${AVOCADO_SDK_PREFIX}/etc/ssl/certs/ca-certificates.crt + echo "[INFO] SSL certificates configured." +fi +"#; + + let run_config = RunConfig { + container_image: container_image.to_string(), + target: target.clone(), + command: env_setup_command.to_string(), + verbose: self.verbose, + source_environment: true, + interactive: false, + repo_url: repo_url.clone(), + repo_release: repo_release.clone(), + container_args: merged_container_args.clone(), + dnf_args: self.dnf_args.clone(), + disable_weak_dependencies: config.get_sdk_disable_weak_dependencies(), + ..Default::default() + }; + + container_helper.run_in_container(run_config).await?; + // Install SDK dependencies (into SDK) let mut sdk_packages = Vec::new(); @@ -142,8 +312,8 @@ $DNF_SDK_HOST \ $DNF_SDK_REPO_CONF \ --disablerepo=${{AVOCADO_TARGET}}-target-ext \ {} \ - install \ {} \ + install \ {} "#, dnf_args_str, @@ -195,15 +365,9 @@ $DNF_SDK_HOST \ let rootfs_command = format!( r#" -RPM_ETCCONFIGDIR=$DNF_SDK_TARGET_PREFIX \ -$DNF_SDK_HOST $DNF_NO_SCRIPTS \ - --setopt=sslcacert=${{SSL_CERT_FILE}} \ - --installroot ${{AVOCADO_PREFIX}}/rootfs \ - $DNF_SDK_TARGET_REPO_CONF \ - {} \ - install \ - {} \ - {} +RPM_ETCCONFIGDIR="$DNF_SDK_TARGET_PREFIX" \ +$DNF_SDK_HOST $DNF_NO_SCRIPTS $DNF_SDK_TARGET_REPO_CONF \ + {} {} --installroot $AVOCADO_PREFIX/rootfs install {} "#, dnf_args_str, yes, rootfs_pkg ); @@ -213,7 +377,7 @@ $DNF_SDK_HOST $DNF_NO_SCRIPTS \ target: target.clone(), command: rootfs_command, verbose: self.verbose, - source_environment: true, + source_environment: false, interactive: !self.force, repo_url: repo_url.clone(), repo_release: repo_release.clone(), @@ -268,22 +432,15 @@ $DNF_SDK_HOST $DNF_NO_SCRIPTS \ "avocado-sdk-target-sysroot".to_string() }; - // Install the target-sysroot with packagegroup-core-standalone-sdk-target plus compile deps + // Install the target-sysroot with avocado-sdk-target-sysroot plus compile deps let command = format!( r#" -RPM_ETCCONFIGDIR=$DNF_SDK_TARGET_PREFIX \ -$DNF_SDK_HOST $DNF_NO_SCRIPTS \ - --setopt=sslcacert=${{SSL_CERT_FILE}} \ - --installroot ${{AVOCADO_SDK_PREFIX}}/target-sysroot \ - --setopt=install_weak_deps=0 \ - --nodocs \ - $DNF_SDK_TARGET_REPO_CONF \ +unset RPM_CONFIGDIR +RPM_ETCCONFIGDIR="$DNF_SDK_TARGET_PREFIX" \ +$DNF_SDK_HOST $DNF_NO_SCRIPTS $DNF_SDK_TARGET_REPO_CONF \ --disablerepo=${{AVOCADO_TARGET}}-target-ext \ - {} \ - install \ - {} \ - {} \ - {} + {} {} --installroot ${{AVOCADO_SDK_PREFIX}}/target-sysroot \ + install {} {} "#, dnf_args_str, yes, diff --git a/src/utils/container.rs b/src/utils/container.rs index 1b7c148..1617b89 100644 --- a/src/utils/container.rs +++ b/src/utils/container.rs @@ -713,17 +713,6 @@ SHELL_EOF %__sh $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/scriptlet-shell.sh MACROS_EOF - RPM_CONFIGDIR="$AVOCADO_SDK_PREFIX/usr/lib/rpm" \ - RPM_ETCCONFIGDIR="$AVOCADO_SDK_PREFIX" \ - $DNF_SDK_HOST $DNF_NO_SCRIPTS $DNF_SDK_HOST_OPTS $DNF_SDK_HOST_REPO_CONF -y install "avocado-sdk-$AVOCADO_TARGET" - - RPM_CONFIGDIR="$AVOCADO_SDK_PREFIX/usr/lib/rpm" \ - RPM_ETCCONFIGDIR="$AVOCADO_SDK_PREFIX" \ - $DNF_SDK_HOST $DNF_SDK_HOST_OPTS $DNF_SDK_REPO_CONF check-update - - RPM_CONFIGDIR="$AVOCADO_SDK_PREFIX/usr/lib/rpm" \ - RPM_ETCCONFIGDIR="$AVOCADO_SDK_PREFIX" \ - $DNF_SDK_HOST $DNF_NO_SCRIPTS $DNF_SDK_HOST_OPTS $DNF_SDK_REPO_CONF -y install avocado-sdk-bootstrap fi "#); } From c53b7919f1f5a2e39271cb155960a81e1d2b9b70 Mon Sep 17 00:00:00 2001 From: Justin Schneck Date: Thu, 25 Dec 2025 18:29:17 -0500 Subject: [PATCH 07/10] move remaining bootstrap from container entrypoint --- src/commands/sdk/install.rs | 173 ++++++++++++++++++++++++++++++++++++ src/utils/container.rs | 155 +------------------------------- 2 files changed, 174 insertions(+), 154 deletions(-) diff --git a/src/commands/sdk/install.rs b/src/commands/sdk/install.rs index be87ff7..d724df5 100644 --- a/src/commands/sdk/install.rs +++ b/src/commands/sdk/install.rs @@ -105,6 +105,179 @@ impl SdkInstallCommand { let container_helper = SdkContainer::from_config(&self.config_path, config)?.verbose(self.verbose); + // Initialize SDK environment first (creates directories, copies configs, sets up wrappers) + print_info("Initializing SDK environment.", OutputLevel::Normal); + + let sdk_init_command = r#" +echo "[INFO] Initializing Avocado SDK." +mkdir -p $AVOCADO_SDK_PREFIX/etc +mkdir -p $AVOCADO_EXT_SYSROOTS +cp /etc/rpmrc $AVOCADO_SDK_PREFIX/etc +cp -r /etc/rpm $AVOCADO_SDK_PREFIX/etc +cp -r /etc/dnf $AVOCADO_SDK_PREFIX/etc +cp -r /etc/yum.repos.d $AVOCADO_SDK_PREFIX/etc + +mkdir -p $AVOCADO_SDK_PREFIX/usr/lib/rpm +cp -r /usr/lib/rpm/* $AVOCADO_SDK_PREFIX/usr/lib/rpm/ + +# Before calling DNF, $AVOCADO_SDK_PREFIX/usr/lib/rpm/macros needs to be updated to point: +# - /usr -> $AVOCADO_SDK_PREFIX/usr +# - /var -> $AVOCADO_SDK_PREFIX/var +sed -i "s|^%_usr[[:space:]]*/usr$|%_usr $AVOCADO_SDK_PREFIX/usr|" $AVOCADO_SDK_PREFIX/usr/lib/rpm/macros +sed -i "s|^%_var[[:space:]]*/var$|%_var $AVOCADO_SDK_PREFIX/var|" $AVOCADO_SDK_PREFIX/usr/lib/rpm/macros + +# Create separate rpm config for versioned extensions with custom %_dbpath +mkdir -p $AVOCADO_SDK_PREFIX/ext-rpm-config +cp -r /usr/lib/rpm/* $AVOCADO_SDK_PREFIX/ext-rpm-config/ +# Update macros for versioned extensions to use extension.d/rpm database location +sed -i "s|^%_dbpath[[:space:]]*%{_var}/lib/rpm$|%_dbpath %{_var}/lib/extension.d/rpm|" $AVOCADO_SDK_PREFIX/ext-rpm-config/macros + +# Create separate rpm config for extension scriptlets with selective execution +# This allows only update-alternatives and opkg to run, blocking other scriptlet commands +mkdir -p $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts +cp -r /usr/lib/rpm/* $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/ + +# Create a bin directory for command wrappers +mkdir -p $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/bin + +# Create update-alternatives wrapper that uses OPKG_OFFLINE_ROOT +cat > $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/bin/update-alternatives << 'UAWRAPPER_EOF' +#!/bin/bash +# update-alternatives wrapper for extension scriptlets +# Sets OPKG_OFFLINE_ROOT to manage alternatives within the extension sysroot + +if [ -n "$AVOCADO_EXT_INSTALLROOT" ]; then + case "$1" in + --install|--remove|--config|--auto|--display|--list|--query|--set) + # Debug: Show what we're doing + echo "update-alternatives: OPKG_OFFLINE_ROOT=$AVOCADO_EXT_INSTALLROOT" + echo "update-alternatives: executing: update-alternatives $*" + + # Set OPKG_OFFLINE_ROOT to the extension's installroot + # This tells opkg-update-alternatives to operate within that root + # Also ensure alternatives directory is created + /usr/bin/mkdir -p "${AVOCADO_EXT_INSTALLROOT}/var/lib/opkg/alternatives" 2>/dev/null || true + + # Set clean PATH and call update-alternatives with OPKG_OFFLINE_ROOT + export OPKG_OFFLINE_ROOT="$AVOCADO_EXT_INSTALLROOT" + PATH="${AVOCADO_SDK_PREFIX}/usr/bin:/usr/bin:/bin" \ + exec ${AVOCADO_SDK_PREFIX}/usr/bin/update-alternatives "$@" + ;; + esac +fi + +# If called without AVOCADO_EXT_INSTALLROOT, fail safely +exit 0 +UAWRAPPER_EOF +chmod +x $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/bin/update-alternatives + +# Create opkg wrapper +cat > $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/bin/opkg << 'OPKGWRAPPER_EOF' +#!/bin/bash +# opkg wrapper for extension scriptlets +exec ${AVOCADO_SDK_PREFIX}/usr/bin/opkg "$@" +OPKGWRAPPER_EOF +chmod +x $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/bin/opkg + +# Create generic noop wrapper for commands we don't want to execute +cat > $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/bin/noop-command << 'NOOP_EOF' +#!/bin/bash +# Generic noop wrapper - always succeeds +exit 0 +NOOP_EOF +chmod +x $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/bin/noop-command + +# Create a smart grep wrapper that pretends users/groups exist +cat > $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/bin/grep << 'GREP_EOF' +#!/bin/bash +# Smart grep wrapper for scriptlet user/group validation +# When checking /etc/passwd or /etc/group, pretend the user/group exists +# For everything else, use the real grep + +# Check if this looks like a user/group existence check +if [[ "$*" =~ /etc/passwd ]] || [[ "$*" =~ /etc/group ]]; then + # Pretend we found a match - output a fake line and exit 0 + echo "placeholder:x:1000:1000::/:/bin/false" + exit 0 +fi + +# For everything else, use real grep (find it in original PATH, not our wrapper dir) +# Remove our wrapper directory from PATH to find the real grep +ORIGINAL_PATH="${PATH#${AVOCADO_SDK_PREFIX}/ext-rpm-config-scripts/bin:}" +exec env PATH="$ORIGINAL_PATH" grep "$@" +GREP_EOF +chmod +x $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/bin/grep + +# Create symlinks for common scriptlet commands that should noop +# Allowlist approach: we create wrappers for what we DON'T want, not for what we DO want +for cmd in useradd groupadd usermod groupmod userdel groupdel chown chmod chgrp \ + flock systemctl systemd-tmpfiles ldconfig depmod udevadm \ + dbus-send killall service update-rc.d invoke-rc.d \ + gtk-update-icon-cache glib-compile-schemas update-desktop-database \ + fc-cache mkfontdir mkfontscale install-info update-mime-database \ + passwd chpasswd gpasswd newusers \ + systemd-sysusers systemd-hwdb kmod insmod modprobe \ + setcap getcap chcon restorecon selinuxenabled getenforce \ + rpm-helper gtk-query-immodules-3.0 \ + gdk-pixbuf-query-loaders gio-querymodules \ + dconf gsettings glib-compile-resources \ + bbnote bbfatal bbwarn bbdebug; do + ln -sf noop-command $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/bin/$cmd +done + +# Create shell wrapper for scriptlet interpreter +cat > $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/scriptlet-shell.sh << 'SHELL_EOF' +#!/bin/bash +# Shell wrapper for RPM scriptlets +# Set OPT=--opt to make Yocto scriptlets skip user/group management +# This is the proper way to tell Yocto scripts we're in a sysroot environment + +# Set PATH to find our command wrappers first, but explicitly exclude the installroot +# Only include: wrapper bin, SDK utilities, and container system paths +export PATH="${AVOCADO_SDK_PREFIX}/ext-rpm-config-scripts/bin:${AVOCADO_SDK_PREFIX}/usr/bin:/usr/bin:/bin" + +# Tell Yocto scriptlets we're in OPT mode (skip user/group creation) +export OPT="--opt" + +exec ${AVOCADO_SDK_PREFIX}/usr/bin/bash "$@" +SHELL_EOF +chmod +x $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/scriptlet-shell.sh + +# Update macros for extension scriptlets +sed -i "s|^%_dbpath[[:space:]]*%{_var}/lib/rpm$|%_dbpath %{_var}/lib/rpm|" $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/macros + +# Add macro overrides for shell interpreter only +cat >> $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/macros << 'MACROS_EOF' + +# Override shell interpreter for scriptlets to use our custom shell +%__bash $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/scriptlet-shell.sh +%__sh $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/scriptlet-shell.sh +MACROS_EOF +"#; + + let run_config = RunConfig { + container_image: container_image.to_string(), + target: target.clone(), + command: sdk_init_command.to_string(), + verbose: self.verbose, + source_environment: true, + interactive: false, + repo_url: repo_url.clone(), + repo_release: repo_release.clone(), + container_args: merged_container_args.clone(), + dnf_args: self.dnf_args.clone(), + disable_weak_dependencies: config.get_sdk_disable_weak_dependencies(), + ..Default::default() + }; + + let init_success = container_helper.run_in_container(run_config).await?; + + if init_success { + print_success("Initialized SDK environment.", OutputLevel::Normal); + } else { + return Err(anyhow::anyhow!("Failed to initialize SDK environment.")); + } + // Install avocado-sdk-{target} with version from distro.version print_info( &format!("Installing SDK for target '{}'.", target), diff --git a/src/utils/container.rs b/src/utils/container.rs index 1617b89..7ce4f2f 100644 --- a/src/utils/container.rs +++ b/src/utils/container.rs @@ -479,7 +479,7 @@ impl SdkContainer { extension_sysroot: Option<&str>, runtime_sysroot: Option<&str>, target: &str, - no_bootstrap: bool, + _no_bootstrap: bool, disable_weak_dependencies: bool, ) -> String { // Conditionally add install_weak_deps flag @@ -564,159 +564,6 @@ echo "${{REPO_URL}}" > ${{DNF_SDK_TARGET_PREFIX}}/etc/dnf/vars/repo_url "# ); - // Only include bootstrap logic if no_bootstrap is false - if !no_bootstrap { - script.push_str(r#" -if [ ! -f "${AVOCADO_SDK_PREFIX}/environment-setup" ]; then - echo "[INFO] Initializing Avocado SDK." - mkdir -p $AVOCADO_SDK_PREFIX/etc - mkdir -p $AVOCADO_EXT_SYSROOTS - cp /etc/rpmrc $AVOCADO_SDK_PREFIX/etc - cp -r /etc/rpm $AVOCADO_SDK_PREFIX/etc - cp -r /etc/dnf $AVOCADO_SDK_PREFIX/etc - cp -r /etc/yum.repos.d $AVOCADO_SDK_PREFIX/etc - - mkdir -p $AVOCADO_SDK_PREFIX/usr/lib/rpm - cp -r /usr/lib/rpm/* $AVOCADO_SDK_PREFIX/usr/lib/rpm/ - - # Before calling DNF, $AVOCADO_SDK_PREFIX/usr/lib/rpm/macros needs to be updated to point: - # - /usr -> $AVOCADO_SDK_PREFIX/usr - # - /var -> $AVOCADO_SDK_PREFIX/var - sed -i "s|^%_usr[[:space:]]*/usr$|%_usr $AVOCADO_SDK_PREFIX/usr|" $AVOCADO_SDK_PREFIX/usr/lib/rpm/macros - sed -i "s|^%_var[[:space:]]*/var$|%_var $AVOCADO_SDK_PREFIX/var|" $AVOCADO_SDK_PREFIX/usr/lib/rpm/macros - - # Create separate rpm config for versioned extensions with custom %_dbpath - mkdir -p $AVOCADO_SDK_PREFIX/ext-rpm-config - cp -r /usr/lib/rpm/* $AVOCADO_SDK_PREFIX/ext-rpm-config/ - # Update macros for versioned extensions to use extension.d/rpm database location - sed -i "s|^%_dbpath[[:space:]]*%{_var}/lib/rpm$|%_dbpath %{_var}/lib/extension.d/rpm|" $AVOCADO_SDK_PREFIX/ext-rpm-config/macros - - # Create separate rpm config for extension scriptlets with selective execution - # This allows only update-alternatives and opkg to run, blocking other scriptlet commands - mkdir -p $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts - cp -r /usr/lib/rpm/* $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/ - - # Create a bin directory for command wrappers - mkdir -p $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/bin - - # Create update-alternatives wrapper that uses OPKG_OFFLINE_ROOT - cat > $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/bin/update-alternatives << 'UAWRAPPER_EOF' -#!/bin/bash -# update-alternatives wrapper for extension scriptlets -# Sets OPKG_OFFLINE_ROOT to manage alternatives within the extension sysroot - -if [ -n "$AVOCADO_EXT_INSTALLROOT" ]; then - case "$1" in - --install|--remove|--config|--auto|--display|--list|--query|--set) - # Debug: Show what we're doing - echo "update-alternatives: OPKG_OFFLINE_ROOT=$AVOCADO_EXT_INSTALLROOT" - echo "update-alternatives: executing: update-alternatives $*" - - # Set OPKG_OFFLINE_ROOT to the extension's installroot - # This tells opkg-update-alternatives to operate within that root - # Also ensure alternatives directory is created - /usr/bin/mkdir -p "${AVOCADO_EXT_INSTALLROOT}/var/lib/opkg/alternatives" 2>/dev/null || true - - # Set clean PATH and call update-alternatives with OPKG_OFFLINE_ROOT - export OPKG_OFFLINE_ROOT="$AVOCADO_EXT_INSTALLROOT" - PATH="${AVOCADO_SDK_PREFIX}/usr/bin:/usr/bin:/bin" \ - exec ${AVOCADO_SDK_PREFIX}/usr/bin/update-alternatives "$@" - ;; - esac -fi - -# If called without AVOCADO_EXT_INSTALLROOT, fail safely -exit 0 -UAWRAPPER_EOF - chmod +x $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/bin/update-alternatives - - # Create opkg wrapper - cat > $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/bin/opkg << 'OPKGWRAPPER_EOF' -#!/bin/bash -# opkg wrapper for extension scriptlets -exec ${AVOCADO_SDK_PREFIX}/usr/bin/opkg "$@" -OPKGWRAPPER_EOF - chmod +x $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/bin/opkg - - # Create generic noop wrapper for commands we don't want to execute - cat > $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/bin/noop-command << 'NOOP_EOF' -#!/bin/bash -# Generic noop wrapper - always succeeds -exit 0 -NOOP_EOF - chmod +x $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/bin/noop-command - - # Create a smart grep wrapper that pretends users/groups exist - cat > $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/bin/grep << 'GREP_EOF' -#!/bin/bash -# Smart grep wrapper for scriptlet user/group validation -# When checking /etc/passwd or /etc/group, pretend the user/group exists -# For everything else, use the real grep - -# Check if this looks like a user/group existence check -if [[ "$*" =~ /etc/passwd ]] || [[ "$*" =~ /etc/group ]]; then - # Pretend we found a match - output a fake line and exit 0 - echo "placeholder:x:1000:1000::/:/bin/false" - exit 0 -fi - -# For everything else, use real grep (find it in original PATH, not our wrapper dir) -# Remove our wrapper directory from PATH to find the real grep -ORIGINAL_PATH="${PATH#${AVOCADO_SDK_PREFIX}/ext-rpm-config-scripts/bin:}" -exec env PATH="$ORIGINAL_PATH" grep "$@" -GREP_EOF - chmod +x $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/bin/grep - - # Create symlinks for common scriptlet commands that should noop - # Allowlist approach: we create wrappers for what we DON'T want, not for what we DO want - for cmd in useradd groupadd usermod groupmod userdel groupdel chown chmod chgrp \ - flock systemctl systemd-tmpfiles ldconfig depmod udevadm \ - dbus-send killall service update-rc.d invoke-rc.d \ - gtk-update-icon-cache glib-compile-schemas update-desktop-database \ - fc-cache mkfontdir mkfontscale install-info update-mime-database \ - passwd chpasswd gpasswd newusers \ - systemd-sysusers systemd-hwdb kmod insmod modprobe \ - setcap getcap chcon restorecon selinuxenabled getenforce \ - rpm-helper gtk-query-immodules-3.0 \ - gdk-pixbuf-query-loaders gio-querymodules \ - dconf gsettings glib-compile-resources \ - bbnote bbfatal bbwarn bbdebug; do - ln -sf noop-command $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/bin/$cmd - done - - # Create shell wrapper for scriptlet interpreter - cat > $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/scriptlet-shell.sh << 'SHELL_EOF' -#!/bin/bash -# Shell wrapper for RPM scriptlets -# Set OPT=--opt to make Yocto scriptlets skip user/group management -# This is the proper way to tell Yocto scripts we're in a sysroot environment - -# Set PATH to find our command wrappers first, but explicitly exclude the installroot -# Only include: wrapper bin, SDK utilities, and container system paths -export PATH="${AVOCADO_SDK_PREFIX}/ext-rpm-config-scripts/bin:${AVOCADO_SDK_PREFIX}/usr/bin:/usr/bin:/bin" - -# Tell Yocto scriptlets we're in OPT mode (skip user/group creation) -export OPT="--opt" - -exec ${AVOCADO_SDK_PREFIX}/usr/bin/bash "$@" -SHELL_EOF - chmod +x $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/scriptlet-shell.sh - - # Update macros for extension scriptlets - sed -i "s|^%_dbpath[[:space:]]*%{_var}/lib/rpm$|%_dbpath %{_var}/lib/rpm|" $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/macros - - # Add macro overrides for shell interpreter only - cat >> $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/macros << 'MACROS_EOF' - -# Override shell interpreter for scriptlets to use our custom shell -%__bash $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/scriptlet-shell.sh -%__sh $AVOCADO_SDK_PREFIX/ext-rpm-config-scripts/scriptlet-shell.sh -MACROS_EOF - -fi -"#); - } - script.push_str( r#" export RPM_ETCCONFIGDIR="$AVOCADO_SDK_PREFIX" From 2a9443a6c0f17ccef6b34d21a443638c115199b5 Mon Sep 17 00:00:00 2001 From: Justin Schneck Date: Thu, 25 Dec 2025 22:31:53 -0500 Subject: [PATCH 08/10] Implement lock.json file for dnf deps --- src/commands/ext/install.rs | 84 ++- src/commands/install.rs | 247 ++++++-- src/commands/runtime/install.rs | 100 ++- src/commands/sdk/install.rs | 303 +++++++-- src/utils/container.rs | 105 ++++ src/utils/lockfile.rs | 1033 +++++++++++++++++++++++++++++++ src/utils/mod.rs | 1 + 7 files changed, 1737 insertions(+), 136 deletions(-) create mode 100644 src/utils/lockfile.rs diff --git a/src/commands/ext/install.rs b/src/commands/ext/install.rs index 3688b40..939ae9f 100644 --- a/src/commands/ext/install.rs +++ b/src/commands/ext/install.rs @@ -1,7 +1,9 @@ use anyhow::{Context, Result}; +use std::path::{Path, PathBuf}; use crate::utils::config::{Config, ExtensionLocation}; use crate::utils::container::{RunConfig, SdkContainer}; +use crate::utils::lockfile::{build_package_spec_with_lock, LockFile, SysrootType}; use crate::utils::output::{print_debug, print_error, print_info, print_success, OutputLevel}; use crate::utils::stamps::{ compute_ext_input_hash, generate_write_stamp_script, Stamp, StampOutputs, @@ -171,9 +173,27 @@ impl ExtInstallCommand { let target = resolve_target_required(self.target.as_deref(), config)?; // Use the container helper to run the setup commands - let container_helper = SdkContainer::new(); + let container_helper = SdkContainer::new().verbose(self.verbose); let total = extensions_to_install.len(); + // Load lock file for reproducible builds + let src_dir = config + .get_resolved_src_dir(&self.config_path) + .unwrap_or_else(|| { + PathBuf::from(&self.config_path) + .parent() + .unwrap_or(std::path::Path::new(".")) + .to_path_buf() + }); + let mut lock_file = LockFile::load(&src_dir).with_context(|| "Failed to load lock file")?; + + if self.verbose && !lock_file.is_empty() { + print_info( + "Using existing lock file for version pinning.", + OutputLevel::Normal, + ); + } + // Install each extension for (index, (ext_name, ext_location)) in extensions_to_install.iter().enumerate() { if self.verbose { @@ -210,6 +230,8 @@ impl ExtInstallCommand { repo_release.as_ref(), &merged_container_args, config.get_sdk_disable_weak_dependencies(), + &mut lock_file, + &src_dir, ) .await? { @@ -271,6 +293,8 @@ impl ExtInstallCommand { repo_release: Option<&String>, merged_container_args: &Option>, disable_weak_dependencies: bool, + lock_file: &mut LockFile, + src_dir: &Path, ) -> Result { // Create the commands to check and set up the directory structure let check_command = format!("[ -d $AVOCADO_EXT_SYSROOTS/{extension} ]"); @@ -332,9 +356,12 @@ impl ExtInstallCommand { // Install dependencies if they exist let dependencies = ext_config.as_ref().and_then(|ec| ec.get("dependencies")); + let sysroot = SysrootType::Extension(extension.to_string()); + if let Some(serde_yaml::Value::Mapping(deps_map)) = dependencies { // Build list of packages to install and handle extension dependencies let mut packages = Vec::new(); + let mut package_names = Vec::new(); let mut extension_dependencies = Vec::new(); for (package_name_val, version_spec) in deps_map { @@ -349,11 +376,15 @@ impl ExtInstallCommand { // Simple string version: "package: version" or "package: '*'" // These are always package repository dependencies serde_yaml::Value::String(version) => { - if version == "*" { - packages.push(package_name.to_string()); - } else { - packages.push(format!("{package_name}-{version}")); - } + let package_spec = build_package_spec_with_lock( + lock_file, + target, + &sysroot, + package_name, + version, + ); + packages.push(package_spec); + package_names.push(package_name.to_string()); } // Object/mapping value: need to check what type of dependency serde_yaml::Value::Mapping(spec_map) => { @@ -410,11 +441,15 @@ impl ExtInstallCommand { // Check for explicit version in object format // Format: { version: "1.0.0" } if let Some(serde_yaml::Value::String(version)) = spec_map.get("version") { - if version == "*" { - packages.push(package_name.to_string()); - } else { - packages.push(format!("{package_name}-{version}")); - } + let package_spec = build_package_spec_with_lock( + lock_file, + target, + &sysroot, + package_name, + version, + ); + packages.push(package_spec); + package_names.push(package_name.to_string()); } // If it's a mapping without compile, ext, or version keys, skip it // (unknown format) @@ -511,6 +546,33 @@ $DNF_SDK_HOST \ ); return Ok(false); } + + // Query installed versions and update lock file + if !package_names.is_empty() { + let installed_versions = container_helper + .query_installed_packages( + &sysroot, + &package_names, + container_image, + target, + repo_url.cloned(), + repo_release.cloned(), + merged_container_args.clone(), + ) + .await?; + + if !installed_versions.is_empty() { + lock_file.update_sysroot_versions(target, &sysroot, installed_versions); + if self.verbose { + print_info( + &format!("Updated lock file with extension '{extension}' package versions."), + OutputLevel::Normal, + ); + } + // Save lock file immediately after extension install + lock_file.save(src_dir)?; + } + } } else if self.verbose { print_debug( &format!("No valid dependencies found for extension '{extension}'."), diff --git a/src/commands/install.rs b/src/commands/install.rs index 404faeb..b6a8f01 100644 --- a/src/commands/install.rs +++ b/src/commands/install.rs @@ -1,6 +1,7 @@ //! Install command implementation that runs SDK, extension, and runtime installs. use anyhow::{Context, Result}; +use std::path::PathBuf; use crate::commands::{ ext::ExtInstallCommand, runtime::RuntimeInstallCommand, sdk::SdkInstallCommand, @@ -8,6 +9,7 @@ use crate::commands::{ use crate::utils::{ config::{ComposedConfig, Config}, container::SdkContainer, + lockfile::{build_package_spec_with_lock, LockFile, SysrootType}, output::{print_info, print_success, OutputLevel}, target::validate_and_log_target, }; @@ -91,6 +93,19 @@ impl InstallCommand { OutputLevel::Normal, ); + // Load lock file for reproducible builds (used for versioned extensions in this command) + let src_dir = config + .get_resolved_src_dir(&self.config_path) + .unwrap_or_else(|| { + PathBuf::from(&self.config_path) + .parent() + .unwrap_or(std::path::Path::new(".")) + .to_path_buf() + }); + + // We'll load the lock file lazily when needed (for external/versioned extensions) + let mut lock_file; + // 1. Install SDK dependencies print_info("Step 1/3: Installing SDK dependencies", OutputLevel::Normal); let sdk_install_cmd = SdkInstallCommand::new( @@ -154,8 +169,11 @@ impl InstallCommand { ); } + // Reload lock file from disk to get latest state from previous installs + lock_file = LockFile::load(&src_dir)?; + // Install external extension to ${AVOCADO_PREFIX}/extensions/ - self.install_external_extension(config, &self.config_path, name, ext_config_path, &_target).await.with_context(|| { + self.install_external_extension(config, &self.config_path, name, ext_config_path, &_target, &mut lock_file).await.with_context(|| { format!("Failed to install external extension '{name}' from config '{ext_config_path}'") })?; } @@ -169,8 +187,11 @@ impl InstallCommand { ); } + // Reload lock file from disk to get latest state from previous installs + lock_file = LockFile::load(&src_dir)?; + // Install versioned extension to its own sysroot - self.install_versioned_extension(config, name, version, &_target).await.with_context(|| { + self.install_versioned_extension(config, name, version, &_target, &mut lock_file).await.with_context(|| { format!("Failed to install versioned extension '{name}' version '{version}'") })?; } @@ -533,6 +554,7 @@ impl InstallCommand { extension_name: &str, external_config_path: &str, target: &str, + lock_file: &mut LockFile, ) -> Result<()> { // Load the external extension configuration let external_extensions = @@ -632,13 +654,17 @@ impl InstallCommand { base_config_path, external_config_path, target, + lock_file, ) .await?; // Process the extension's dependencies (packages, not extension or compile dependencies) + let sysroot = SysrootType::Extension(extension_name.to_string()); + if let Some(serde_yaml::Value::Mapping(deps_map)) = extension_config.get("dependencies") { if !deps_map.is_empty() { let mut packages = Vec::new(); + let mut package_names = Vec::new(); // Process package dependencies (not extension or compile dependencies) for (package_name_val, version_spec) in deps_map { @@ -667,27 +693,28 @@ impl InstallCommand { } // Process package dependencies only (simple string versions or version objects) - match version_spec { - serde_yaml::Value::String(version) => { - if version == "*" { - packages.push(package_name.to_string()); - } else { - packages.push(format!("{package_name}-{version}")); - } - } + let config_version = match version_spec { + serde_yaml::Value::String(version) => version.clone(), serde_yaml::Value::Mapping(spec_map) => { // Only process if it has a "version" key (already checked it doesn't have ext/compile) - if let Some(version) = spec_map.get("version").and_then(|v| v.as_str()) - { - if version == "*" { - packages.push(package_name.to_string()); - } else { - packages.push(format!("{package_name}-{version}")); - } - } + spec_map + .get("version") + .and_then(|v| v.as_str()) + .unwrap_or("*") + .to_string() } - _ => {} - } + _ => continue, + }; + + let package_spec = build_package_spec_with_lock( + lock_file, + target, + &sysroot, + package_name, + &config_version, + ); + packages.push(package_spec); + package_names.push(package_name.to_string()); } if !packages.is_empty() { @@ -737,9 +764,9 @@ $DNF_SDK_HOST \ verbose: self.verbose, source_environment: false, // don't source environment interactive: !self.force, // interactive if not forced - repo_url, - repo_release, - container_args: merged_container_args, + repo_url: repo_url.clone(), + repo_release: repo_release.clone(), + container_args: merged_container_args.clone(), dnf_args: self.dnf_args.clone(), disable_weak_dependencies: config.get_sdk_disable_weak_dependencies(), ..Default::default() @@ -752,6 +779,45 @@ $DNF_SDK_HOST \ &format!("Installed {} package(s) for external extension '{extension_name}'.", packages.len()), crate::utils::output::OutputLevel::Normal, ); + + // Query installed versions and update lock file + if !package_names.is_empty() { + let installed_versions = container_helper + .query_installed_packages( + &sysroot, + &package_names, + container_image, + target, + repo_url, + repo_release, + merged_container_args, + ) + .await?; + + if !installed_versions.is_empty() { + lock_file.update_sysroot_versions( + target, + &sysroot, + installed_versions, + ); + if self.verbose { + print_info( + &format!("Updated lock file with external extension '{extension_name}' package versions."), + crate::utils::output::OutputLevel::Normal, + ); + } + // Save lock file immediately after external extension install + let src_dir = PathBuf::from(&self.config_path) + .parent() + .ok_or_else(|| { + anyhow::anyhow!( + "Failed to get parent directory of config file" + ) + })? + .to_path_buf(); + lock_file.save(&src_dir)?; + } + } } else { return Err(anyhow::anyhow!( "Failed to install package dependencies for external extension '{extension_name}'" @@ -828,9 +894,10 @@ $DNF_SDK_HOST \ extension_name: &str, version: &str, target: &str, + lock_file: &mut LockFile, ) -> Result<()> { // Get container configuration - let container_helper = SdkContainer::new(); + let container_helper = SdkContainer::new().verbose(self.verbose); let container_image = config.get_sdk_image().ok_or_else(|| { anyhow::anyhow!("No container image specified in config under 'sdk.image'") })?; @@ -892,11 +959,11 @@ $DNF_SDK_HOST \ } // Install the versioned extension package using DNF - let package_spec = if version == "*" { - extension_name.to_string() - } else { - format!("{extension_name}-{version}") - }; + // Use the sysroot_name for lock file key (this is the extension name) + // Note: VersionedExtension uses different RPM_CONFIGDIR than local extensions + let sysroot = SysrootType::VersionedExtension(sysroot_name.clone()); + let package_spec = + build_package_spec_with_lock(lock_file, target, &sysroot, extension_name, version); let installroot = format!("$AVOCADO_EXT_SYSROOTS/{sysroot_name}"); let yes = if self.force { "-y" } else { "" }; @@ -941,9 +1008,9 @@ $DNF_SDK_HOST \ verbose: self.verbose, source_environment: false, interactive: !self.force, - repo_url, - repo_release, - container_args: merged_container_args, + repo_url: repo_url.clone(), + repo_release: repo_release.clone(), + container_args: merged_container_args.clone(), dnf_args: self.dnf_args.clone(), disable_weak_dependencies: config.get_sdk_disable_weak_dependencies(), ..Default::default() @@ -957,6 +1024,37 @@ $DNF_SDK_HOST \ )); } + // Query installed version and update lock file + let installed_versions = container_helper + .query_installed_packages( + &sysroot, + &[extension_name.to_string()], + container_image, + target, + repo_url, + repo_release, + merged_container_args, + ) + .await?; + + if !installed_versions.is_empty() { + lock_file.update_sysroot_versions(target, &sysroot, installed_versions); + if self.verbose { + print_info( + &format!( + "Updated lock file with versioned extension '{extension_name}' version." + ), + crate::utils::output::OutputLevel::Normal, + ); + } + // Save lock file immediately after versioned extension install + let src_dir = PathBuf::from(&self.config_path) + .parent() + .ok_or_else(|| anyhow::anyhow!("Failed to get parent directory of config file"))? + .to_path_buf(); + lock_file.save(&src_dir)?; + } + let version_msg = if version == "*" { "latest version".to_string() } else { @@ -980,6 +1078,7 @@ $DNF_SDK_HOST \ base_config_path: &str, external_config_path: &str, target: &str, + lock_file: &mut LockFile, ) -> Result<()> { // Resolve the external config path let resolved_external_config_path = @@ -1029,37 +1128,34 @@ $DNF_SDK_HOST \ return Ok(()); }; - // Build list of SDK packages to install + // Build list of SDK packages to install (using lock file for version pinning) let mut sdk_packages = Vec::new(); + let mut sdk_package_names = Vec::new(); for (pkg_name_val, version_spec) in sdk_deps_map { let pkg_name = match pkg_name_val.as_str() { Some(name) => name, None => continue, }; - match version_spec { - serde_yaml::Value::String(version) => { - if version == "*" { - sdk_packages.push(pkg_name.to_string()); - } else { - sdk_packages.push(format!("{pkg_name}-{version}")); - } - } - serde_yaml::Value::Mapping(spec_map) => { - if let Some(version) = spec_map.get("version").and_then(|v| v.as_str()) { - if version == "*" { - sdk_packages.push(pkg_name.to_string()); - } else { - sdk_packages.push(format!("{pkg_name}-{version}")); - } - } else { - sdk_packages.push(pkg_name.to_string()); - } - } - _ => { - sdk_packages.push(pkg_name.to_string()); - } - } + let config_version = match version_spec { + serde_yaml::Value::String(version) => version.clone(), + serde_yaml::Value::Mapping(spec_map) => spec_map + .get("version") + .and_then(|v| v.as_str()) + .unwrap_or("*") + .to_string(), + _ => "*".to_string(), + }; + + let package_spec = build_package_spec_with_lock( + lock_file, + target, + &SysrootType::Sdk, + pkg_name, + &config_version, + ); + sdk_packages.push(package_spec); + sdk_package_names.push(pkg_name.to_string()); } if sdk_packages.is_empty() { @@ -1129,9 +1225,9 @@ $DNF_SDK_HOST \ verbose: self.verbose, source_environment: true, interactive: !self.force, - repo_url, - repo_release, - container_args: merged_container_args, + repo_url: repo_url.clone(), + repo_release: repo_release.clone(), + container_args: merged_container_args.clone(), dnf_args: self.dnf_args.clone(), disable_weak_dependencies: config.get_sdk_disable_weak_dependencies(), ..Default::default() @@ -1145,6 +1241,39 @@ $DNF_SDK_HOST \ )); } + // Query installed versions and update lock file + if !sdk_package_names.is_empty() { + let installed_versions = container_helper + .query_installed_packages( + &SysrootType::Sdk, + &sdk_package_names, + container_image, + target, + repo_url, + repo_release, + merged_container_args, + ) + .await?; + + if !installed_versions.is_empty() { + lock_file.update_sysroot_versions(target, &SysrootType::Sdk, installed_versions); + if self.verbose { + print_info( + &format!("Updated lock file with SDK dependencies from external config '{external_config_path}'."), + OutputLevel::Normal, + ); + } + // Save lock file immediately after external extension SDK deps install + let src_dir = PathBuf::from(base_config_path) + .parent() + .ok_or_else(|| { + anyhow::anyhow!("Failed to get parent directory of config file") + })? + .to_path_buf(); + lock_file.save(&src_dir)?; + } + } + print_info( &format!( "Installed {} SDK dependencies from external config '{external_config_path}'.", diff --git a/src/commands/runtime/install.rs b/src/commands/runtime/install.rs index 60193f9..ffb7773 100644 --- a/src/commands/runtime/install.rs +++ b/src/commands/runtime/install.rs @@ -1,7 +1,9 @@ use anyhow::{Context, Result}; +use std::path::{Path, PathBuf}; use crate::utils::config::Config; use crate::utils::container::{RunConfig, SdkContainer}; +use crate::utils::lockfile::{build_package_spec_with_lock, LockFile, SysrootType}; use crate::utils::output::{print_debug, print_error, print_info, print_success, OutputLevel}; use crate::utils::stamps::{ compute_runtime_input_hash, generate_write_stamp_script, Stamp, StampOutputs, @@ -120,7 +122,25 @@ impl RuntimeInstallCommand { .context("No SDK container image specified in configuration")?; // Initialize container helper - let container_helper = SdkContainer::new(); + let container_helper = SdkContainer::new().verbose(self.verbose); + + // Load lock file for reproducible builds + let src_dir = config + .get_resolved_src_dir(&self.config_path) + .unwrap_or_else(|| { + PathBuf::from(&self.config_path) + .parent() + .unwrap_or(std::path::Path::new(".")) + .to_path_buf() + }); + let mut lock_file = LockFile::load(&src_dir).with_context(|| "Failed to load lock file")?; + + if self.verbose && !lock_file.is_empty() { + print_info( + "Using existing lock file for version pinning.", + OutputLevel::Normal, + ); + } // Install dependencies for each runtime for runtime_name in &runtimes_to_install { @@ -139,6 +159,8 @@ impl RuntimeInstallCommand { repo_url.as_ref(), repo_release.as_ref(), &merged_container_args, + &mut lock_file, + &src_dir, ) .await?; @@ -212,6 +234,8 @@ impl RuntimeInstallCommand { repo_url: Option<&String>, repo_release: Option<&String>, merged_container_args: &Option>, + lock_file: &mut LockFile, + src_dir: &Path, ) -> Result { // Get runtime configuration let runtime_config = config_toml["runtime"][runtime].clone(); @@ -286,9 +310,12 @@ impl RuntimeInstallCommand { .as_ref() .and_then(|merged| merged.get("dependencies")); + let sysroot = SysrootType::Runtime(runtime.to_string()); + if let Some(serde_yaml::Value::Mapping(deps_map)) = dependencies { // Build list of packages to install (excluding extension references) let mut packages = Vec::new(); + let mut package_names = Vec::new(); for (package_name_val, version_spec) in deps_map { // Convert package name from Value to String let package_name = match package_name_val.as_str() { @@ -322,29 +349,27 @@ impl RuntimeInstallCommand { } } - let package_name_and_version = if version_spec.as_str().is_some() { - let version = version_spec.as_str().unwrap(); - if version == "*" { - package_name.to_string() - } else { - format!("{package_name}-{version}") - } + let config_version = if let Some(version) = version_spec.as_str() { + version.to_string() } else if let serde_yaml::Value::Mapping(spec_map) = version_spec { - if let Some(version) = spec_map.get("version") { - let version = version.as_str().unwrap_or("*"); - if version == "*" { - package_name.to_string() - } else { - format!("{package_name}-{version}") - } - } else { - package_name.to_string() - } + spec_map + .get("version") + .and_then(|v| v.as_str()) + .unwrap_or("*") + .to_string() } else { - package_name.to_string() + "*".to_string() }; - packages.push(package_name_and_version); + let package_spec = build_package_spec_with_lock( + lock_file, + &target_arch, + &sysroot, + package_name, + &config_version, + ); + packages.push(package_spec); + package_names.push(package_name.to_string()); } if !packages.is_empty() { @@ -393,7 +418,7 @@ $DNF_SDK_HOST \ target: target_arch.clone(), command: dnf_command, verbose: self.verbose, - source_environment: true, // need environment for DNF + source_environment: false, // Don't source environment - matches rootfs install behavior interactive: !self.force, repo_url: repo_url.cloned(), repo_release: repo_release.cloned(), @@ -416,6 +441,39 @@ $DNF_SDK_HOST \ &format!("Successfully installed packages for runtime '{runtime}'"), OutputLevel::Normal, ); + + // Query installed versions and update lock file + if !package_names.is_empty() { + let installed_versions = container_helper + .query_installed_packages( + &sysroot, + &package_names, + container_image, + &target_arch, + repo_url.cloned(), + repo_release.cloned(), + merged_container_args.clone(), + ) + .await?; + + if !installed_versions.is_empty() { + lock_file.update_sysroot_versions( + &target_arch, + &sysroot, + installed_versions, + ); + if self.verbose { + print_info( + &format!( + "Updated lock file with runtime '{runtime}' package versions." + ), + OutputLevel::Normal, + ); + } + // Save lock file immediately after runtime install + lock_file.save(src_dir)?; + } + } } else { print_info( &format!("No packages to install for runtime '{runtime}'"), diff --git a/src/commands/sdk/install.rs b/src/commands/sdk/install.rs index d724df5..8c489b1 100644 --- a/src/commands/sdk/install.rs +++ b/src/commands/sdk/install.rs @@ -2,10 +2,12 @@ use anyhow::{Context, Result}; use std::collections::HashMap; +use std::path::PathBuf; use crate::utils::{ config::Config, container::{RunConfig, SdkContainer}, + lockfile::{build_package_spec_with_lock, LockFile, SysrootType}, output::{print_info, print_success, OutputLevel}, stamps::{compute_sdk_input_hash, generate_write_stamp_script, Stamp, StampOutputs}, target::validate_and_log_target, @@ -105,6 +107,24 @@ impl SdkInstallCommand { let container_helper = SdkContainer::from_config(&self.config_path, config)?.verbose(self.verbose); + // Load lock file for reproducible builds + let src_dir = config + .get_resolved_src_dir(&self.config_path) + .unwrap_or_else(|| { + PathBuf::from(&self.config_path) + .parent() + .unwrap_or(std::path::Path::new(".")) + .to_path_buf() + }); + let mut lock_file = LockFile::load(&src_dir).with_context(|| "Failed to load lock file")?; + + if self.verbose && !lock_file.is_empty() { + print_info( + "Using existing lock file for version pinning.", + OutputLevel::Normal, + ); + } + // Initialize SDK environment first (creates directories, copies configs, sets up wrappers) print_info("Initializing SDK environment.", OutputLevel::Normal); @@ -284,11 +304,19 @@ MACROS_EOF OutputLevel::Normal, ); - let sdk_target_pkg = if let Some(version) = config.get_distro_version() { - format!("avocado-sdk-{}-{}", target, version) - } else { - format!("avocado-sdk-{}", target) - }; + // Build package name and spec with lock file support + let sdk_target_pkg_name = format!("avocado-sdk-{}", target); + let sdk_target_config_version = config + .get_distro_version() + .map(|s| s.as_str()) + .unwrap_or("*"); + let sdk_target_pkg = build_package_spec_with_lock( + &lock_file, + &target, + &SysrootType::Sdk, + &sdk_target_pkg_name, + sdk_target_config_version, + ); let sdk_target_command = format!( r#" @@ -321,11 +349,16 @@ $DNF_SDK_HOST $DNF_NO_SCRIPTS \ let sdk_target_success = container_helper.run_in_container(run_config).await?; + // Track all SDK packages installed for lock file update at the end + let mut all_sdk_package_names: Vec = Vec::new(); + if sdk_target_success { print_success( &format!("Installed SDK for target '{}'.", target), OutputLevel::Normal, ); + // Add to list for later query (after environment is fully set up) + all_sdk_package_names.push(sdk_target_pkg_name); } else { return Err(anyhow::anyhow!( "Failed to install SDK for target '{}'.", @@ -363,11 +396,18 @@ $DNF_SDK_HOST \ // Install avocado-sdk-bootstrap with version from distro.version print_info("Installing SDK bootstrap.", OutputLevel::Normal); - let bootstrap_pkg = if let Some(version) = config.get_distro_version() { - format!("avocado-sdk-bootstrap-{}", version) - } else { - "avocado-sdk-bootstrap".to_string() - }; + let bootstrap_pkg_name = "avocado-sdk-bootstrap"; + let bootstrap_config_version = config + .get_distro_version() + .map(|s| s.as_str()) + .unwrap_or("*"); + let bootstrap_pkg = build_package_spec_with_lock( + &lock_file, + &target, + &SysrootType::Sdk, + bootstrap_pkg_name, + bootstrap_config_version, + ); let bootstrap_command = format!( r#" @@ -402,6 +442,8 @@ $DNF_SDK_HOST $DNF_NO_SCRIPTS \ if bootstrap_success { print_success("Installed SDK bootstrap.", OutputLevel::Normal); + // Add to list for later query (after environment is fully set up) + all_sdk_package_names.push(bootstrap_pkg_name.to_string()); } else { return Err(anyhow::anyhow!("Failed to install SDK bootstrap.")); } @@ -450,10 +492,17 @@ fi // Install SDK dependencies (into SDK) let mut sdk_packages = Vec::new(); + let mut sdk_package_names = Vec::new(); // Add regular SDK dependencies if let Some(ref dependencies) = sdk_dependencies { - sdk_packages.extend(self.build_package_list(dependencies)); + sdk_packages.extend(self.build_package_list_with_lock( + dependencies, + &lock_file, + &target, + &SysrootType::Sdk, + )); + sdk_package_names.extend(self.extract_package_names(dependencies)); } // Add extension SDK dependencies to the package list @@ -464,8 +513,10 @@ fi OutputLevel::Normal, ); } - let ext_packages = self.build_package_list(ext_deps); + let ext_packages = + self.build_package_list_with_lock(ext_deps, &lock_file, &target, &SysrootType::Sdk); sdk_packages.extend(ext_packages); + sdk_package_names.extend(self.extract_package_names(ext_deps)); } if !sdk_packages.is_empty() { @@ -513,6 +564,8 @@ $DNF_SDK_HOST \ if install_success { print_success("Installed SDK dependencies.", OutputLevel::Normal); + // Add SDK dependency package names to the list + all_sdk_package_names.extend(sdk_package_names); } else { return Err(anyhow::anyhow!("Failed to install SDK package(s).")); } @@ -520,14 +573,52 @@ $DNF_SDK_HOST \ print_success("No dependencies configured.", OutputLevel::Normal); } + // Query all SDK packages at once (bootstrap + dependencies) + // This is done after environment-setup is sourced for reliability + if !all_sdk_package_names.is_empty() { + let installed_versions = container_helper + .query_installed_packages( + &SysrootType::Sdk, + &all_sdk_package_names, + container_image, + &target, + repo_url.clone(), + repo_release.clone(), + merged_container_args.clone(), + ) + .await?; + + if !installed_versions.is_empty() { + lock_file.update_sysroot_versions(&target, &SysrootType::Sdk, installed_versions); + if self.verbose { + print_info( + &format!( + "Updated lock file with {} SDK package versions.", + all_sdk_package_names.len() + ), + OutputLevel::Normal, + ); + } + // Save lock file immediately after SDK install + lock_file.save(&src_dir)?; + } + } + // Install rootfs sysroot with version from distro.version print_info("Installing rootfs sysroot.", OutputLevel::Normal); - let rootfs_pkg = if let Some(version) = config.get_distro_version() { - format!("avocado-pkg-rootfs-{}", version) - } else { - "avocado-pkg-rootfs".to_string() - }; + let rootfs_base_pkg = "avocado-pkg-rootfs"; + let rootfs_config_version = config + .get_distro_version() + .map(|s| s.as_str()) + .unwrap_or("*"); + let rootfs_pkg = build_package_spec_with_lock( + &lock_file, + &target, + &SysrootType::Rootfs, + rootfs_base_pkg, + rootfs_config_version, + ); let yes = if self.force { "-y" } else { "" }; let dnf_args_str = if let Some(args) = &self.dnf_args { @@ -564,6 +655,35 @@ $DNF_SDK_HOST $DNF_NO_SCRIPTS $DNF_SDK_TARGET_REPO_CONF \ if rootfs_success { print_success("Installed rootfs sysroot.", OutputLevel::Normal); + + // Query installed version and update lock file + let installed_versions = container_helper + .query_installed_packages( + &SysrootType::Rootfs, + &[rootfs_base_pkg.to_string()], + container_image, + &target, + repo_url.clone(), + repo_release.clone(), + merged_container_args.clone(), + ) + .await?; + + if !installed_versions.is_empty() { + lock_file.update_sysroot_versions( + &target, + &SysrootType::Rootfs, + installed_versions, + ); + if self.verbose { + print_info( + "Updated lock file with rootfs package version.", + OutputLevel::Normal, + ); + } + // Save lock file immediately after rootfs install + lock_file.save(&src_dir)?; + } } else { return Err(anyhow::anyhow!("Failed to install rootfs sysroot.")); } @@ -572,16 +692,25 @@ $DNF_SDK_HOST $DNF_NO_SCRIPTS $DNF_SDK_TARGET_REPO_CONF \ // This aggregates all dependencies from all compile sections (main config + external extensions) let compile_dependencies = config.get_compile_dependencies(); if !compile_dependencies.is_empty() { - // Aggregate all compile dependencies into a single list + // Aggregate all compile dependencies into a single list (with lock file support) let mut all_compile_packages: Vec = Vec::new(); + let mut all_compile_package_names: Vec = Vec::new(); for dependencies in compile_dependencies.values() { - let packages = self.build_package_list(dependencies); + let packages = self.build_package_list_with_lock( + dependencies, + &lock_file, + &target, + &SysrootType::TargetSysroot, + ); all_compile_packages.extend(packages); + all_compile_package_names.extend(self.extract_package_names(dependencies)); } // Deduplicate packages all_compile_packages.sort(); all_compile_packages.dedup(); + all_compile_package_names.sort(); + all_compile_package_names.dedup(); print_info( &format!( @@ -598,12 +727,19 @@ $DNF_SDK_HOST $DNF_NO_SCRIPTS $DNF_SDK_TARGET_REPO_CONF \ String::new() }; - // Build the target-sysroot package spec with version from distro.version - let target_sysroot_pkg = if let Some(version) = config.get_distro_version() { - format!("avocado-sdk-target-sysroot-{}", version) - } else { - "avocado-sdk-target-sysroot".to_string() - }; + // Build the target-sysroot package spec with version from distro.version (with lock) + let target_sysroot_base_pkg = "avocado-sdk-target-sysroot"; + let target_sysroot_config_version = config + .get_distro_version() + .map(|s| s.as_str()) + .unwrap_or("*"); + let target_sysroot_pkg = build_package_spec_with_lock( + &lock_file, + &target, + &SysrootType::TargetSysroot, + target_sysroot_base_pkg, + target_sysroot_config_version, + ); // Install the target-sysroot with avocado-sdk-target-sysroot plus compile deps let command = format!( @@ -626,7 +762,7 @@ $DNF_SDK_HOST $DNF_NO_SCRIPTS $DNF_SDK_TARGET_REPO_CONF \ target: target.clone(), command, verbose: self.verbose, - source_environment: true, + source_environment: false, // Don't source environment - matches rootfs install behavior interactive: !self.force, repo_url: repo_url.clone(), repo_release: repo_release.clone(), @@ -643,6 +779,38 @@ $DNF_SDK_HOST $DNF_NO_SCRIPTS $DNF_SDK_TARGET_REPO_CONF \ "Installed target-sysroot with compile dependencies.", OutputLevel::Normal, ); + + // Query installed versions and update lock file + let mut packages_to_query = all_compile_package_names; + packages_to_query.push(target_sysroot_base_pkg.to_string()); + + let installed_versions = container_helper + .query_installed_packages( + &SysrootType::TargetSysroot, + &packages_to_query, + container_image, + &target, + repo_url.clone(), + repo_release.clone(), + merged_container_args.clone(), + ) + .await?; + + if !installed_versions.is_empty() { + lock_file.update_sysroot_versions( + &target, + &SysrootType::TargetSysroot, + installed_versions, + ); + if self.verbose { + print_info( + "Updated lock file with target-sysroot package versions.", + OutputLevel::Normal, + ); + } + // Save lock file immediately after target-sysroot install + lock_file.save(&src_dir)?; + } } else { return Err(anyhow::anyhow!( "Failed to install target-sysroot with compile dependencies." @@ -682,30 +850,43 @@ $DNF_SDK_HOST $DNF_NO_SCRIPTS $DNF_SDK_TARGET_REPO_CONF \ Ok(()) } - /// Build a list of packages from dependencies HashMap - fn build_package_list(&self, dependencies: &HashMap) -> Vec { + /// Build a list of packages from dependencies HashMap, using lock file for pinned versions + fn build_package_list_with_lock( + &self, + dependencies: &HashMap, + lock_file: &LockFile, + target: &str, + sysroot: &SysrootType, + ) -> Vec { let mut packages = Vec::new(); for (package_name, version) in dependencies { - match version { - serde_yaml::Value::String(v) if v == "*" => { - packages.push(package_name.clone()); - } - serde_yaml::Value::String(v) => { - packages.push(format!("{package_name}-{v}")); - } - serde_yaml::Value::Mapping(_) => { - // Handle dictionary version format like {'core2_64': '*'} - packages.push(package_name.clone()); - } - _ => { - packages.push(package_name.clone()); - } - } + let config_version = match version { + serde_yaml::Value::String(v) => v.clone(), + serde_yaml::Value::Mapping(_) => "*".to_string(), + _ => "*".to_string(), + }; + + let package_spec = build_package_spec_with_lock( + lock_file, + target, + sysroot, + package_name, + &config_version, + ); + packages.push(package_spec); } packages } + + /// Extract just the package names from a dependencies HashMap + fn extract_package_names( + &self, + dependencies: &HashMap, + ) -> Vec { + dependencies.keys().cloned().collect() + } } #[cfg(test)] @@ -715,8 +896,10 @@ mod tests { use std::collections::HashMap; #[test] - fn test_build_package_list() { + fn test_build_package_list_with_lock() { let cmd = SdkInstallCommand::new("test.yaml".to_string(), false, false, None, None, None); + let lock_file = LockFile::new(); + let target = "qemux86-64"; let mut deps = HashMap::new(); deps.insert("package1".to_string(), Value::String("*".to_string())); @@ -726,7 +909,8 @@ mod tests { serde_yaml::Value::Mapping(serde_yaml::Mapping::new()), ); - let packages = cmd.build_package_list(&deps); + let packages = + cmd.build_package_list_with_lock(&deps, &lock_file, target, &SysrootType::Sdk); assert_eq!(packages.len(), 3); assert!(packages.contains(&"package1".to_string())); @@ -734,6 +918,35 @@ mod tests { assert!(packages.contains(&"package3".to_string())); } + #[test] + fn test_build_package_list_with_lock_uses_locked_version() { + let cmd = SdkInstallCommand::new("test.yaml".to_string(), false, false, None, None, None); + let mut lock_file = LockFile::new(); + let target = "qemux86-64"; + + // Add a locked version for package1 + lock_file.update_sysroot_versions( + target, + &SysrootType::Sdk, + [("package1".to_string(), "2.0.0-r0.x86_64".to_string())] + .into_iter() + .collect(), + ); + + let mut deps = HashMap::new(); + deps.insert("package1".to_string(), Value::String("*".to_string())); + deps.insert("package2".to_string(), Value::String("1.0.0".to_string())); + + let packages = + cmd.build_package_list_with_lock(&deps, &lock_file, target, &SysrootType::Sdk); + + assert_eq!(packages.len(), 2); + // package1 should use locked version instead of "*" + assert!(packages.contains(&"package1-2.0.0-r0.x86_64".to_string())); + // package2 has no lock entry, uses config version + assert!(packages.contains(&"package2-1.0.0".to_string())); + } + #[test] fn test_new() { let cmd = SdkInstallCommand::new( diff --git a/src/utils/container.rs b/src/utils/container.rs index 7ce4f2f..cdf696f 100644 --- a/src/utils/container.rs +++ b/src/utils/container.rs @@ -415,6 +415,111 @@ impl SdkContainer { } } + /// Query installed package versions from a sysroot using rpm -q + /// + /// This runs an rpm query command inside the container to get the actual + /// installed versions of packages. Used for lock file generation. + /// + /// # Arguments + /// * `sysroot` - The sysroot type to query + /// * `packages` - List of package names to query + /// * `container_image` - Container image to use + /// * `target` - Target architecture + /// * `repo_url` - Optional repository URL + /// * `repo_release` - Optional repository release + /// * `container_args` - Optional additional container arguments + /// + /// # Returns + /// A HashMap of package name to version string (NEVRA format without name prefix) + #[allow(clippy::too_many_arguments)] + pub async fn query_installed_packages( + &self, + sysroot: &crate::utils::lockfile::SysrootType, + packages: &[String], + container_image: &str, + target: &str, + repo_url: Option, + repo_release: Option, + container_args: Option>, + ) -> Result> { + if packages.is_empty() { + return Ok(std::collections::HashMap::new()); + } + + let rpm_config = sysroot.get_rpm_query_config(); + let query_command = rpm_config.build_query_command(packages); + + if self.verbose { + print_info( + &format!( + "Querying installed packages for lock file (sysroot: {:?}): {}", + sysroot, query_command + ), + OutputLevel::Normal, + ); + } + + let run_config = RunConfig { + container_image: container_image.to_string(), + target: target.to_string(), + command: query_command, + verbose: self.verbose, + // Don't source environment-setup for RPM queries - we only need the + // basic env vars which are set in the entrypoint, not the full SDK env + source_environment: false, + use_entrypoint: true, + interactive: false, + repo_url, + repo_release, + container_args, + ..Default::default() + }; + + match self.run_in_container_with_output(run_config).await? { + Some(output) => { + let versions = crate::utils::lockfile::parse_rpm_query_output(&output); + if self.verbose { + print_info( + &format!( + "Found {} installed package versions for lock file", + versions.len() + ), + OutputLevel::Normal, + ); + for (name, version) in &versions { + print_info(&format!(" {} = {}", name, version), OutputLevel::Normal); + } + } + // Warn if we expected packages but got none (likely a parse or query issue) + if versions.is_empty() && !packages.is_empty() && self.verbose { + print_info( + &format!( + "Warning: RPM query returned no parseable packages. Raw output: {}", + if output.len() > 200 { + format!("{}...", &output[..200]) + } else { + output + } + ), + OutputLevel::Normal, + ); + } + Ok(versions) + } + None => { + // Command failed - this is important for lock file accuracy, so always warn + print_info( + &format!( + "Warning: RPM query for lock file failed for packages: {}", + packages.join(", ") + ), + OutputLevel::Normal, + ); + Ok(std::collections::HashMap::new()) + } + } + } + /// Execute the container command async fn execute_container_command( &self, diff --git a/src/utils/lockfile.rs b/src/utils/lockfile.rs new file mode 100644 index 0000000..3fe4d0b --- /dev/null +++ b/src/utils/lockfile.rs @@ -0,0 +1,1033 @@ +//! Lock file utilities for reproducible DNF package installations. +//! +//! This module provides functionality to track and pin package versions +//! across different sysroots to ensure reproducible builds. + +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; + +/// Current lock file format version +const LOCKFILE_VERSION: u32 = 1; + +/// Lock file name +const LOCKFILE_NAME: &str = "lock.json"; + +/// Lock file directory within src_dir +const LOCKFILE_DIR: &str = ".avocado"; + +/// Represents different sysroot types for package installation +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum SysrootType { + /// SDK sysroot ($AVOCADO_SDK_PREFIX) + Sdk, + /// Rootfs sysroot ($AVOCADO_PREFIX/rootfs) + Rootfs, + /// Target sysroot ($AVOCADO_SDK_PREFIX/target-sysroot) + TargetSysroot, + /// Local/external extension sysroot ($AVOCADO_EXT_SYSROOTS/{name}) + /// Uses ext-rpm-config-scripts for RPM database + Extension(String), + /// Versioned extension sysroot ($AVOCADO_EXT_SYSROOTS/{name}) + /// Uses ext-rpm-config for RPM database (different location than local extensions) + VersionedExtension(String), + /// Runtime sysroot ($AVOCADO_PREFIX/runtimes/{name}) + Runtime(String), +} + +impl SysrootType { + /// Convert sysroot type to its string key for the lock file + pub fn to_key(&self) -> String { + match self { + SysrootType::Sdk => "sdk".to_string(), + SysrootType::Rootfs => "rootfs".to_string(), + SysrootType::TargetSysroot => "target-sysroot".to_string(), + // Both Extension and VersionedExtension use the same key format + // They're distinguished at query time but stored the same in lock file + SysrootType::Extension(name) | SysrootType::VersionedExtension(name) => { + format!("extensions/{}", name) + } + SysrootType::Runtime(name) => format!("runtimes/{}", name), + } + } + + /// Parse a string key back to a SysrootType + #[allow(dead_code)] + pub fn from_key(key: &str) -> Option { + match key { + "sdk" => Some(SysrootType::Sdk), + "rootfs" => Some(SysrootType::Rootfs), + "target-sysroot" => Some(SysrootType::TargetSysroot), + _ if key.starts_with("extensions/") => Some(SysrootType::Extension( + key.strip_prefix("extensions/")?.to_string(), + )), + _ if key.starts_with("runtimes/") => Some(SysrootType::Runtime( + key.strip_prefix("runtimes/")?.to_string(), + )), + _ => None, + } + } + + /// Get the RPM query command environment and root path for this sysroot type + /// Returns (rpm_etcconfigdir, rpm_configdir, root_path) as shell variable expressions + /// + /// For SDK packages, root_path is None because SDK packages are installed into + /// the native container root but tracked via custom RPM_CONFIGDIR macros that + /// point to $AVOCADO_SDK_PREFIX/var/lib/rpm. + pub fn get_rpm_query_config(&self) -> RpmQueryConfig { + match self { + SysrootType::Sdk => RpmQueryConfig { + // SDK needs custom RPM config to find the SDK's RPM database + rpm_etcconfigdir: Some("$AVOCADO_SDK_PREFIX".to_string()), + rpm_configdir: Some("$AVOCADO_SDK_PREFIX/usr/lib/rpm".to_string()), + // No --root flag - SDK packages use native root with custom RPM_CONFIGDIR + root_path: None, + }, + SysrootType::Rootfs => RpmQueryConfig { + // For installroots, we don't need RPM_ETCCONFIGDIR - the --root flag is sufficient + // Setting it can interfere with the query by pointing to the wrong rpmrc + rpm_etcconfigdir: None, + rpm_configdir: None, + root_path: Some("$AVOCADO_PREFIX/rootfs".to_string()), + }, + SysrootType::TargetSysroot => RpmQueryConfig { + // Target-sysroot: same approach as rootfs - unset config and use --root + rpm_etcconfigdir: None, + rpm_configdir: None, + root_path: Some("$AVOCADO_SDK_PREFIX/target-sysroot".to_string()), + }, + SysrootType::Extension(name) => RpmQueryConfig { + // Local/external extensions use ext-rpm-config-scripts + // The database is at standard location, so --root is sufficient + rpm_etcconfigdir: None, + rpm_configdir: None, + root_path: Some(format!("$AVOCADO_EXT_SYSROOTS/{}", name)), + }, + SysrootType::VersionedExtension(name) => RpmQueryConfig { + // Versioned extensions use ext-rpm-config which puts database at custom location + // We need to set RPM_CONFIGDIR to find the database correctly + rpm_etcconfigdir: None, + rpm_configdir: Some("$AVOCADO_SDK_PREFIX/ext-rpm-config".to_string()), + root_path: Some(format!("$AVOCADO_EXT_SYSROOTS/{}", name)), + }, + SysrootType::Runtime(name) => RpmQueryConfig { + // Runtime: same approach as rootfs - unset config and use --root + rpm_etcconfigdir: None, + rpm_configdir: None, + root_path: Some(format!("$AVOCADO_PREFIX/runtimes/{}", name)), + }, + } + } +} + +/// Configuration for RPM query command +#[derive(Debug, Clone)] +pub struct RpmQueryConfig { + /// RPM_ETCCONFIGDIR environment variable value (optional - only needed for SDK) + pub rpm_etcconfigdir: Option, + /// RPM_CONFIGDIR environment variable value (optional) + pub rpm_configdir: Option, + /// Root path for --root flag (None means query native/default database) + pub root_path: Option, +} + +impl RpmQueryConfig { + /// Build the rpm -q command with proper environment and flags + pub fn build_query_command(&self, packages: &[String]) -> String { + // Build rpm command with query format + // Output format: NAME VERSION-RELEASE.ARCH + // Note: We append "|| true" because the entrypoint uses "set -e" and rpm -q + // returns non-zero if ANY package is not found, which would cause the script + // to exit. We want to get partial results even if some packages aren't found. + + if let Some(ref root_path) = self.root_path { + // For installroot queries, we use a subshell to control env vars precisely + // The container entrypoint sets RPM_ETCCONFIGDIR and RPM_CONFIGDIR to SDK values + // which can interfere with --root queries, so we need to override them. + + let mut env_setup = String::new(); + + // Build the environment variable setup + // We unset both first, then set only what we need + env_setup.push_str("unset RPM_ETCCONFIGDIR RPM_CONFIGDIR; "); + + if let Some(ref etcconfigdir) = self.rpm_etcconfigdir { + env_setup.push_str(&format!("export RPM_ETCCONFIGDIR=\"{}\"; ", etcconfigdir)); + } + if let Some(ref configdir) = self.rpm_configdir { + env_setup.push_str(&format!("export RPM_CONFIGDIR=\"{}\"; ", configdir)); + } + + format!( + "({}rpm -q --root=\"{}\" --qf '%{{NAME}} %{{VERSION}}-%{{RELEASE}}.%{{ARCH}}\\n' {}) || true", + env_setup, + root_path, + packages.join(" ") + ) + } else { + // For SDK native queries (no --root), set the custom RPM config paths + let mut cmd = String::new(); + if let Some(ref etcconfigdir) = self.rpm_etcconfigdir { + cmd.push_str(&format!("RPM_ETCCONFIGDIR=\"{}\" ", etcconfigdir)); + } + if let Some(ref configdir) = self.rpm_configdir { + cmd.push_str(&format!("RPM_CONFIGDIR=\"{}\" ", configdir)); + } + cmd.push_str(&format!( + "rpm -q --qf '%{{NAME}} %{{VERSION}}-%{{RELEASE}}.%{{ARCH}}\\n' {} || true", + packages.join(" ") + )); + cmd + } + } +} + +/// Lock file structure for tracking installed package versions +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LockFile { + /// Lock file format version + pub version: u32, + /// Package versions organized by target architecture, then by sysroot + /// Structure: targets -> target_name -> sysroot_key -> package_name -> version + /// Example: targets["qemux86-64"]["sdk"]["avocado-sdk-toolchain"] = "0.1.0-r0.x86_64" + pub targets: HashMap>>, +} + +impl Default for LockFile { + fn default() -> Self { + Self::new() + } +} + +impl LockFile { + /// Create a new empty lock file + pub fn new() -> Self { + Self { + version: LOCKFILE_VERSION, + targets: HashMap::new(), + } + } + + /// Get the lock file path for a given src_dir + pub fn get_path(src_dir: &Path) -> PathBuf { + src_dir.join(LOCKFILE_DIR).join(LOCKFILE_NAME) + } + + /// Load lock file from disk, or return a new one if it doesn't exist + pub fn load(src_dir: &Path) -> Result { + let path = Self::get_path(src_dir); + + if !path.exists() { + return Ok(Self::new()); + } + + let content = fs::read_to_string(&path) + .with_context(|| format!("Failed to read lock file: {}", path.display()))?; + + let lock_file: LockFile = serde_json::from_str(&content) + .with_context(|| format!("Failed to parse lock file: {}", path.display()))?; + + // Check version compatibility + if lock_file.version > LOCKFILE_VERSION { + anyhow::bail!( + "Lock file version {} is newer than supported version {}. Please upgrade avocado-cli.", + lock_file.version, + LOCKFILE_VERSION + ); + } + + Ok(lock_file) + } + + /// Save lock file to disk + pub fn save(&self, src_dir: &Path) -> Result<()> { + let path = Self::get_path(src_dir); + + // Ensure the .avocado directory exists + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).with_context(|| { + format!("Failed to create lock file directory: {}", parent.display()) + })?; + } + + let content = + serde_json::to_string_pretty(self).with_context(|| "Failed to serialize lock file")?; + + fs::write(&path, content) + .with_context(|| format!("Failed to write lock file: {}", path.display()))?; + + Ok(()) + } + + /// Get the locked version for a package in a specific target and sysroot + pub fn get_locked_version( + &self, + target: &str, + sysroot: &SysrootType, + package: &str, + ) -> Option<&String> { + let sysroot_key = sysroot.to_key(); + self.targets + .get(target) + .and_then(|sysroots| sysroots.get(&sysroot_key)) + .and_then(|packages| packages.get(package)) + } + + /// Set the locked version for a package in a specific target and sysroot + #[allow(dead_code)] + pub fn set_locked_version( + &mut self, + target: &str, + sysroot: &SysrootType, + package: &str, + version: &str, + ) { + let sysroot_key = sysroot.to_key(); + self.targets + .entry(target.to_string()) + .or_default() + .entry(sysroot_key) + .or_default() + .insert(package.to_string(), version.to_string()); + } + + /// Update multiple package versions for a target and sysroot at once + pub fn update_sysroot_versions( + &mut self, + target: &str, + sysroot: &SysrootType, + versions: HashMap, + ) { + let sysroot_key = sysroot.to_key(); + let entry = self + .targets + .entry(target.to_string()) + .or_default() + .entry(sysroot_key) + .or_default(); + for (package, version) in versions { + entry.insert(package, version); + } + } + + /// Get all locked versions for a target and sysroot + #[allow(dead_code)] + pub fn get_sysroot_versions( + &self, + target: &str, + sysroot: &SysrootType, + ) -> Option<&HashMap> { + let sysroot_key = sysroot.to_key(); + self.targets + .get(target) + .and_then(|sysroots| sysroots.get(&sysroot_key)) + } + + /// Check if the lock file has any entries + pub fn is_empty(&self) -> bool { + self.targets.is_empty() + || self.targets.values().all(|sysroots| { + sysroots.is_empty() || sysroots.values().all(|packages| packages.is_empty()) + }) + } +} + +/// Parse rpm -q output into a map of package names to versions +/// Expected format: "NAME VERSION-RELEASE.ARCH" per line +pub fn parse_rpm_query_output(output: &str) -> HashMap { + let mut result = HashMap::new(); + + for line in output.lines() { + let line = line.trim(); + if line.is_empty() { + continue; + } + + // Skip lines that indicate package not installed + if line.contains("is not installed") { + continue; + } + + // Skip info/error/debug lines from container output + if line.starts_with("[INFO]") + || line.starts_with("[ERROR]") + || line.starts_with("[SUCCESS]") + || line.starts_with("[DEBUG]") + || line.starts_with("[WARNING]") + { + continue; + } + + // Split on first space: NAME VERSION-RELEASE.ARCH + if let Some((name, version)) = line.split_once(' ') { + // Additional validation: package names shouldn't contain brackets or special chars + if name.starts_with('[') || name.contains('=') { + continue; + } + result.insert(name.to_string(), version.to_string()); + } + } + + result +} + +/// Build a package specification for DNF install, using locked version if available +/// Returns the package spec string (e.g., "curl" or "curl-7.88.1-r0.core2_64") +pub fn build_package_spec_with_lock( + lock_file: &LockFile, + target: &str, + sysroot: &SysrootType, + package_name: &str, + config_version: &str, +) -> String { + // First, check if we have a locked version for this target + if let Some(locked_version) = lock_file.get_locked_version(target, sysroot, package_name) { + // Use the full locked version (NEVRA format) + format!("{}-{}", package_name, locked_version) + } else if config_version == "*" { + // No lock and config says latest - just use package name + package_name.to_string() + } else { + // No lock but config specifies a version + format!("{}-{}", package_name, config_version) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[test] + fn test_sysroot_type_to_key() { + assert_eq!(SysrootType::Sdk.to_key(), "sdk"); + assert_eq!(SysrootType::Rootfs.to_key(), "rootfs"); + assert_eq!(SysrootType::TargetSysroot.to_key(), "target-sysroot"); + assert_eq!( + SysrootType::Extension("my-app".to_string()).to_key(), + "extensions/my-app" + ); + // VersionedExtension and Extension produce the same key format + // They're only distinguished at query time, not in the lock file + assert_eq!( + SysrootType::VersionedExtension("my-versioned-app".to_string()).to_key(), + "extensions/my-versioned-app" + ); + assert_eq!( + SysrootType::Runtime("dev".to_string()).to_key(), + "runtimes/dev" + ); + } + + #[test] + fn test_sysroot_type_from_key() { + assert_eq!(SysrootType::from_key("sdk"), Some(SysrootType::Sdk)); + assert_eq!(SysrootType::from_key("rootfs"), Some(SysrootType::Rootfs)); + assert_eq!( + SysrootType::from_key("target-sysroot"), + Some(SysrootType::TargetSysroot) + ); + assert_eq!( + SysrootType::from_key("extensions/my-app"), + Some(SysrootType::Extension("my-app".to_string())) + ); + assert_eq!( + SysrootType::from_key("runtimes/dev"), + Some(SysrootType::Runtime("dev".to_string())) + ); + assert_eq!(SysrootType::from_key("invalid"), None); + } + + #[test] + fn test_lock_file_new() { + let lock = LockFile::new(); + assert_eq!(lock.version, LOCKFILE_VERSION); + assert!(lock.targets.is_empty()); + } + + #[test] + fn test_lock_file_get_set_version() { + let mut lock = LockFile::new(); + let target = "qemux86-64"; + + lock.set_locked_version(target, &SysrootType::Sdk, "test-package", "1.0.0-r0.x86_64"); + + assert_eq!( + lock.get_locked_version(target, &SysrootType::Sdk, "test-package"), + Some(&"1.0.0-r0.x86_64".to_string()) + ); + + assert_eq!( + lock.get_locked_version(target, &SysrootType::Sdk, "nonexistent"), + None + ); + + assert_eq!( + lock.get_locked_version(target, &SysrootType::Rootfs, "test-package"), + None + ); + + // Different target should not have the package + assert_eq!( + lock.get_locked_version("qemuarm64", &SysrootType::Sdk, "test-package"), + None + ); + } + + #[test] + fn test_lock_file_save_load() { + let temp_dir = TempDir::new().unwrap(); + let src_dir = temp_dir.path(); + let target = "qemux86-64"; + + let mut lock = LockFile::new(); + lock.set_locked_version( + target, + &SysrootType::Extension("my-app".to_string()), + "curl", + "7.88.1-r0.core2_64", + ); + + lock.save(src_dir).unwrap(); + + let loaded = LockFile::load(src_dir).unwrap(); + assert_eq!(loaded.version, LOCKFILE_VERSION); + assert_eq!( + loaded.get_locked_version( + target, + &SysrootType::Extension("my-app".to_string()), + "curl" + ), + Some(&"7.88.1-r0.core2_64".to_string()) + ); + } + + #[test] + fn test_lock_file_load_nonexistent() { + let temp_dir = TempDir::new().unwrap(); + let lock = LockFile::load(temp_dir.path()).unwrap(); + assert!(lock.is_empty()); + } + + #[test] + fn test_parse_rpm_query_output() { + let output = r#"curl 7.88.1-r0.core2_64 +openssl 3.0.8-r0.core2_64 +package-xyz is not installed +wget 1.21-r0.core2_64 +"#; + + let result = parse_rpm_query_output(output); + assert_eq!(result.len(), 3); + assert_eq!(result.get("curl"), Some(&"7.88.1-r0.core2_64".to_string())); + assert_eq!( + result.get("openssl"), + Some(&"3.0.8-r0.core2_64".to_string()) + ); + assert_eq!(result.get("wget"), Some(&"1.21-r0.core2_64".to_string())); + assert_eq!(result.get("package-xyz"), None); + } + + #[test] + fn test_build_package_spec_with_lock() { + let mut lock = LockFile::new(); + let target = "qemux86-64"; + lock.set_locked_version(target, &SysrootType::Sdk, "curl", "7.88.1-r0.x86_64"); + + // Should use locked version + assert_eq!( + build_package_spec_with_lock(&lock, target, &SysrootType::Sdk, "curl", "*"), + "curl-7.88.1-r0.x86_64" + ); + + // No lock, config says latest + assert_eq!( + build_package_spec_with_lock(&lock, target, &SysrootType::Sdk, "wget", "*"), + "wget" + ); + + // No lock, config specifies version + assert_eq!( + build_package_spec_with_lock(&lock, target, &SysrootType::Sdk, "wget", "1.21"), + "wget-1.21" + ); + + // Different target should not have curl locked + assert_eq!( + build_package_spec_with_lock(&lock, "qemuarm64", &SysrootType::Sdk, "curl", "*"), + "curl" + ); + } + + #[test] + fn test_rpm_query_config_build_command() { + // Test with root_path (for installroot sysroot queries) + // These must explicitly UNSET the env vars to override entrypoint settings + let config = RpmQueryConfig { + rpm_etcconfigdir: None, + rpm_configdir: None, + root_path: Some("$AVOCADO_EXT_SYSROOTS/my-ext".to_string()), + }; + + let cmd = config.build_query_command(&["curl".to_string(), "wget".to_string()]); + // Should use a subshell with unset to properly remove env vars + assert!(cmd.contains("unset RPM_ETCCONFIGDIR RPM_CONFIGDIR")); + assert!(cmd.contains("--root=\"$AVOCADO_EXT_SYSROOTS/my-ext\"")); + assert!(cmd.contains("curl wget")); + + // Test without root_path (for SDK native queries) + let sdk_config = RpmQueryConfig { + rpm_etcconfigdir: Some("$AVOCADO_SDK_PREFIX".to_string()), + rpm_configdir: Some("$AVOCADO_SDK_PREFIX/usr/lib/rpm".to_string()), + root_path: None, + }; + + let sdk_cmd = sdk_config.build_query_command(&["pkg1".to_string(), "pkg2".to_string()]); + assert!(sdk_cmd.contains("RPM_ETCCONFIGDIR=\"$AVOCADO_SDK_PREFIX\"")); + assert!(sdk_cmd.contains("RPM_CONFIGDIR=\"$AVOCADO_SDK_PREFIX/usr/lib/rpm\"")); + assert!(!sdk_cmd.contains("--root")); + assert!(sdk_cmd.contains("pkg1 pkg2")); + } + + #[test] + fn test_update_sysroot_versions() { + let mut lock = LockFile::new(); + let target = "qemux86-64"; + + let mut versions = HashMap::new(); + versions.insert("pkg1".to_string(), "1.0.0-r0.x86_64".to_string()); + versions.insert("pkg2".to_string(), "2.0.0-r0.x86_64".to_string()); + + lock.update_sysroot_versions(target, &SysrootType::Sdk, versions); + + assert_eq!( + lock.get_locked_version(target, &SysrootType::Sdk, "pkg1"), + Some(&"1.0.0-r0.x86_64".to_string()) + ); + assert_eq!( + lock.get_locked_version(target, &SysrootType::Sdk, "pkg2"), + Some(&"2.0.0-r0.x86_64".to_string()) + ); + } + + #[test] + fn test_multiple_targets() { + let mut lock = LockFile::new(); + + // Set versions for two different targets + lock.set_locked_version("qemux86-64", &SysrootType::Sdk, "curl", "7.88.1-r0.x86_64"); + lock.set_locked_version("qemuarm64", &SysrootType::Sdk, "curl", "7.88.1-r0.aarch64"); + + // Each target should have its own version + assert_eq!( + lock.get_locked_version("qemux86-64", &SysrootType::Sdk, "curl"), + Some(&"7.88.1-r0.x86_64".to_string()) + ); + assert_eq!( + lock.get_locked_version("qemuarm64", &SysrootType::Sdk, "curl"), + Some(&"7.88.1-r0.aarch64".to_string()) + ); + } + + #[test] + fn test_is_empty() { + let lock = LockFile::new(); + assert!(lock.is_empty()); + + let mut lock = LockFile::new(); + lock.set_locked_version("qemux86-64", &SysrootType::Sdk, "curl", "7.88.1-r0.x86_64"); + assert!(!lock.is_empty()); + } + + #[test] + fn test_lock_file_path() { + let path = LockFile::get_path(std::path::Path::new("/home/user/project")); + assert_eq!( + path, + std::path::PathBuf::from("/home/user/project/.avocado/lock.json") + ); + } + + #[test] + fn test_multiple_sysroots_same_target() { + let mut lock = LockFile::new(); + let target = "qemux86-64"; + + // Set versions for different sysroots under the same target + lock.set_locked_version(target, &SysrootType::Sdk, "toolchain", "1.0.0-r0.x86_64"); + lock.set_locked_version( + target, + &SysrootType::Rootfs, + "base-files", + "3.0.0-r0.core2_64", + ); + lock.set_locked_version( + target, + &SysrootType::Extension("my-app".to_string()), + "libfoo", + "2.0.0-r0.core2_64", + ); + lock.set_locked_version( + target, + &SysrootType::Runtime("dev".to_string()), + "runtime-base", + "1.5.0-r0.core2_64", + ); + lock.set_locked_version( + target, + &SysrootType::TargetSysroot, + "glibc", + "2.37-r0.core2_64", + ); + + // Verify each sysroot has its package + assert_eq!( + lock.get_locked_version(target, &SysrootType::Sdk, "toolchain"), + Some(&"1.0.0-r0.x86_64".to_string()) + ); + assert_eq!( + lock.get_locked_version(target, &SysrootType::Rootfs, "base-files"), + Some(&"3.0.0-r0.core2_64".to_string()) + ); + assert_eq!( + lock.get_locked_version( + target, + &SysrootType::Extension("my-app".to_string()), + "libfoo" + ), + Some(&"2.0.0-r0.core2_64".to_string()) + ); + assert_eq!( + lock.get_locked_version( + target, + &SysrootType::Runtime("dev".to_string()), + "runtime-base" + ), + Some(&"1.5.0-r0.core2_64".to_string()) + ); + assert_eq!( + lock.get_locked_version(target, &SysrootType::TargetSysroot, "glibc"), + Some(&"2.37-r0.core2_64".to_string()) + ); + + // Verify cross-sysroot isolation + assert_eq!( + lock.get_locked_version(target, &SysrootType::Sdk, "base-files"), + None + ); + } + + #[test] + fn test_version_update_overwrites() { + let mut lock = LockFile::new(); + let target = "qemux86-64"; + + lock.set_locked_version(target, &SysrootType::Sdk, "curl", "7.88.0-r0.x86_64"); + assert_eq!( + lock.get_locked_version(target, &SysrootType::Sdk, "curl"), + Some(&"7.88.0-r0.x86_64".to_string()) + ); + + // Update to new version + lock.set_locked_version(target, &SysrootType::Sdk, "curl", "7.88.1-r0.x86_64"); + assert_eq!( + lock.get_locked_version(target, &SysrootType::Sdk, "curl"), + Some(&"7.88.1-r0.x86_64".to_string()) + ); + } + + #[test] + fn test_update_sysroot_versions_merges() { + let mut lock = LockFile::new(); + let target = "qemux86-64"; + + // Add initial packages + let mut versions1 = HashMap::new(); + versions1.insert("pkg1".to_string(), "1.0.0-r0.x86_64".to_string()); + lock.update_sysroot_versions(target, &SysrootType::Sdk, versions1); + + // Add more packages (should merge, not replace) + let mut versions2 = HashMap::new(); + versions2.insert("pkg2".to_string(), "2.0.0-r0.x86_64".to_string()); + lock.update_sysroot_versions(target, &SysrootType::Sdk, versions2); + + // Both packages should exist + assert_eq!( + lock.get_locked_version(target, &SysrootType::Sdk, "pkg1"), + Some(&"1.0.0-r0.x86_64".to_string()) + ); + assert_eq!( + lock.get_locked_version(target, &SysrootType::Sdk, "pkg2"), + Some(&"2.0.0-r0.x86_64".to_string()) + ); + } + + #[test] + fn test_get_sysroot_versions() { + let mut lock = LockFile::new(); + let target = "qemux86-64"; + + lock.set_locked_version(target, &SysrootType::Sdk, "pkg1", "1.0.0-r0.x86_64"); + lock.set_locked_version(target, &SysrootType::Sdk, "pkg2", "2.0.0-r0.x86_64"); + + let versions = lock.get_sysroot_versions(target, &SysrootType::Sdk); + assert!(versions.is_some()); + let versions = versions.unwrap(); + assert_eq!(versions.len(), 2); + assert_eq!(versions.get("pkg1"), Some(&"1.0.0-r0.x86_64".to_string())); + assert_eq!(versions.get("pkg2"), Some(&"2.0.0-r0.x86_64".to_string())); + + // Non-existent sysroot should return None + assert!(lock + .get_sysroot_versions(target, &SysrootType::Rootfs) + .is_none()); + + // Non-existent target should return None + assert!(lock + .get_sysroot_versions("nonexistent", &SysrootType::Sdk) + .is_none()); + } + + #[test] + fn test_parse_rpm_query_output_edge_cases() { + // Empty output + let result = parse_rpm_query_output(""); + assert!(result.is_empty()); + + // Whitespace only + let result = parse_rpm_query_output(" \n \n "); + assert!(result.is_empty()); + + // Only "not installed" messages + let result = + parse_rpm_query_output("package-a is not installed\npackage-b is not installed"); + assert!(result.is_empty()); + + // Mixed valid and invalid + let output = + "valid-pkg 1.0.0-r0.x86_64\nbad-pkg is not installed\nanother-pkg 2.0.0-r0.x86_64"; + let result = parse_rpm_query_output(output); + assert_eq!(result.len(), 2); + assert!(result.contains_key("valid-pkg")); + assert!(result.contains_key("another-pkg")); + } + + #[test] + fn test_parse_rpm_query_output_filters_info_lines() { + // Output mixed with container info messages + let output = r#"[INFO] Using repo URL: 'http://192.168.1.10:8080' +[INFO] Using repo release: 'latest/apollo/edge' +curl 7.88.1-r0.core2_64 +openssl 3.0.8-r0.core2_64 +[ERROR] Some error message +[SUCCESS] Something succeeded +wget 1.21-r0.core2_64 +[DEBUG] Debug info +[WARNING] Warning message +"#; + + let result = parse_rpm_query_output(output); + assert_eq!(result.len(), 3); + assert_eq!(result.get("curl"), Some(&"7.88.1-r0.core2_64".to_string())); + assert_eq!( + result.get("openssl"), + Some(&"3.0.8-r0.core2_64".to_string()) + ); + assert_eq!(result.get("wget"), Some(&"1.21-r0.core2_64".to_string())); + // These should NOT be in the result + assert_eq!(result.get("[INFO]"), None); + assert_eq!(result.get("[ERROR]"), None); + assert_eq!(result.get("[SUCCESS]"), None); + assert_eq!(result.get("[DEBUG]"), None); + assert_eq!(result.get("[WARNING]"), None); + } + + #[test] + fn test_rpm_query_config_without_configdir() { + // For installroot queries, we must explicitly UNSET env vars to override entrypoint + let config = RpmQueryConfig { + rpm_etcconfigdir: None, + rpm_configdir: None, + root_path: Some("$AVOCADO_PREFIX/rootfs".to_string()), + }; + + let cmd = config.build_query_command(&["curl".to_string()]); + // Should use a subshell with unset to properly remove env vars + assert!(cmd.contains("unset RPM_ETCCONFIGDIR RPM_CONFIGDIR")); + assert!(cmd.contains("--root=\"$AVOCADO_PREFIX/rootfs\"")); + // Command should start with subshell + assert!(cmd.starts_with("(unset")); + } + + #[test] + fn test_sysroot_type_get_rpm_query_config() { + // Test SDK config - no root_path because SDK uses native container with custom RPM_CONFIGDIR + let sdk_config = SysrootType::Sdk.get_rpm_query_config(); + assert_eq!( + sdk_config.rpm_etcconfigdir, + Some("$AVOCADO_SDK_PREFIX".to_string()) + ); + assert!(sdk_config.rpm_configdir.is_some()); + assert!(sdk_config.root_path.is_none()); // SDK doesn't use --root + + // Test Rootfs config - installroots don't need RPM_ETCCONFIGDIR, just --root + let rootfs_config = SysrootType::Rootfs.get_rpm_query_config(); + assert!(rootfs_config.rpm_etcconfigdir.is_none()); + assert!(rootfs_config.rpm_configdir.is_none()); + assert_eq!( + rootfs_config.root_path, + Some("$AVOCADO_PREFIX/rootfs".to_string()) + ); + + // Test Extension config - local/external extensions don't need RPM_CONFIGDIR + let ext_config = SysrootType::Extension("my-ext".to_string()).get_rpm_query_config(); + assert!(ext_config.rpm_etcconfigdir.is_none()); + assert!(ext_config.rpm_configdir.is_none()); + assert_eq!( + ext_config.root_path, + Some("$AVOCADO_EXT_SYSROOTS/my-ext".to_string()) + ); + + // Test VersionedExtension config - needs RPM_CONFIGDIR for ext-rpm-config database location + let versioned_ext_config = + SysrootType::VersionedExtension("my-versioned-ext".to_string()).get_rpm_query_config(); + assert!(versioned_ext_config.rpm_etcconfigdir.is_none()); + assert_eq!( + versioned_ext_config.rpm_configdir, + Some("$AVOCADO_SDK_PREFIX/ext-rpm-config".to_string()) + ); + assert_eq!( + versioned_ext_config.root_path, + Some("$AVOCADO_EXT_SYSROOTS/my-versioned-ext".to_string()) + ); + + // Test Runtime config - same as rootfs, no custom config needed + let runtime_config = SysrootType::Runtime("dev".to_string()).get_rpm_query_config(); + assert!(runtime_config.rpm_etcconfigdir.is_none()); + assert!(runtime_config.rpm_configdir.is_none()); + assert_eq!( + runtime_config.root_path, + Some("$AVOCADO_PREFIX/runtimes/dev".to_string()) + ); + + // Test TargetSysroot config - same as rootfs, no custom config needed + let target_config = SysrootType::TargetSysroot.get_rpm_query_config(); + assert!(target_config.rpm_etcconfigdir.is_none()); + assert!(target_config.rpm_configdir.is_none()); + assert_eq!( + target_config.root_path, + Some("$AVOCADO_SDK_PREFIX/target-sysroot".to_string()) + ); + } + + #[test] + fn test_lock_file_json_format() { + let mut lock = LockFile::new(); + lock.set_locked_version("qemux86-64", &SysrootType::Sdk, "curl", "7.88.1-r0.x86_64"); + lock.set_locked_version( + "qemux86-64", + &SysrootType::Extension("app".to_string()), + "libfoo", + "1.0.0-r0.core2_64", + ); + lock.set_locked_version("qemuarm64", &SysrootType::Sdk, "curl", "7.88.1-r0.aarch64"); + + let json = serde_json::to_string_pretty(&lock).unwrap(); + + // Verify JSON structure + let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed["version"], 1); + assert!(parsed["targets"]["qemux86-64"]["sdk"]["curl"].is_string()); + assert!(parsed["targets"]["qemux86-64"]["extensions/app"]["libfoo"].is_string()); + assert!(parsed["targets"]["qemuarm64"]["sdk"]["curl"].is_string()); + } + + #[test] + fn test_lock_file_persistence_multiple_targets() { + let temp_dir = TempDir::new().unwrap(); + let src_dir = temp_dir.path(); + + // Create lock file with multiple targets and sysroots + let mut lock = LockFile::new(); + lock.set_locked_version( + "qemux86-64", + &SysrootType::Sdk, + "toolchain", + "1.0.0-r0.x86_64", + ); + lock.set_locked_version( + "qemux86-64", + &SysrootType::Rootfs, + "base", + "1.0.0-r0.core2_64", + ); + lock.set_locked_version( + "qemuarm64", + &SysrootType::Sdk, + "toolchain", + "1.0.0-r0.aarch64", + ); + lock.set_locked_version( + "qemuarm64", + &SysrootType::Extension("app".to_string()), + "libapp", + "2.0.0-r0.cortexa57", + ); + + lock.save(src_dir).unwrap(); + + // Load and verify + let loaded = LockFile::load(src_dir).unwrap(); + + assert_eq!( + loaded.get_locked_version("qemux86-64", &SysrootType::Sdk, "toolchain"), + Some(&"1.0.0-r0.x86_64".to_string()) + ); + assert_eq!( + loaded.get_locked_version("qemux86-64", &SysrootType::Rootfs, "base"), + Some(&"1.0.0-r0.core2_64".to_string()) + ); + assert_eq!( + loaded.get_locked_version("qemuarm64", &SysrootType::Sdk, "toolchain"), + Some(&"1.0.0-r0.aarch64".to_string()) + ); + assert_eq!( + loaded.get_locked_version( + "qemuarm64", + &SysrootType::Extension("app".to_string()), + "libapp" + ), + Some(&"2.0.0-r0.cortexa57".to_string()) + ); + } + + #[test] + fn test_build_package_spec_locked_overrides_config() { + let mut lock = LockFile::new(); + let target = "qemux86-64"; + + // Set a locked version + lock.set_locked_version(target, &SysrootType::Sdk, "curl", "7.88.1-r0.x86_64"); + + // Even if config specifies a different version, locked version should be used + assert_eq!( + build_package_spec_with_lock(&lock, target, &SysrootType::Sdk, "curl", "7.80.0"), + "curl-7.88.1-r0.x86_64" + ); + + // And if config says "*", locked version should still be used + assert_eq!( + build_package_spec_with_lock(&lock, target, &SysrootType::Sdk, "curl", "*"), + "curl-7.88.1-r0.x86_64" + ); + } + + #[test] + fn test_default_impl() { + let lock: LockFile = Default::default(); + assert_eq!(lock.version, LOCKFILE_VERSION); + assert!(lock.targets.is_empty()); + } +} diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 52c78ad..6789f86 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -2,6 +2,7 @@ pub mod config; pub mod container; pub mod image_signing; pub mod interpolation; +pub mod lockfile; pub mod output; pub mod pkcs11_devices; pub mod signing_keys; From d5066b60ec06191b93d31889aee8067ffd119a14 Mon Sep 17 00:00:00 2001 From: Justin Schneck Date: Thu, 25 Dec 2025 23:04:06 -0500 Subject: [PATCH 09/10] use jcs for lockfile storage --- Cargo.lock | 18 +++++ Cargo.toml | 1 + src/utils/container.rs | 4 +- src/utils/lockfile.rs | 153 +++++++++++++++++++++++++++++++++++++---- 4 files changed, 162 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 82f7bce..e04a832 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -149,6 +149,7 @@ dependencies = [ "reqwest", "rpassword", "serde", + "serde_jcs", "serde_json", "serde_yaml", "serial_test", @@ -1409,6 +1410,12 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "ryu-js" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6518fc26bced4d53678a22d6e423e9d8716377def84545fe328236e3af070e7f" + [[package]] name = "same-file" version = "1.0.6" @@ -1478,6 +1485,17 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_jcs" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cacecf649bc1a7c5f0e299cc813977c6a78116abda2b93b1ee01735b71ead9a8" +dependencies = [ + "ryu-js", + "serde", + "serde_json", +] + [[package]] name = "serde_json" version = "1.0.145" diff --git a/Cargo.toml b/Cargo.toml index 51f6455..837b7d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ serde_yaml = "0.9" anyhow = "1.0" clap = { version = "4.0", features = ["derive"] } serde_json = "1.0" +serde_jcs = "0.1" tokio = { version = "1.0", features = [ "macros", "rt-multi-thread", diff --git a/src/utils/container.rs b/src/utils/container.rs index cdf696f..ec607d7 100644 --- a/src/utils/container.rs +++ b/src/utils/container.rs @@ -477,7 +477,9 @@ impl SdkContainer { match self.run_in_container_with_output(run_config).await? { Some(output) => { - let versions = crate::utils::lockfile::parse_rpm_query_output(&output); + // For SDK sysroots, strip architecture to make lock file portable across host architectures + let strip_arch = matches!(sysroot, crate::utils::lockfile::SysrootType::Sdk); + let versions = crate::utils::lockfile::parse_rpm_query_output(&output, strip_arch); if self.verbose { print_info( &format!( diff --git a/src/utils/lockfile.rs b/src/utils/lockfile.rs index 3fe4d0b..03361bb 100644 --- a/src/utils/lockfile.rs +++ b/src/utils/lockfile.rs @@ -241,7 +241,8 @@ impl LockFile { Ok(lock_file) } - /// Save lock file to disk + /// Save lock file to disk using JSON Canonicalization Scheme (RFC 8785) + /// This ensures deterministic output with sorted keys and consistent formatting pub fn save(&self, src_dir: &Path) -> Result<()> { let path = Self::get_path(src_dir); @@ -252,10 +253,14 @@ impl LockFile { })?; } - let content = - serde_json::to_string_pretty(self).with_context(|| "Failed to serialize lock file")?; + // Use JSON Canonicalization Scheme for deterministic output + let content = serde_jcs::to_string(self) + .with_context(|| "Failed to serialize lock file using JCS")?; - fs::write(&path, content) + // Add a newline at the end for better git diffs + let content_with_newline = format!("{}\n", content); + + fs::write(&path, content_with_newline) .with_context(|| format!("Failed to write lock file: {}", path.display()))?; Ok(()) @@ -336,7 +341,10 @@ impl LockFile { /// Parse rpm -q output into a map of package names to versions /// Expected format: "NAME VERSION-RELEASE.ARCH" per line -pub fn parse_rpm_query_output(output: &str) -> HashMap { +/// +/// For SDK packages, we strip the architecture suffix to make the lock file portable +/// across different host architectures (x86_64, aarch64, etc.) +pub fn parse_rpm_query_output(output: &str, strip_arch: bool) -> HashMap { let mut result = HashMap::new(); for line in output.lines() { @@ -366,7 +374,20 @@ pub fn parse_rpm_query_output(output: &str) -> HashMap { if name.starts_with('[') || name.contains('=') { continue; } - result.insert(name.to_string(), version.to_string()); + + let version_to_store = if strip_arch { + // Strip the architecture suffix (.ARCH) from the version + // Format: VERSION-RELEASE.ARCH -> VERSION-RELEASE + if let Some(idx) = version.rfind('.') { + version[..idx].to_string() + } else { + version.to_string() + } + } else { + version.to_string() + }; + + result.insert(name.to_string(), version_to_store); } } @@ -519,7 +540,8 @@ package-xyz is not installed wget 1.21-r0.core2_64 "#; - let result = parse_rpm_query_output(output); + // Test without stripping architecture + let result = parse_rpm_query_output(output, false); assert_eq!(result.len(), 3); assert_eq!(result.get("curl"), Some(&"7.88.1-r0.core2_64".to_string())); assert_eq!( @@ -528,6 +550,16 @@ wget 1.21-r0.core2_64 ); assert_eq!(result.get("wget"), Some(&"1.21-r0.core2_64".to_string())); assert_eq!(result.get("package-xyz"), None); + + // Test with stripping architecture (for SDK packages) + let result_stripped = parse_rpm_query_output(output, true); + assert_eq!(result_stripped.len(), 3); + assert_eq!(result_stripped.get("curl"), Some(&"7.88.1-r0".to_string())); + assert_eq!( + result_stripped.get("openssl"), + Some(&"3.0.8-r0".to_string()) + ); + assert_eq!(result_stripped.get("wget"), Some(&"1.21-r0".to_string())); } #[test] @@ -793,22 +825,24 @@ wget 1.21-r0.core2_64 #[test] fn test_parse_rpm_query_output_edge_cases() { // Empty output - let result = parse_rpm_query_output(""); + let result = parse_rpm_query_output("", false); assert!(result.is_empty()); // Whitespace only - let result = parse_rpm_query_output(" \n \n "); + let result = parse_rpm_query_output(" \n \n ", false); assert!(result.is_empty()); // Only "not installed" messages - let result = - parse_rpm_query_output("package-a is not installed\npackage-b is not installed"); + let result = parse_rpm_query_output( + "package-a is not installed\npackage-b is not installed", + false, + ); assert!(result.is_empty()); // Mixed valid and invalid let output = "valid-pkg 1.0.0-r0.x86_64\nbad-pkg is not installed\nanother-pkg 2.0.0-r0.x86_64"; - let result = parse_rpm_query_output(output); + let result = parse_rpm_query_output(output, false); assert_eq!(result.len(), 2); assert!(result.contains_key("valid-pkg")); assert!(result.contains_key("another-pkg")); @@ -828,7 +862,7 @@ wget 1.21-r0.core2_64 [WARNING] Warning message "#; - let result = parse_rpm_query_output(output); + let result = parse_rpm_query_output(output, false); assert_eq!(result.len(), 3); assert_eq!(result.get("curl"), Some(&"7.88.1-r0.core2_64".to_string())); assert_eq!( @@ -844,6 +878,47 @@ wget 1.21-r0.core2_64 assert_eq!(result.get("[WARNING]"), None); } + #[test] + fn test_parse_rpm_query_output_strips_arch_for_sdk() { + // Test SDK package output with architecture stripping + let output = r#"nativesdk-curl 7.88.1-r0.x86_64_avocadosdk +nativesdk-openssl 3.0.8-r0.x86_64_avocadosdk +avocado-sdk-toolchain 0.1.0-r0.x86_64_avocadosdk +"#; + + // With strip_arch=true (for SDK packages) + let result_stripped = parse_rpm_query_output(output, true); + assert_eq!(result_stripped.len(), 3); + assert_eq!( + result_stripped.get("nativesdk-curl"), + Some(&"7.88.1-r0".to_string()) + ); + assert_eq!( + result_stripped.get("nativesdk-openssl"), + Some(&"3.0.8-r0".to_string()) + ); + assert_eq!( + result_stripped.get("avocado-sdk-toolchain"), + Some(&"0.1.0-r0".to_string()) + ); + + // With strip_arch=false (for non-SDK packages) + let result_full = parse_rpm_query_output(output, false); + assert_eq!(result_full.len(), 3); + assert_eq!( + result_full.get("nativesdk-curl"), + Some(&"7.88.1-r0.x86_64_avocadosdk".to_string()) + ); + assert_eq!( + result_full.get("nativesdk-openssl"), + Some(&"3.0.8-r0.x86_64_avocadosdk".to_string()) + ); + assert_eq!( + result_full.get("avocado-sdk-toolchain"), + Some(&"0.1.0-r0.x86_64_avocadosdk".to_string()) + ); + } + #[test] fn test_rpm_query_config_without_configdir() { // For installroot queries, we must explicitly UNSET env vars to override entrypoint @@ -1030,4 +1105,56 @@ wget 1.21-r0.core2_64 assert_eq!(lock.version, LOCKFILE_VERSION); assert!(lock.targets.is_empty()); } + + #[test] + fn test_jcs_deterministic_output() { + use std::fs; + use tempfile::TempDir; + + let temp_dir = TempDir::new().unwrap(); + let src_dir = temp_dir.path(); + + // Create a lock file with packages in non-alphabetical order + let mut lock1 = LockFile::new(); + lock1.set_locked_version("qemux86-64", &SysrootType::Sdk, "zebra", "1.0.0-r0"); + lock1.set_locked_version("qemux86-64", &SysrootType::Sdk, "alpha", "2.0.0-r0"); + lock1.set_locked_version("qemux86-64", &SysrootType::Rootfs, "beta", "3.0.0-r0"); + lock1.set_locked_version("qemuarm64", &SysrootType::Sdk, "gamma", "4.0.0-r0"); + + lock1.save(src_dir).unwrap(); + let content1 = fs::read_to_string(LockFile::get_path(src_dir)).unwrap(); + + // Create another lock file with same data but added in different order + let mut lock2 = LockFile::new(); + lock2.set_locked_version("qemuarm64", &SysrootType::Sdk, "gamma", "4.0.0-r0"); + lock2.set_locked_version("qemux86-64", &SysrootType::Rootfs, "beta", "3.0.0-r0"); + lock2.set_locked_version("qemux86-64", &SysrootType::Sdk, "alpha", "2.0.0-r0"); + lock2.set_locked_version("qemux86-64", &SysrootType::Sdk, "zebra", "1.0.0-r0"); + + // Remove the first lock file and save the second + fs::remove_file(LockFile::get_path(src_dir)).unwrap(); + lock2.save(src_dir).unwrap(); + let content2 = fs::read_to_string(LockFile::get_path(src_dir)).unwrap(); + + // JCS ensures both produce identical output regardless of insertion order + assert_eq!( + content1, content2, + "JCS should produce identical output regardless of insertion order" + ); + + // Verify keys are sorted (targets should be alphabetically ordered) + assert!( + content1.find("qemuarm64").unwrap() < content1.find("qemux86-64").unwrap(), + "Target keys should be alphabetically sorted" + ); + + // Verify package keys are sorted within each sysroot + let sdk_start = content1.find("\"sdk\"").unwrap(); + let alpha_pos = content1[sdk_start..].find("\"alpha\"").unwrap() + sdk_start; + let zebra_pos = content1[sdk_start..].find("\"zebra\"").unwrap() + sdk_start; + assert!( + alpha_pos < zebra_pos, + "Package keys should be alphabetically sorted" + ); + } } From 8e3b20f0da45bf888d0247031a9b750915e61286 Mon Sep 17 00:00:00 2001 From: Justin Schneck Date: Thu, 25 Dec 2025 22:40:39 -0500 Subject: [PATCH 10/10] 0.20.0 release --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e04a832..d26ef4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -130,7 +130,7 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "avocado-cli" -version = "0.19.2-dev" +version = "0.20.0" dependencies = [ "anyhow", "base64", diff --git a/Cargo.toml b/Cargo.toml index 837b7d2..467045e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "avocado-cli" -version = "0.19.2-dev" +version = "0.20.0" edition = "2021" description = "Command line interface for Avocado." authors = ["Avocado"]