tests: Add cross-consumer staging scenario to loader-entries-source

crates/lib/src/bootc_kargs.rs

@@ -194,9 +194,10 @@ fn get_kargs_from_ostree(
     Ok(ret)
 }
 
-/// Compute the kernel arguments for the new deployment. This starts from the booted
-/// karg, but applies the diff between the bootc karg files in /usr/lib/bootc/kargs.d
-/// between the booted deployment and the new one.
+/// Compute the kernel arguments for the new deployment. This starts from the kargs
+/// of the deployment being replaced -- the staged one if there is one, otherwise the
+/// merge (booted) deployment -- and applies the diff between the bootc karg files in
+/// /usr/lib/bootc/kargs.d of that deployment and the new one.
 pub(crate) fn get_kargs(
     sysroot: &Storage,
     merge_deployment: &Deployment,
@@ -207,16 +208,25 @@ pub(crate) fn get_kargs(
     let repo = &ostree.repo();
     let sys_arch = std::env::consts::ARCH;
 
-    // Get the kargs used for the merge in the bootloader config
-    let mut kargs = ostree::Deployment::bootconfig(merge_deployment)
+    // A staged deployment carries kargs changes that are not in the booted
+    // entry yet (e.g. from `bootc loader-entries set-options-for-source` or
+    // `rpm-ostree kargs`).  Staging replaces it, so build on it rather than
+    // silently discarding those changes.
+    let staged = ostree
+        .staged_deployment()
+        .filter(|s| s.stateroot() == merge_deployment.stateroot());
+    let base_deployment = staged.as_ref().unwrap_or(merge_deployment);
+
+    // Get the kargs used for the base deployment in the bootloader config
+    let mut kargs = ostree::Deployment::bootconfig(base_deployment)
         .and_then(|bootconfig| {
             ostree::BootconfigParser::get(&bootconfig, "options")
                 .map(|options| Cmdline::from(options.to_string()))
         })
         .unwrap_or_default();
 
-    // Get the kargs in kargs.d of the merge
-    let merge_root = &crate::utils::deployment_fd(ostree, merge_deployment)?;
+    // Get the kargs in kargs.d of the base deployment
+    let merge_root = &crate::utils::deployment_fd(ostree, base_deployment)?;
     let existing_kargs = get_kargs_in_root(merge_root, sys_arch)?;
 
     // Get the kargs in kargs.d of the pending image

crates/lib/src/deploy.rs

@@ -904,9 +904,8 @@ async fn deploy(
     lock_finalization: bool,
 ) -> Result<Deployment> {
     // Compute the kernel argument overrides. In practice today this API is always expecting
-    // a merge deployment. The kargs code also always looks at the booted root (which
-    // is a distinct minor issue, but not super important as right now the install path
-    // doesn't use this API).
+    // a merge deployment; the kargs code builds on the staged deployment instead when
+    // there is one, so that a pending kargs change is not lost.
     let (stateroot, override_kargs) = match &from {
         MergeState::MergeDeployment(deployment) => {
             let kargs = crate::bootc_kargs::get_kargs(sysroot, &deployment, image)?;

crates/lib/src/loader_entries.rs

@@ -11,7 +11,7 @@
 
 use anyhow::{Context, Result, ensure};
 use fn_error_context::context;
-use linux_kernel_cmdline::utf8::{Cmdline, CmdlineOwned};
+use linux_kernel_cmdline::utf8::{Cmdline, CmdlineOwned, Parameter};
 use ostree::{gio, glib};
 use ostree_ext::ostree;
 use std::collections::BTreeMap;
@@ -85,34 +85,50 @@ fn extract_source_options_from_bls(content: &str) -> BTreeMap<String, CmdlineOwn
 ///
 /// The algorithm:
 /// 1. Start with the current options line
-/// 2. Remove all options that belong to the old value of the specified source
-/// 3. Add the new options for the specified source
+/// 2. Replace the options that belong to the old value of the specified source,
+///    in place, with the new options for that source
 ///
-/// Options not tracked by any source are preserved as-is.
+/// Options not tracked by any source are preserved as-is, in their original
+/// position.  Keeping positions stable matters: for parameters where the last
+/// occurrence wins, moving a source's options to the end would change their
+/// meaning, and re-applying an unchanged source would otherwise produce a
+/// different string and look like a change.
 fn compute_merged_options(
     current_options: &str,
     source_options: &BTreeMap<String, CmdlineOwned>,
     target_source: &SourceName,
     new_options: Option<&str>,
 ) -> CmdlineOwned {
-    let mut merged = CmdlineOwned::from(current_options.to_owned());
-
-    // Remove old options from the target source (if it was previously tracked)
-    if let Some(old_source_opts) = source_options.get(&**target_source) {
-        for param in old_source_opts.iter() {
-            merged.remove_exact(&param);
+    let current = Cmdline::from(current_options);
+    let old_params: Vec<Parameter> = source_options
+        .get(&**target_source)
+        .map(|old| old.iter().collect())
+        .unwrap_or_default();
+    let new_cmdline = new_options.filter(|v| !v.is_empty()).map(Cmdline::from);
+    let new_params: Vec<String> = new_cmdline
+        .iter()
+        .flat_map(|c| c.iter())
+        .map(|p| p.to_string())
+        .collect();
+
+    let mut merged: Vec<String> = Vec::new();
+    let mut inserted = false;
+    for param in current.iter() {
+        if old_params.contains(&param) {
+            // First old parameter: emit the new ones here; drop the rest.
+            if !inserted {
+                merged.extend(new_params.iter().cloned());
+                inserted = true;
+            }
+            continue;
         }
+        merged.push(param.to_string());
     }
-
-    // Add new options for the target source
-    if let Some(new_opts) = new_options.filter(|v| !v.is_empty()) {
-        let new_cmdline = Cmdline::from(new_opts);
-        for param in new_cmdline.iter() {
-            merged.add(&param);
-        }
+    if !inserted {
+        merged.extend(new_params);
     }
 
-    merged
+    CmdlineOwned::from(merged.join(" "))
 }
 
 /// Read x-options-source-* keys from the staged deployment data file.
@@ -556,7 +572,40 @@ x-options-source-admin nohz=full
                 ],
                 "tuned",
                 Some("nohz=full"),
-                "root=UUID=abc rw rd.driver.pre=vfio-pci nohz=full",
+                "root=UUID=abc rw nohz=full rd.driver.pre=vfio-pci",
+            ),
+            (
+                "re-applying an unchanged source is a no-op even when followed by other options",
+                "root=UUID=abc rw nohz=on rcu_nocbs=2-7 cross1=a rpmarg=yes",
+                &[
+                    ("tuned", "nohz=on rcu_nocbs=2-7"),
+                    ("crosstest", "cross1=a"),
+                ],
+                "tuned",
+                Some("nohz=on rcu_nocbs=2-7"),
+                "root=UUID=abc rw nohz=on rcu_nocbs=2-7 cross1=a rpmarg=yes",
+            ),
+            (
+                "updating a source keeps its position",
+                "root=UUID=abc rw nohz=on rcu_nocbs=2-7 cross1=a rpmarg=yes",
+                &[
+                    ("tuned", "nohz=on rcu_nocbs=2-7"),
+                    ("crosstest", "cross1=a"),
+                ],
+                "tuned",
+                Some("nohz=on skew_tick=1"),
+                "root=UUID=abc rw nohz=on skew_tick=1 cross1=a rpmarg=yes",
+            ),
+            (
+                "removing a source keeps the order of the rest",
+                "root=UUID=abc rw nohz=on cross1=a rcu_nocbs=2-7 rpmarg=yes",
+                &[
+                    ("tuned", "nohz=on rcu_nocbs=2-7"),
+                    ("crosstest", "cross1=a"),
+                ],
+                "tuned",
+                None,
+                "root=UUID=abc rw cross1=a rpmarg=yes",
             ),
         ];
 

docs/src/building/kernel-arguments.md

@@ -78,22 +78,137 @@ mount -o remount,rw /boot
 # tool to edit /boot/loader/entries
 ```
 
-At the current time, `bootc` does not itself offer
-an API to manipulate kernel arguments maintained per-machine.
+`bootc loader-entries set-options-for-source` manages per-machine
+kernel arguments on behalf of a named *source*; see
+[Source-tracked kernel arguments](#source-tracked-kernel-arguments) below.
 
-Other projects such as `rpm-ostree` do, via e.g. `rpm-ostree kargs`,
+Other projects such as `rpm-ostree` do too, via e.g. `rpm-ostree kargs`,
 which is just a frontend for editing the bootloader configuration
 files. Note an important detail is that `rpm-ostree kargs` always
 creates a new deployment.
 
-`rpm-ostree kargs` and bootc will interoperate as they both
-use the ostree backend today, and any kernel arguments changed
-via that mechanism will persist across upgrades.
+`rpm-ostree kargs` and bootc interoperate as they both use the ostree
+backend today: any kernel arguments changed via either mechanism persist
+across upgrades, and each builds on the deployment it replaces, so a
+change staged by one is not lost when the other stages next.
 
 It is currently undefined behavior to remove kernel arguments
 locally that are included in the base image via
 `/usr/lib/bootc/kargs.d`.
 
+## Source-tracked kernel arguments
+
+A tool that adds kernel arguments — TuneD applying a profile is the
+motivating case — needs to later replace or remove exactly the arguments
+it added, without keeping state in `/etc` (which may be transient).
+`bootc loader-entries set-options-for-source` records ownership next to
+the arguments themselves, in the BLS entry:
+
+```
+options root=UUID=... rw ostree=/ostree/boot.0/... console=ttyS0 nohz=full isolcpus=1-3 rd.driver.pre=vfio-pci
+x-options-source-tuned nohz=full isolcpus=1-3
+x-options-source-dracut rd.driver.pre=vfio-pci
+```
+
+* `options` is the kernel command line and the ground truth.
+* `x-options-source-NAME` is bookkeeping: the arguments source `NAME`
+  currently owns. Bootloaders ignore unknown keys. Names are limited to
+  `[A-Za-z0-9_-]+`.
+* A source key with an **empty value is a tombstone** ("this source owns
+  nothing"). bootc never deletes a key, it clears it.
+
+```bash
+# set or replace TuneD's arguments
+bootc loader-entries set-options-for-source --source tuned --options "nohz=full isolcpus=1-3"
+# remove them
+bootc loader-entries set-options-for-source --source tuned
+```
+
+Both stage a new deployment (unless nothing would change); the arguments
+take effect on the next boot. See
+`man bootc-loader-entries-set-options-for-source`.
+
+### How a call is processed
+
+```mermaid
+flowchart TD
+    s([set-options-for-source --source S --options O]) --> v[validate S]
+    v --> base{Staged deployment exists?}
+    base -- yes --> b1[base = staged deployment<br/>commit, origin, options from it]
+    base -- no --> b2[base = booted deployment]
+    b1 --> disc1[sources = names from the booted entry,<br/>values from the staged bootconfig,<br/>plus names only in the staged data]
+    b2 --> disc2[sources = parse the booted entry]
+    disc1 --> merge
+    disc2 --> merge[merged = options with S's old<br/>arguments replaced in place by O]
+    merge --> idem{merged == options<br/>and old S == O?}
+    idem -- yes --> noop([No changes needed])
+    idem -- no --> set[On the booted deployment's bootconfig:<br/>clear every known source key,<br/>re-set all sources except S,<br/>set S = O if given]
+    set --> stage[stage_tree_with_options<br/>merge = booted, commit/origin = base,<br/>override_kernel_argv = merged]
+```
+
+* **Base vs. merge deployment.** The commit, origin and current
+  `options` come from the *staged* deployment when one exists, so a
+  pending `bootc upgrade` is kept. The merge deployment (for the `/etc`
+  merge, and where the source keys are written) is always the booted one.
+* **In-place replacement.** The source's arguments are replaced where
+  they were, so re-applying an unchanged source is byte-identical and
+  a no-op even when other arguments follow, and the relative order of
+  arguments is preserved (it matters where the last occurrence wins).
+* **The full set is written on every call**, tombstones included. This
+  is what lets ostree carry the keys correctly through staging.
+
+### How the keys survive staging
+
+A staged deployment has no BLS entry until finalization at shutdown, and
+any later staging in the same boot — `bootc upgrade`, `rpm-ostree kargs`,
+another `set-options-for-source` — replaces it. ostree carries the
+`x-options-source-*` keys across that gap in the staged deployment data
+(`bootconfig-extra`) and decides which set to carry when a staging is
+replaced; that mechanism, and the contract bootc follows as an "aware"
+caller, are documented in
+[Extension BLS keys and staged deployments](https://ostreedev.github.io/ostree/bootconfig-extra/)
+on the ostree side. It requires ostree 2026.1 (2026.5 for the case where
+another tool re-stages in the same boot); bootc checks the version at
+runtime.
+
+### Interaction with `bootc upgrade` / `switch`
+
+```mermaid
+flowchart TD
+    s([upgrade / switch]) --> base{Staged deployment in the<br/>same stateroot exists?}
+    base -- yes --> b1[base = staged deployment]
+    base -- no --> b2[base = booted deployment]
+    b1 --> k[kargs = base's options line]
+    b2 --> k
+    k --> old[old = kargs.d of base's tree]
+    old --> new{new image has<br/>/usr/lib/bootc/kargs.d?}
+    new -- no --> add[kargs += old] --> stage
+    new -- yes --> diff[remove old - new from kargs,<br/>add new - old to kargs] --> stage[stage_tree_with_options<br/>override_kernel_argv = kargs]
+```
+
+Upgrading builds on the staged deployment's arguments when there is one,
+so a source change (or an `rpm-ostree kargs` change) staged earlier in
+the same boot survives the upgrade; only the `kargs.d` *diff* between the
+two images is applied on top. The upgrade path never touches source keys;
+ostree carries them forward.
+
+### Interaction with `rpm-ostree` and direct edits
+
+`rpm-ostree kargs` likewise builds on the pending deployment, so the two
+can be interleaved in either order. It does not know about source keys;
+ostree carries them. If something edits `options` directly (for example
+`rpm-ostree kargs --delete` of an argument a source owns), the source's
+record is stale until that source is next written — a subsequent
+`set-options-for-source` for it simply finds nothing to remove.
+
+### Consumers
+
+TuneD (2.27+ with the bootc bootloader support) uses `--source tuned`,
+declaring its profile's full argument set on every apply and clearing it
+on unapply. On images with a transient `/etc`, set TuneD's
+`profile_mode` to `manual` in the image so it does not auto-select a
+different profile on each boot and clear the administrator's arguments.
+
 ## Injecting default arguments into custom kernels
 
 The Linux kernel supports building in arguments into the kernel

docs/src/man/bootc-loader-entries-set-options-for-source.8.md

@@ -39,6 +39,8 @@ preserving the pending upgrade while layering the kargs change on top.
 This command requires ostree >= 2026.1 with `bootconfig-extra` support
 for preserving extension BLS keys through staged deployment roundtrips.
 On older ostree versions, the command will exit with an error.
+ostree >= 2026.5 is needed for the source keys to survive when another
+tool (e.g. `rpm-ostree kargs`) re-stages before the reboot.
 
 # EXAMPLES