Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions docs/user/reference/config/components.md
Original file line number Diff line number Diff line change
Expand Up @@ -373,11 +373,13 @@ Use `origin.type = "custom"` when a source archive must be assembled or modified

Custom sources are regenerated on every source preparation rather than restored from lookaside. The generated archive is validated against its configured hash, so changes to the script or its inputs fail with a hash mismatch until the hash is intentionally refreshed.

For an upstream component, each script filename is resolved relative to the TOML file that declares that `source-files` entry. This remains true when the component is assembled from multiple included configuration files. For a local component, the script remains a sidecar beside the component's spec file.

The `script`, `mock-packages`, and `inputs` fields are nested under `[origin]`:

| Field | TOML Key | Type | Required | Description |
|-------|----------|------|----------|-------------|
| Script | `origin.script` | string | **Yes** | Script filename (relative to the component's spec dir) to run in mock. Required for `origin.type = "custom"`. |
| Script | `origin.script` | string | **Yes** | Script filename to run in mock. Relative to the declaring TOML file for upstream components, or the spec directory for local components. Required for `origin.type = "custom"`. |
Comment thread
Tonisal-byte marked this conversation as resolved.
| Mock packages | `origin.mock-packages` | array of string | No | Extra RPM packages to install in the mock chroot before the script runs. |
| Inputs | `origin.inputs` | array of string | No | Unique filenames to make available in the mock chroot before the script runs. Each file must already be present in the fetched source output directory — upstream source tarballs, sidecar files (patches, scripts), and any earlier `source-files` entries are all placed there by the upstream fetch before custom scripts run. |

Expand All @@ -389,7 +391,7 @@ filename = "yara-4.5.4-azl-stripped.tar.gz"
hash-type = "SHA512"
hash = "abc123..." # from: prep-sources --allow-no-hashes
origin.type = "custom"
origin.script = "gen-yara-stripped.sh" # relative to the component's spec directory
origin.script = "gen-yara-stripped.sh" # beside this TOML file for an upstream component
origin.mock-packages = ["cmake"] # omit if not needed
origin.inputs = ["yara-4.5.4.tar.gz"] # available to the script as ./yara-4.5.4.tar.gz
```
Expand Down
4 changes: 2 additions & 2 deletions internal/app/azldev/cmds/component/history_customizations.go
Original file line number Diff line number Diff line change
Expand Up @@ -230,10 +230,10 @@ func appendSourceFileItems(
})
}

if sourceFile.Origin.Script != "" {
if scriptName := sourceFile.Origin.EffectiveScriptName(); scriptName != "" {
items = append(items, CustomizationItem{
Kind: "source-files.script",
Value: sourceFile.Origin.Script,
Value: scriptName,
})
}

Expand Down
20 changes: 20 additions & 0 deletions internal/app/azldev/cmds/component/history_internal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources"
"github.com/microsoft/azure-linux-dev-tools/internal/projectconfig"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

// TestHasExplicitComponentSelection pins the NEW-1 fix: only an exact name or
Expand Down Expand Up @@ -256,6 +257,25 @@ func TestCollectCustomizationsEmitsEveryKind(t *testing.T) {
}
}

func TestCollectCustomizationsUsesEffectiveScriptName(t *testing.T) {
t.Parallel()

config := projectconfig.ComponentConfig{
SourceFiles: []projectconfig.SourceFileReference{{
Filename: "generated.tar.gz",
Origin: projectconfig.Origin{
Type: projectconfig.OriginTypeCustom,
Script: "/project/components/generate.sh",
},
}},
}

items := collectCustomizations("comp", &config)

require.Len(t, items, 2)
assert.Equal(t, CustomizationItem{Kind: "source-files.script", Value: "generate.sh"}, items[1])
}

// TestFingerprintChangeDTOMirrorsSource guards the direction the explicit
// field-by-field copy in [toFingerprintChanges] cannot: a NEW field added to
// [sources.FingerprintChange] / [sources.CommitMetadata] would compile fine
Expand Down
47 changes: 45 additions & 2 deletions internal/app/azldev/cmds/config/dump.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,11 @@ package config
import (
"encoding/json"
"fmt"
"maps"
"slices"

"github.com/microsoft/azure-linux-dev-tools/internal/app/azldev"
"github.com/microsoft/azure-linux-dev-tools/internal/projectconfig"
"github.com/pelletier/go-toml/v2"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
Expand Down Expand Up @@ -97,16 +100,18 @@ issues or inspecting effective values.`,
}

func DumpConfig(env *azldev.Env, format configDumpFormat) (string, error) {
config := portableConfigCopy(env.Config())

switch format {
case ConfigDumpFormatTOML:
tomlBytes, err := toml.Marshal(env.Config())
tomlBytes, err := toml.Marshal(config)
if err != nil {
return "", fmt.Errorf("failed to serialize config to TOML:\n%w", err)
}

return string(tomlBytes), nil
case ConfigDumpFormatJSON:
jsonBytes, err := json.MarshalIndent(env.Config(), "", " ")
jsonBytes, err := json.MarshalIndent(config, "", " ")
if err != nil {
return "", fmt.Errorf("failed to serialize config to JSON:\n%w", err)
}
Expand All @@ -116,3 +121,41 @@ func DumpConfig(env *azldev.Env, format configDumpFormat) (string, error) {
return "", fmt.Errorf("unsupported format: %#q", format)
}
}

func portableConfigCopy(config *projectconfig.ProjectConfig) *projectconfig.ProjectConfig {
result := *config

result.Components = maps.Clone(config.Components)
for name, component := range result.Components {
normalizeCustomScriptNames(&component)
result.Components[name] = component
}

normalizeCustomScriptNames(&result.DefaultComponentConfig)

result.ComponentGroups = maps.Clone(config.ComponentGroups)
for name, group := range result.ComponentGroups {
normalizeCustomScriptNames(&group.DefaultComponentConfig)
result.ComponentGroups[name] = group
}

result.Distros = maps.Clone(config.Distros)
for distroName, distro := range result.Distros {
distro.Versions = maps.Clone(distro.Versions)
for versionName, version := range distro.Versions {
normalizeCustomScriptNames(&version.DefaultComponentConfig)
distro.Versions[versionName] = version
}

result.Distros[distroName] = distro
}

return &result
}

func normalizeCustomScriptNames(component *projectconfig.ComponentConfig) {
component.SourceFiles = slices.Clone(component.SourceFiles)
for i := range component.SourceFiles {
component.SourceFiles[i].Origin.Script = component.SourceFiles[i].Origin.EffectiveScriptName()
}
}
17 changes: 17 additions & 0 deletions internal/app/azldev/cmds/config/dump_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,17 @@ func TestDumpConfig(t *testing.T) {
WorkDir: testWorkDir,
OutputDir: testOutputDir,
},
Components: map[string]projectconfig.ComponentConfig{
"example": {
SourceFiles: []projectconfig.SourceFileReference{{
Filename: "generated.tar.gz",
Origin: projectconfig.Origin{
Type: projectconfig.OriginTypeCustom,
Script: "/project/components/generate.sh",
},
}},
},
},
}

ctx, cancelFunc := context.WithCancel(t.Context())
Expand All @@ -47,8 +58,14 @@ func TestDumpConfig(t *testing.T) {
configText, err := config.DumpConfig(env, config.ConfigDumpFormatTOML)
require.NoError(t, err)
require.NotEmpty(t, configText)
require.Contains(t, configText, "generate.sh")
require.NotContains(t, configText, "/project/components")

configText, err = config.DumpConfig(env, config.ConfigDumpFormatJSON)
require.NoError(t, err)
require.NotEmpty(t, configText)
require.Contains(t, configText, "generate.sh")
require.NotContains(t, configText, "/project/components")
require.Equal(t, "/project/components/generate.sh",
cfg.Components["example"].SourceFiles[0].Origin.Script)
}
8 changes: 8 additions & 0 deletions internal/fingerprint/fingerprint.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"encoding/hex"
"fmt"
"io"
"slices"
"sort"
"strconv"

Expand Down Expand Up @@ -107,6 +108,13 @@ func ComputeIdentity(
}

// 3. Hash the resolved config struct (excluding fingerprint:"-" fields).
// Script paths are absolute internally so merged definitions retain their
// declaration context; hash only their checkout-independent filenames.
component.SourceFiles = slices.Clone(component.SourceFiles)
for i := range component.SourceFiles {
component.SourceFiles[i].Origin.Script = component.SourceFiles[i].Origin.EffectiveScriptName()
}

configHash, err := hashstructure.Hash(component, hashstructure.FormatV2, &hashstructure.HashOptions{
TagName: hashstructureTagName,
})
Expand Down
38 changes: 38 additions & 0 deletions internal/fingerprint/fingerprint_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -382,6 +382,44 @@ func TestComputeIdentity_SourceFilesChange(t *testing.T) {
assert.NotEqual(t, fp1, fp2, "different source file hash must produce different fingerprints")
}

func TestComputeIdentity_CustomScriptPathIsCheckoutIndependent(t *testing.T) {
ctx := newTestFS(t, map[string]string{
"/specs/test.spec": "Name: testpkg\nVersion: 1.0",
})

comp1 := baseComponent()
comp1.SourceFiles = []projectconfig.SourceFileReference{{
Filename: "source.tar.gz",
Hash: "aaa111",
HashType: fileutils.HashTypeSHA256,
Origin: projectconfig.Origin{
Type: projectconfig.OriginTypeCustom,
Script: "/home/user1/repo/generate.sh",
},
}}

comp2 := comp1
comp2.SourceFiles = []projectconfig.SourceFileReference{{
Filename: "source.tar.gz",
Hash: "aaa111",
HashType: fileutils.HashTypeSHA256,
Origin: projectconfig.Origin{
Type: projectconfig.OriginTypeCustom,
Script: "/home/user2/repo/generate.sh",
},
}}

fp1 := computeFingerprint(t, ctx, comp1, testReleaseVer, 0)
fp2 := computeFingerprint(t, ctx, comp2, testReleaseVer, 0)

assert.Equal(t, fp1, fp2)
assert.Equal(t, "/home/user1/repo/generate.sh", comp1.SourceFiles[0].Origin.Script)

comp2.SourceFiles[0].Origin.Script = "/home/user2/repo/different.sh"
fp2 = computeFingerprint(t, ctx, comp2, testReleaseVer, 0)
assert.NotEqual(t, fp1, fp2)
}

func TestComputeIdentity_SourceFileOriginExcluded(t *testing.T) {
ctx := newTestFS(t, map[string]string{
"/specs/test.spec": "Name: testpkg\nVersion: 1.0",
Expand Down
45 changes: 42 additions & 3 deletions internal/projectconfig/component.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package projectconfig
import (
"errors"
"fmt"
"path/filepath"
"slices"
"sort"
"strings"
Expand Down Expand Up @@ -71,10 +72,11 @@ type Origin struct {
// Uri to download the source file from if origin type is 'download'. Ignored for other origin types.
Uri string `toml:"uri,omitempty" json:"uri,omitempty" jsonschema:"title=URI,description=URI to download the source file from if origin type is 'download',example=https://example.com/source.tar.gz" fingerprint:"-"`

// Script is the filename of a shell script, relative to the component's spec directory,
// that is run inside a mock chroot to generate this source file.
// Script is the filename of a shell script run inside a mock chroot to generate this source file.
// For upstream components it is relative to the declaring config file; for local components
// it is relative to the component's spec directory.
// Required when [Origin.Type] is 'custom'; must be empty otherwise.
Script string `toml:"script,omitempty" json:"script,omitempty" jsonschema:"title=Script,description=Shell script filename (relative to the component spec directory) to run in mock to generate this source file. Required when origin type is 'custom'."`
Script string `toml:"script,omitempty" json:"script,omitempty" jsonschema:"title=Script,description=Shell script filename to run in mock to generate this source file. Relative to the declaring config file for upstream components or the component spec directory for local components. Required when origin type is 'custom'."`

// MockPackages is a list of RPM package names to install in the mock chroot before
// running [Origin.Script]. Only valid when [Origin.Type] is 'custom'.
Expand All @@ -86,6 +88,15 @@ type Origin struct {
Inputs []string `toml:"inputs,omitempty" json:"inputs,omitempty" jsonschema:"title=Inputs,description=Source-output filenames to make available next to the generation script before it runs. Only valid when origin type is 'custom'."`
}

// EffectiveScriptName returns the checkout-independent filename of [Origin.Script].
Comment thread
Tonisal-byte marked this conversation as resolved.
func (o Origin) EffectiveScriptName() string {
if o.Script == "" {
return ""
}

return filepath.Base(o.Script)
}

// HashInclude implements the hashstructure [Includable] interface so that
// [Origin.Script], [Origin.MockPackages], and [Origin.Inputs] are omitted from
// the component fingerprint when they hold their zero values.
Expand Down Expand Up @@ -447,9 +458,25 @@ func (c *ComponentConfig) MergeUpdatesFrom(other *ComponentConfig) error {
c.OverlayFiles = otherOverlayFiles
}

c.resolveLocalCustomScriptPaths()

return nil
}

func (c *ComponentConfig) resolveLocalCustomScriptPaths() {
if c.Spec.SourceType != SpecSourceTypeLocal || c.Spec.Path == "" {
return
}

scriptDir := filepath.Dir(c.Spec.Path)
for i := range c.SourceFiles {
origin := &c.SourceFiles[i].Origin
if origin.Type == OriginTypeCustom && origin.Script != "" {
origin.Script = filepath.Join(scriptDir, origin.EffectiveScriptName())
}
}
}

// EffectiveUpstreamCommit returns the commit to use for upstream operations.
// Prefers the locked commit (resolved reality) over the config pin (user intent).
// Falls back to Spec.UpstreamCommit for SkipLockValidation paths (update, list,
Expand Down Expand Up @@ -530,6 +557,18 @@ func (c *ComponentConfig) WithAbsolutePaths(referenceDir string) *ComponentConfi
// Fix up paths.
result.Spec.Path = makeAbsolute(referenceDir, result.Spec.Path)

scriptDir := referenceDir
if result.Spec.SourceType == SpecSourceTypeLocal && result.Spec.Path != "" {
scriptDir = filepath.Dir(result.Spec.Path)
}
Comment thread
Tonisal-byte marked this conversation as resolved.

for i := range result.SourceFiles {
origin := &result.SourceFiles[i].Origin
if origin.Type == OriginTypeCustom {
origin.Script = makeAbsolute(scriptDir, origin.Script)
}
}

// Copy and fix up overlays.
if c.Overlays != nil {
result.Overlays = make([]ComponentOverlay, len(c.Overlays))
Expand Down
Loading
Loading