diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0a1fa57c04..a5a1f7d26e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -87,6 +87,9 @@ scripts/exclude-from-green-ci.sh @DataDog/libdatadog-core scripts/major-bumps-level.sh @DataDog/libdatadog-core scripts/publication-order.sh @DataDog/libdatadog-core scripts/reformat_copyright.sh @DataDog/libdatadog-core +scripts/release-generate-changelogs.sh @DataDog/libdatadog-core +scripts/release-version-bumps.sh @DataDog/libdatadog-core +scripts/release-version-major-bumps.sh @DataDog/libdatadog-core scripts/semver-level.sh @DataDog/libdatadog-core scripts/update_license_3rdparty.sh @DataDog/libdatadog-core scripts/Dockerfile.license @DataDog/libdatadog-core diff --git a/.github/workflows/release-proposal-dispatch.yml b/.github/workflows/release-proposal-dispatch.yml index bcbb740b91..19a89e1e36 100644 --- a/.github/workflows/release-proposal-dispatch.yml +++ b/.github/workflows/release-proposal-dispatch.yml @@ -433,9 +433,11 @@ jobs: run: | # Get commits since release for each crate and save to file "${WORKFLOW_SCRIPTS_ROOT}/commits-since-release.sh" "$(cat /tmp/crates.json)" > /tmp/commits-by-crate.json - - # Capture ephemeral release branch tip now. Use this in Release version bumps - # so tag/merge-base resolution uses the same ref the script used. + + # Capture the ephemeral release branch tip now: later steps run from the proposal + # branch, whose HEAD has moved on. commits-since-release.sh already ends every + # crate's `range` at this same commit; this is for the steps that need the tip on + # its own (the commit list handed to the verified push). git rev-parse HEAD > /tmp/release_head_sha echo "Release branch HEAD (saved for later): $(cat /tmp/release_head_sha)" echo "release_head_sha=$(cat /tmp/release_head_sha)" >> "$GITHUB_OUTPUT" @@ -467,8 +469,13 @@ jobs: - name: Release version bumps id: release-version-bumps + env: + PROPOSAL_BRANCH: ${{ steps.proposal-branch.outputs.branch_name }} + EPHEMERAL_BRANCH: ${{ steps.ephemeral-branch.outputs.ephemeral_branch }} + IS_HOTFIX: ${{ steps.ephemeral-branch.outputs.is_hotfix }} + BYPASS_STANDARD_CHECKS: ${{ inputs.bypass_standard_checks }} run: | - echo "Release version bumps..." + set -euo pipefail # TODO: check if this is really necessary, we should have the full history from # previous steps. @@ -477,156 +484,19 @@ jobs: git fetch --unshallow fi - # Initialize results array - echo "[]" > /tmp/api-changes.json - - # Crates with no commits of their own are not released here, but recorded as candidates: - echo "[]" > /tmp/pending-major-only.json - - # Use release branch tip from when we ran commits-since-release (same ref the script used). - # Avoids tag/merge-base resolution failures after switching to the new proposal branch. - ORIGINAL_HEAD=$(cat /tmp/release_head_sha) - echo "ORIGINAL_HEAD: $ORIGINAL_HEAD" - - BRANCH_NAME="${{ steps.proposal-branch.outputs.branch_name }}" - - # iterate over the commits and execute cargo release for each crate - jq -c '.[]' /tmp/commits-by-crate.json | while read -r crate; do - NAME=$(echo "$crate" | jq -r '.name') - TAG=$(echo "$crate" | jq -r '.tag') - TAG_PREFIX="$NAME-v" - CRATE_PATH=$(echo "$crate" | jq -r '.path') - TAG_EXISTS=$(echo "$crate" | jq -r '.tag_exists') - COMMITS=$(echo "$crate" | jq -r '.commits') - INITIAL_RELEASE=false - - # if there are no commits and there is an existing tag, do not release the crate here. - # but record it as a pending candidate - if [ "$COMMITS" = "[]" ] && [ "$TAG_EXISTS" = "true" ]; then - VERSION=$(echo "$crate" | jq -r '.version') - echo "No commits since last release for $NAME; deferring to the libdd-* major-bump check" - jq --arg name "$NAME" \ - --arg tag "$TAG" \ - --arg version "$VERSION" \ - --arg path "$CRATE_PATH" \ - '. += [{"name": $name, "level": "none", "tag": $tag, "prev_tag": $tag, "version": $version, "range": "", "commits": [], "path": $path, "initial_release": "false", "pending_release": "true"}]' \ - /tmp/pending-major-only.json > /tmp/pending-major-only.tmp && mv /tmp/pending-major-only.tmp /tmp/pending-major-only.json - continue - fi - - if [ "$TAG_EXISTS" = "true" ]; then - # Explicitly dereference annotated tags to their underlying commit. - # Several git commands do not consistently dereference annotated tag objects - # across all git versions. - TAG_COMMIT=$(git rev-parse "${TAG}^{}" 2>/dev/null || echo "") - if [ -z "$TAG_COMMIT" ]; then - echo "ERROR: Could not dereference tag $TAG to a commit" >&2 - exit 1 - fi - - RANGE="$TAG_COMMIT..$ORIGINAL_HEAD" - echo "Using $RANGE as range (tag: $TAG)" - - if git merge-base --is-ancestor "$TAG_COMMIT" "$ORIGINAL_HEAD" 2>/dev/null; then - echo " Tag $TAG is ancestor of HEAD" - else - MERGE_BASE=$(git merge-base "$TAG_COMMIT" "$ORIGINAL_HEAD" 2>/dev/null || echo "") - if [ -n "$MERGE_BASE" ]; then - RANGE="$MERGE_BASE..$ORIGINAL_HEAD" - echo " Tag $TAG is NOT ancestor of HEAD, using merge-base: $RANGE" - else - # No common ancestor, tag is on unrelated history. Derive the range start from - # the parent of the oldest commit found by commits-since-release.sh. That way - # git can compute TREESAME correctly and the path filter won't include unrelated commits. - OLDEST_COMMIT=$(echo "$COMMITS" | jq -r '.[-1].hash // empty') - OLDEST_PARENT=$(git rev-parse "${OLDEST_COMMIT}^" 2>/dev/null || echo "") - if [ -n "$OLDEST_PARENT" ]; then - RANGE="$OLDEST_PARENT..$ORIGINAL_HEAD" - echo " No common ancestor with tag $TAG: using parent of oldest commit as range start: $RANGE" - else - echo " WARNING: Could not find merge-base for tag $TAG, using $RANGE" - fi - fi - fi - - BRANCHES=$(git branch --contains "$TAG_COMMIT" 2>/dev/null || echo "") - if [ -n "$BRANCHES" ]; then - echo "Tag $TAG is in branches: $BRANCHES" - else - echo "Tag $TAG (commit $TAG_COMMIT) is not in any local branch (normal for squash-merged releases)" - fi - - # if there is a tag more recent than $TAG, continue the loop - LATEST_TAG=$(git tag -l "$TAG_PREFIX*" --sort=-v:refname | head -1) - if [ "$LATEST_TAG" != "$TAG" ]; then - echo "Tag $TAG is not the latest. Latest is: $LATEST_TAG. main branch has the latest release for $NAME" - - # do not skip the release for hotfix branches - if [ "${{ steps.ephemeral-branch.outputs.is_hotfix }}" = "true" ]; then - echo "Continuing with the release for $NAME because it is a hotfix" - else - if [ "${{ inputs.bypass_standard_checks }}" = "false" ]; then - echo "Skipping release for $NAME" - continue - else - echo "Continuing with the release for $NAME because bypass_standard_checks is true" - fi - fi - fi - - echo "Executing semver-level.sh for $NAME since $RANGE (tag: $TAG)..." - SEMVER_LEVEL=$("${WORKFLOW_SCRIPTS_ROOT}/semver-level.sh" "$NAME" "refs/tags/$TAG" 2>&1) - echo "Semver level: $SEMVER_LEVEL" - - LEVEL=$(echo "$SEMVER_LEVEL" | jq -r '.level') - - echo "Executing cargo release for $NAME since $TAG with level $LEVEL..." - cargo release version -p "$NAME" --prev-tag-name "$TAG" --allow-branch "$BRANCH_NAME" -x $LEVEL --no-confirm - - else - echo "No previous release tag for $NAME, preparing initial release..." - - # Use the version from the crate metadata - VERSION=$(echo "$crate" | jq -r '.version') - LEVEL="major" - TAG="" - RANGE="" - - # fail when the version is not an initial release - if [ "$VERSION" != "0.1.0" ]; then - echo "Error: $NAME is not a 0.1.0 release" >&2 - exit 1 - fi + ARGS=() + if [ "$IS_HOTFIX" = "true" ]; then ARGS+=(--hotfix); fi + if [ "$BYPASS_STANDARD_CHECKS" = "true" ]; then ARGS+=(--bypass-standard-checks); fi - INITIAL_RELEASE=true - - echo "Executing cargo release for $NAME with level $LEVEL..." - cargo release version -p "$NAME" --allow-branch "$BRANCH_NAME" -x $LEVEL --no-confirm - fi - - # Commit the changes - cargo release commit --no-confirm -x - - NEXT_VERSION=$(cargo metadata --format-version=1 --no-deps | jq -r --arg name "$NAME" '.packages[] | select(.name == $name) | .version') - NEXT_TAG="$TAG_PREFIX$NEXT_VERSION" - - # Add to results array - jq --arg name "$NAME" \ - --arg level "$LEVEL" \ - --arg tag "$NEXT_TAG" \ - --arg prev_tag "$TAG" \ - --arg version "$NEXT_VERSION" \ - --arg range "$RANGE" \ - --argjson commits "$COMMITS" \ - --arg path "$CRATE_PATH" \ - --arg initial_release "$INITIAL_RELEASE" \ - '. += [{"name": $name, "level": $level, "tag": $tag, "prev_tag": $prev_tag, "version": $version, "range": $range, "commits": $commits, "path": $path, "initial_release": $initial_release}]' \ - /tmp/api-changes.json > /tmp/api-changes.tmp && mv /tmp/api-changes.tmp /tmp/api-changes.json - done + "${WORKFLOW_SCRIPTS_ROOT}/release-version-bumps.sh" \ + --commits-by-crate /tmp/commits-by-crate.json \ + --out /tmp/api-changes.json \ + --branch "$PROPOSAL_BRANCH" \ + "${ARGS[@]}" # Check if there are commits to push or pending - if git diff --quiet "${{ steps.ephemeral-branch.outputs.ephemeral_branch }}"; then - PENDING_COUNT=$(jq 'length' /tmp/pending-major-only.json) + if git diff --quiet "$EPHEMERAL_BRANCH"; then + PENDING_COUNT=$(jq '[.[] | select(.pending_release == "true")] | length' /tmp/api-changes.json) if [ "$PENDING_COUNT" -gt 0 ]; then echo "No direct version bumps yet, but $PENDING_COUNT crate(s) are pending libdd-* major-bump evaluation; continuing." else @@ -635,222 +505,30 @@ jobs: fi fi - # Output the results - echo "API changes summary:" - jq . /tmp/api-changes.json - - name: Update version for crates with libdd-* direct dependency major bumps since last release + env: + PROPOSAL_BRANCH: ${{ steps.proposal-branch.outputs.branch_name }} run: | set -euo pipefail - BRANCH_NAME="${{ steps.proposal-branch.outputs.branch_name }}" - - # Audit input: crates released in the previous step (api-changes.json) plus the pending - # no-commit candidates. The pending rows carry "pending_release": "true" so we can tell - # them apart below; every row is checked the same way for direct libdd-* major bumps. - jq -s '.[0] + .[1]' /tmp/api-changes.json /tmp/pending-major-only.json > /tmp/major-bumps-input.json - echo "Major-bump audit input:" - jq . /tmp/major-bumps-input.json - - # Run the audit in a throwaway worktree so extra worktrees / cargo metadata do not touch - # the job checkout. Check it out at the proposal branch tip (HEAD) — the released ref plus - # this run's version bumps from the previous step. This is deliberate on both ends: - # - It includes the dependency-requirement rewrites cargo-release made in the previous - # step, so a dependency bumped to a new major IN THIS proposal propagates a major bump - # to its dependents (e.g. protobuf 3->4 forces its dependents major). - # - It is built from the released ref, NOT github.sha, so changes present only on current - # main (and absent from a hotfix/older-ref release) never trigger a spurious bump. - MAJOR_BUMPS_WT=$(mktemp -d "${RUNNER_TEMP:-/tmp}/major-bumps-wt.XXXXXX") - PROPOSAL_SHA=$(git rev-parse HEAD) - - git worktree add --detach "$MAJOR_BUMPS_WT" "$PROPOSAL_SHA" - set +e - ( cd "$MAJOR_BUMPS_WT" && "${WORKFLOW_SCRIPTS_ROOT}/major-bumps-level.sh" /tmp/major-bumps-input.json ) \ - > /tmp/api-changes-with-major-bumps-pre-commit.json - MB_RC=$? - git worktree remove --force "$MAJOR_BUMPS_WT" || true - set -e - if [[ "$MB_RC" -ne 0 ]]; then - echo "Major bumps level script failed with code $MB_RC" - echo "Major bumps level script output:" - cat /tmp/api-changes-with-major-bumps-pre-commit.json - exit "$MB_RC" - fi - # Seed the result with every already-released crate. Pending crates are appended below - # only if they earn a major bump; those that do not stay out of the release entirely. - jq '[.[] | select(.pending_release != "true") | del(.pending_release)]' \ - /tmp/api-changes-with-major-bumps-pre-commit.json > /tmp/api-changes-with-major-bumps.json - - # iterate over the crates and, where a direct libdd-* dependency had a major bump, update the version - jq -c '.[]' /tmp/api-changes-with-major-bumps-pre-commit.json | while read -r bump; do - NAME=$(echo "$bump" | jq -r '.name') - LEVEL=$(echo "$bump" | jq -r '.level') - PREV_TAG=$(echo "$bump" | jq -r '.prev_tag') - TAG=$(echo "$bump" | jq -r '.tag') - VERSION=$(echo "$bump" | jq -r '.version') - PENDING=$(echo "$bump" | jq -r '.pending_release // "false"') - MAJOR_BUMPS=$(echo "$bump" | jq -c '.major_bumps') - - if [ "$MAJOR_BUMPS" = "[]" ]; then - if [ "$PENDING" = "true" ]; then - echo "No commits and no direct dependency major bumps for $NAME, keeping it out of the release" - fi - continue - fi - - # A crate already bumped to major in the previous step needs nothing more. Pending - # crates always have level "none" here, so this only short-circuits released crates. - if [ "$LEVEL" = "major" ]; then - echo "Skipping $NAME: already bumped at major level in the previous step (major_bumps: $MAJOR_BUMPS)" - continue - fi + "${WORKFLOW_SCRIPTS_ROOT}/release-version-major-bumps.sh" \ + --api-changes /tmp/api-changes.json \ + --out /tmp/api-changes-with-major-bumps.json \ + --branch "$PROPOSAL_BRANCH" - # Bump to major: either a pending (no-commit) crate whose direct dependency went major, - # or a released crate bumped below major in the previous step. Both are handled the same. - echo "Bumping $NAME to major due to direct dependency major bumps: $MAJOR_BUMPS" - cargo release version -p "$NAME" --prev-tag-name "$PREV_TAG" --allow-branch "$BRANCH_NAME" -x major --no-confirm - - git commit -am "chore(release): update version for $NAME with major bumps" - - NEXT_VERSION=$(cargo metadata --format-version=1 --no-deps | jq -r --arg name "$NAME" '.packages[] | select(.name == $name) | .version') - NEXT_TAG="$NAME-v$NEXT_VERSION" - - echo "Updating tag $TAG to $NEXT_TAG and version $VERSION to $NEXT_VERSION for $NAME" - - # Released crates are already in the result (seeded above): update them in place. Pending - # crates are not: append them. The row is derived from the audit entry either way. - ROW=$(echo "$bump" | jq --arg version "$NEXT_VERSION" --arg tag "$NEXT_TAG" \ - 'del(.pending_release) | . + {level: "major", version: $version, tag: $tag}') - jq --argjson row "$ROW" \ - 'if any(.[]; .name == $row.name) - then map(if .name == $row.name then $row else . end) - else . + [$row] end' \ - /tmp/api-changes-with-major-bumps.json > /tmp/api-changes-with-major-bumps.tmp \ - && mv /tmp/api-changes-with-major-bumps.tmp /tmp/api-changes-with-major-bumps.json - done - - # Output the results - echo "API changes with major bumps summary:" - jq . /tmp/api-changes-with-major-bumps.json - - name: Generate CHANGELOGS id: generate-changelogs + env: + EPHEMERAL_BRANCH: ${{ steps.ephemeral-branch.outputs.ephemeral_branch }} run: | set -euo pipefail ORIGINAL_HEAD=$(cat /tmp/release_head_sha) - echo "Generating CHANGELOGS" - - jq -c '.[]' /tmp/api-changes-with-major-bumps.json | while read -r bump; do - COMMITS=$(echo "$bump" | jq -r '.commits') - RANGE=$(echo "$bump" | jq -r '.range') - NAME=$(echo "$bump" | jq -r '.name') - TAG=$(echo "$bump" | jq -r '.prev_tag') - NEXT_TAG=$(echo "$bump" | jq -r '.tag') - VERSION=$(echo "$bump" | jq -r '.version') - CRATE_PATH=$(echo "$bump" | jq -r '.path') - INITIAL_RELEASE=$(echo "$bump" | jq -r '.initial_release') - MAJOR_BUMPS=$(echo "$bump" | jq -c '.major_bumps // []') - - if [ "$INITIAL_RELEASE" = "true" ]; then - echo "Initial release for $NAME" - - # Use the existing CHANGELOG.md if present, otherwise create a minimal one - if [ ! -f "$CRATE_PATH/CHANGELOG.md" ]; then - echo "Creating CHANGELOG.md for $NAME..." - RELEASE_DATE=$(date +%Y-%m-%d) - printf '# Changelog\n\n\n## %s - %s\n\nInitial release.\n' "$VERSION" "$RELEASE_DATE" > "$CRATE_PATH/CHANGELOG.md" - - git add "$CRATE_PATH/CHANGELOG.md" - git commit -m "chore(release): update CHANGELOG.md for $NAME" - else - echo "Using existing CHANGELOG.md for $NAME..." - fi - continue - fi - - # FIXME: $COMMITS could be empty if there are no commits since last release - if [ "$COMMITS" = "[]" ]; then - if [ "$MAJOR_BUMPS" != "[]" ] && [ "$MAJOR_BUMPS" != "null" ]; then - echo "No commits for $NAME but direct dependency major bumps; writing a minimal CHANGELOG entry" - RELEASE_DATE=$(date +%Y-%m-%d) - DEP_LINES=$(echo "$MAJOR_BUMPS" | jq -r '.[] | "- Bump `\(.dependency)` to a new major version (`\(.previous_req)` → `\(.current_req)`)"') - - # Match git-cliff's header (see cliff.toml): link the version to a compare view - # against the previous tag when one exists. - REMOTE_URL="https://github.com/datadog/libdatadog" - if [ -n "$TAG" ] && [ "$TAG" != "null" ]; then - HEADER="## [$VERSION]($REMOTE_URL/compare/$TAG..$NEXT_TAG) - $RELEASE_DATE" - else - HEADER="## [$VERSION] - $RELEASE_DATE" - fi - - ENTRY_FILE=$(mktemp /tmp/changelog-entry-XXXXXX.md) - printf '%s\n\n### Changed\n\n%s\n\n' "$HEADER" "$DEP_LINES" > "$ENTRY_FILE" - - if [ -f "$CRATE_PATH/CHANGELOG.md" ]; then - # Insert the new section above the first existing release section (newest-first), - # mirroring git-cliff --prepend placement and leaving the rest of the file intact. - awk 'NR==FNR { e = e $0 ORS; next } - !inserted && /^## / { printf "%s", e; inserted=1 } - { print } - END { if (!inserted) printf "%s", e }' \ - "$ENTRY_FILE" "$CRATE_PATH/CHANGELOG.md" > "$CRATE_PATH/CHANGELOG.md.tmp" - mv "$CRATE_PATH/CHANGELOG.md.tmp" "$CRATE_PATH/CHANGELOG.md" - else - printf '# Changelog\n\n\n' > "$CRATE_PATH/CHANGELOG.md" - cat "$ENTRY_FILE" >> "$CRATE_PATH/CHANGELOG.md" - fi - rm -f "$ENTRY_FILE" - - git add "$CRATE_PATH/CHANGELOG.md" - git commit -m "chore(release): update CHANGELOG.md for $NAME" - else - echo "No commits since last release for $NAME, skipping CHANGELOG generation" - fi - continue - fi - - # Build a tight range from commits already found by commits-since-release.sh. - # This will save some time analising unnecessary commits and prevent unrelated commits - # go through git-cliff filtering process. - NEWEST_COMMIT=$(echo "$COMMITS" | jq -r '.[0].hash // empty') - OLDEST_COMMIT=$(echo "$COMMITS" | jq -r '.[-1].hash // empty') - OLDEST_PARENT=$(git rev-parse "${OLDEST_COMMIT}^" 2>/dev/null || echo "") - if [ -n "$OLDEST_PARENT" ] && [ -n "$NEWEST_COMMIT" ]; then - COMMITS_RANGE="$OLDEST_PARENT..$NEWEST_COMMIT" - else - COMMITS_RANGE="$RANGE" - fi - echo "Executing git cliff for $NAME since $COMMITS_RANGE (oldest: $OLDEST_COMMIT, newest: $NEWEST_COMMIT), next tag: $NEXT_TAG..." - - # git-cliff's --include-path uses cumulative tree diffs rather than per-commit - # diffs. This causes commits that don't touch the crate to pass the filter if an - # earlier commit in the range does touch it. In order to avoid that a first pass - # will generate the context inside the commit range and then a second step will - # filter the the commits according to the previously computed range stored in COMMITS. - CLIFF_CONTEXT_FILE=$(mktemp /tmp/git-cliff-context-XXXXXX.json) - CLIFF_HASHES_FILE=$(mktemp /tmp/git-cliff-hashes-XXXXXX.json) - CLIFF_FILTERED_FILE=$(mktemp /tmp/git-cliff-filtered-XXXXXX.json) - - git cliff --context --tag "$NEXT_TAG" --ignore-tags ".*" -v "$COMMITS_RANGE" > "$CLIFF_CONTEXT_FILE" - echo "$COMMITS" | jq '[.[].hash]' > "$CLIFF_HASHES_FILE" - jq --slurpfile hashes "$CLIFF_HASHES_FILE" \ - --arg prev_tag "$TAG" \ - 'map(. + { - commits: [.commits[] | select(.id | IN($hashes[0][]))], - previous: (.previous + {"version": $prev_tag}) - })' \ - "$CLIFF_CONTEXT_FILE" > "$CLIFF_FILTERED_FILE" - git cliff --from-context "$CLIFF_FILTERED_FILE" -u -v --prepend "$CRATE_PATH/CHANGELOG.md" - rm -f "$CLIFF_CONTEXT_FILE" "$CLIFF_HASHES_FILE" "$CLIFF_FILTERED_FILE" - - git add "$CRATE_PATH/CHANGELOG.md" - git commit -m "chore(release): update CHANGELOG.md for $NAME" - done + "${WORKFLOW_SCRIPTS_ROOT}/release-generate-changelogs.sh" \ + --api-changes /tmp/api-changes-with-major-bumps.json # Check if there are commits to push - if git diff --quiet "${{ steps.ephemeral-branch.outputs.ephemeral_branch }}"; then + if git diff --quiet "$EPHEMERAL_BRANCH"; then echo "No changes to push. Cancelling the workflow." exit 1 fi @@ -963,7 +641,9 @@ jobs: NON_DEFAULT="${NON_DEFAULT}"$'\n\n' fi - # PR body from api-changes-with-major-bumps.json (same crates as api-changes.json; tags/versions updated after libdd major bumps). + # PR body from api-changes-with-major-bumps.json: the crates actually being released, + # i.e. api-changes.json minus the pending candidates that did not earn a major bump, + # with tags/versions updated for those that did. # Note: read returns 1 when it reaches EOF, which is expected for heredocs read -r -d '' JQ_FILTER << 'EOF' || true [ $api[0][] diff --git a/scripts/check_cargo_metadata.sh b/scripts/check_cargo_metadata.sh index f2e17f0637..52074db32a 100755 --- a/scripts/check_cargo_metadata.sh +++ b/scripts/check_cargo_metadata.sh @@ -142,8 +142,9 @@ check_internal_dependencies() { # Main validation function validate_cargo_toml() { local file="$1" - local crate_dir=$(dirname "$file") - local crate_name=$(basename "$crate_dir") + local crate_dir crate_name + crate_dir=$(dirname "$file") + crate_name=$(basename "$crate_dir") local has_error=0 # Skip the root workspace Cargo.toml diff --git a/scripts/commits-since-release.sh b/scripts/commits-since-release.sh index 8b5a0a9f32..1fd1631fd0 100755 --- a/scripts/commits-since-release.sh +++ b/scripts/commits-since-release.sh @@ -67,7 +67,22 @@ ${arg#--exclude=}" echo " ./commits-since-release.sh --exclude='^chore:' --exclude='^ci:' \"\$JSON\"" echo "" echo "Output JSON format:" - echo ' [{"name":"crate-name","version":"1.0.0","tag":"crate-name-v1.0.0","tag_exists":true,"commits":[...]}]' + echo ' [{"name":"crate-name","version":"1.0.0","path":"crate-name","tag":"crate-name-v1.0.0",' + echo ' "tag_exists":true,"tag_ancestor":"true","tag_commit":"",' + echo ' "tag_in_local_branch":true,"latest_tag":"crate-name-v1.0.0",' + echo ' "range":"..","commits":[...]}]' + echo "" + echo ' "tag_in_local_branch" is false when no local branch contains the tagged commit,' + echo ' which is normal for squash-merged releases. Always false when there is no tag.' + echo "" + echo ' "latest_tag" is the highest release tag that exists for the crate, which is not' + echo ' necessarily "tag": the manifest version lags behind when a release was cut' + echo ' elsewhere. Empty when the crate was never released.' + echo "" + echo ' "range" is the commit range the listed commits were taken from, resolved to' + echo ' SHAs: the tag commit, or the merge-base when the tag is not an ancestor of' + echo ' HEAD, or the parent of the oldest commit found when there is no common' + echo ' ancestor at all. Empty when the crate has no previous release tag.' exit 0 ;; -*) @@ -99,6 +114,10 @@ METADATA=$(cargo metadata --format-version=1 --no-deps 2>/dev/null) # Get workspace root (for determining crate paths) WORKSPACE_ROOT=$(echo "$METADATA" | jq -r '.workspace_root' || pwd) +# Resolve HEAD once, so every crate's exported range ends at the same commit and callers +# can reuse the range later without re-resolving HEAD (which may have moved on by then). +HEAD_COMMIT=$(git rev-parse HEAD) + log_verbose() { if [ "$VERBOSE" = true ]; then echo "$@" >&2 @@ -153,19 +172,35 @@ while read -r crate; do log_verbose " Crate path: $CRATE_PATH" + # Highest release tag that exists for this crate, which is not necessarily $TAG: the + # manifest version can lag behind the tags when a release was cut elsewhere (a hotfix + # branch, or a release already merged to main). Empty when the crate was never released. + # The `-v` in the glob keeps sibling crates out: libdd-common-v* cannot match + # libdd-common-ffi-v1.0.0. + LATEST_TAG=$(git tag -l "${NAME}-v*" --sort=-v:refname | head -1) + log_verbose " Latest tag: ${LATEST_TAG:-}" + # Check if tag exists TAG_EXISTS=false TAG_ANCESTOR="unknown" + TAG_COMMIT="" + TAG_IN_LOCAL_BRANCH=false + RANGE="" + RANGE_START="" COMMITS_JSON="[]" - + if git rev-parse "refs/tags/$TAG" >/dev/null 2>&1; then TAG_EXISTS=true log_verbose " Tag exists, finding commits since $TAG..." - + # Check if tag is an ancestor of HEAD (i.e., release was merged back to main) # If not, use merge-base to find the common ancestor. # Explicitly dereference annotated tags to their underlying commit: git merge-base does # not consistently dereference annotated tag objects across all git versions. + # + # RANGE_START is the same decision expressed as a commit SHA, and becomes the + # exported `range`. Callers need a range they can hand to other git tooling, so + # it is resolved to SHAs rather than left as a tag name. TAG_COMMIT=$(git rev-parse "${TAG}^{}" 2>/dev/null || echo "") if [ -z "$TAG_COMMIT" ]; then COMMIT_RANGE="$TAG..HEAD" @@ -174,20 +209,34 @@ while read -r crate; do elif git merge-base --is-ancestor "$TAG_COMMIT" HEAD 2>/dev/null; then COMMIT_RANGE="$TAG..HEAD" TAG_ANCESTOR="true" + RANGE_START="$TAG_COMMIT" log_verbose " Tag is ancestor of HEAD, using $COMMIT_RANGE" else MERGE_BASE=$(git merge-base "$TAG_COMMIT" HEAD 2>/dev/null || echo "") if [ -n "$MERGE_BASE" ]; then COMMIT_RANGE="$MERGE_BASE..HEAD" TAG_ANCESTOR="$MERGE_BASE" + RANGE_START="$MERGE_BASE" log_verbose " Tag is NOT ancestor of HEAD, using merge-base: $COMMIT_RANGE" else + # Tag is on unrelated history. RANGE_START is derived from the commits + # below, once we know which ones there are. COMMIT_RANGE="$TAG..HEAD" TAG_ANCESTOR="no merge-base" log_verbose " WARNING: Could not find merge-base, using $TAG..HEAD" fi fi - + + # Is the tagged commit reachable from any local branch? Squash-merged releases leave + # the tag on history no branch points at, which is normal but worth surfacing: it is + # also what a tag pushed from an abandoned branch looks like. Only the yes/no matters, + # so the branch names are deliberately not collected. + if [ -n "$TAG_COMMIT" ] && git branch --contains "$TAG_COMMIT" 2>/dev/null | grep -q .; then + TAG_IN_LOCAL_BRANCH=true + else + log_verbose " WARNING: $TAG ($TAG_COMMIT) is not in any local branch" + fi + # Get commits since tag that affect this crate's directory # Use ASCII unit separator (0x1F) as delimiter - won't appear in commit messages COMMITS_JSON="[" @@ -216,9 +265,37 @@ while read -r crate; do done < <(git log "$COMMIT_RANGE" --format="%H%x1F%s%x1F%an%x1F%aI" -- "$CRATE_PATH" 2>/dev/null || true) COMMITS_JSON+="]" - + COMMIT_COUNT=$(echo "$COMMITS_JSON" | jq 'length') log_verbose " Found $COMMIT_COUNT commits since $TAG" + + # No common ancestor with the tag: `$TAG..HEAD` spans HEAD's entire history, which + # is fine for the path-filtered log above but far too wide to hand to git-cliff. + # Start the exported range at the parent of the oldest commit we actually found, + # so it covers those commits and nothing else. + if [ -z "$RANGE_START" ] && [ -n "$TAG_COMMIT" ]; then + OLDEST_COMMIT=$(echo "$COMMITS_JSON" | jq -r '.[-1].hash // empty') + OLDEST_PARENT="" + if [ -n "$OLDEST_COMMIT" ]; then + # --verify matters: plain `git rev-parse ^` exits non-zero but + # still echoes "^" on stdout, so the `|| echo ""` fallback would never + # fire and the range would start at a ref that does not resolve. + OLDEST_PARENT=$(git rev-parse --verify "${OLDEST_COMMIT}^" 2>/dev/null || echo "") + fi + if [ -n "$OLDEST_PARENT" ]; then + RANGE_START="$OLDEST_PARENT" + log_verbose " No common ancestor with $TAG: range starts at the parent of the oldest commit" + else + RANGE_START="$TAG_COMMIT" + log_verbose " WARNING: Could not derive a range start for $TAG, falling back to the tag commit" + fi + fi + + # Empty only when the tag could not be dereferenced at all. + if [ -n "$RANGE_START" ]; then + RANGE="${RANGE_START}..${HEAD_COMMIT}" + log_verbose " Range: $RANGE" + fi else log_verbose " Tag does NOT exist - no previous release found" fi @@ -230,7 +307,7 @@ while read -r crate; do OUTPUT_JSON+="," fi - OUTPUT_JSON+="{\"name\":\"$NAME\",\"version\":\"$VERSION\",\"path\":\"$CRATE_PATH\",\"tag\":\"$TAG\",\"tag_exists\":$TAG_EXISTS,\"tag_ancestor\":\"$TAG_ANCESTOR\",\"commits\":$COMMITS_JSON}" + OUTPUT_JSON+="{\"name\":\"$NAME\",\"version\":\"$VERSION\",\"path\":\"$CRATE_PATH\",\"tag\":\"$TAG\",\"tag_exists\":$TAG_EXISTS,\"tag_ancestor\":\"$TAG_ANCESTOR\",\"tag_commit\":\"$TAG_COMMIT\",\"tag_in_local_branch\":$TAG_IN_LOCAL_BRANCH,\"latest_tag\":\"$LATEST_TAG\",\"range\":\"$RANGE\",\"commits\":$COMMITS_JSON}" done < <(echo "$INPUT_JSON" | jq -c '.[]') diff --git a/scripts/major-bumps-level.sh b/scripts/major-bumps-level.sh index 6bc333a0f4..136578ceb7 100755 --- a/scripts/major-bumps-level.sh +++ b/scripts/major-bumps-level.sh @@ -21,15 +21,6 @@ usage() { API_JSON=$1 [[ -f "$API_JSON" ]] || { echo "Not a file: $API_JSON" >&2; exit 2; } -level_rank() { - case "${1:-}" in - patch) echo 0 ;; - minor) echo 1 ;; - major) echo 2 ;; - *) echo -1 ;; - esac -} - libdd_deps_for_crate() { local manifest crate manifest=$1 @@ -90,7 +81,6 @@ trap cleanup EXIT n=$(jq length "$API_JSON") OUT=$(mktemp) echo '[]' >"$OUT" -FAIL=0 for ((i = 0; i < n; i++)); do row=$(jq -c ".[$i]" "$API_JSON") diff --git a/scripts/release-generate-changelogs.sh b/scripts/release-generate-changelogs.sh new file mode 100755 index 0000000000..a3d0b1f681 --- /dev/null +++ b/scripts/release-generate-changelogs.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash + +# Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +# SPDX-License-Identifier: Apache-2.0 + +# Release Generate CHANGELOGs Script +# Writes and commits a CHANGELOG.md entry for every crate in the release set. +# +# Usage: ./release-generate-changelogs.sh --api-changes FILE +# +# Input is the audited release set from release-version-major-bumps.sh. Each crate +# takes one of four paths: +# +# initial release an existing CHANGELOG.md is left alone; otherwise a minimal +# "Initial release." file is created. +# no commits, but a direct libdd-* dependency went major: a minimal entry listing +# the dependency bumps, formatted to match git-cliff's header so +# the file stays consistent. +# no commits at all no entry; nothing to say. +# commits git-cliff, in two passes (see below). +# +# Every entry that is written is committed as "chore(release): update CHANGELOG.md +# for ". Runs from the repository root, where cliff.toml lives. + +set -euo pipefail + +API_CHANGES="" +REMOTE_URL="https://github.com/datadog/libdatadog" + +usage() { + echo "Usage: $0 --api-changes FILE" + echo "" + echo "Options:" + echo " --api-changes FILE Audited release set from release-version-major-bumps.sh (required)" + echo " --remote-url URL Repository URL used in generated compare links" + echo " (default: $REMOTE_URL)" + echo " --help, -h Show this message" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --api-changes) API_CHANGES="${2:?--api-changes needs a value}"; shift 2 ;; + --remote-url) REMOTE_URL="${2:?--remote-url needs a value}"; shift 2 ;; + --help|-h) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage >&2; exit 1 ;; + esac +done + +[ -n "$API_CHANGES" ] || { echo "ERROR: --api-changes is required" >&2; exit 1; } +[ -f "$API_CHANGES" ] || { echo "ERROR: not a file: $API_CHANGES" >&2; exit 1; } +jq -e 'type == "array"' "$API_CHANGES" >/dev/null \ + || { echo "ERROR: $API_CHANGES is not a JSON array" >&2; exit 1; } + +echo "Generating CHANGELOGS" + +# Materialize the rows before looping. `done < <(jq ...)` would run jq in a process +# substitution, whose exit status no shell option reports: set -e and pipefail both +# ignore it, so a jq that dies mid-stream would leave the loop with no input and this +# script would exit 0 having written no CHANGELOG at all. The caller cannot catch that +# either: its no-changes-to-push guard sees the version-bump commits from the previous +# step and concludes there is something to release. +RELEASE_ROWS=$(jq -c '.[]' "$API_CHANGES") + +while read -r bump; do + # $ROWS is empty when there are no candidates; <<< still feeds one blank line. + [ -n "$bump" ] || continue + COMMITS=$(echo "$bump" | jq -r '.commits') + RANGE=$(echo "$bump" | jq -r '.range') + NAME=$(echo "$bump" | jq -r '.name') + TAG=$(echo "$bump" | jq -r '.prev_tag') + NEXT_TAG=$(echo "$bump" | jq -r '.tag') + VERSION=$(echo "$bump" | jq -r '.version') + CRATE_PATH=$(echo "$bump" | jq -r '.path') + INITIAL_RELEASE=$(echo "$bump" | jq -r '.initial_release') + MAJOR_BUMPS=$(echo "$bump" | jq -c '.major_bumps // []') + + if [ "$INITIAL_RELEASE" = "true" ]; then + echo "Initial release for $NAME" + + # Use the existing CHANGELOG.md if present, otherwise create a minimal one + if [ ! -f "$CRATE_PATH/CHANGELOG.md" ]; then + echo "Creating CHANGELOG.md for $NAME..." + RELEASE_DATE=$(date +%Y-%m-%d) + printf '# Changelog\n\n\n## %s - %s\n\nInitial release.\n' "$VERSION" "$RELEASE_DATE" > "$CRATE_PATH/CHANGELOG.md" + + git add "$CRATE_PATH/CHANGELOG.md" + git commit -m "chore(release): update CHANGELOG.md for $NAME" + else + echo "Using existing CHANGELOG.md for $NAME..." + fi + continue + fi + + # FIXME: $COMMITS could be empty if there are no commits since last release + if [ "$COMMITS" = "[]" ]; then + if [ "$MAJOR_BUMPS" != "[]" ] && [ "$MAJOR_BUMPS" != "null" ]; then + echo "No commits for $NAME but direct dependency major bumps; writing a minimal CHANGELOG entry" + RELEASE_DATE=$(date +%Y-%m-%d) + DEP_LINES=$(echo "$MAJOR_BUMPS" | jq -r '.[] | "- Bump `\(.dependency)` to a new major version (`\(.previous_req)` → `\(.current_req)`)"') + + # Match git-cliff's header (see cliff.toml): link the version to a compare view + # against the previous tag when one exists. + if [ -n "$TAG" ] && [ "$TAG" != "null" ]; then + HEADER="## [$VERSION]($REMOTE_URL/compare/$TAG..$NEXT_TAG) - $RELEASE_DATE" + else + HEADER="## [$VERSION] - $RELEASE_DATE" + fi + + ENTRY_FILE=$(mktemp "${TMPDIR:-/tmp}/changelog-entry-XXXXXX.md") + printf '%s\n\n### Changed\n\n%s\n\n' "$HEADER" "$DEP_LINES" > "$ENTRY_FILE" + + if [ -f "$CRATE_PATH/CHANGELOG.md" ]; then + # Insert the new section above the first existing release section (newest-first), + # mirroring git-cliff --prepend placement and leaving the rest of the file intact. + awk 'NR==FNR { e = e $0 ORS; next } + !inserted && /^## / { printf "%s", e; inserted=1 } + { print } + END { if (!inserted) printf "%s", e }' \ + "$ENTRY_FILE" "$CRATE_PATH/CHANGELOG.md" > "$CRATE_PATH/CHANGELOG.md.tmp" + mv "$CRATE_PATH/CHANGELOG.md.tmp" "$CRATE_PATH/CHANGELOG.md" + else + printf '# Changelog\n\n\n' > "$CRATE_PATH/CHANGELOG.md" + cat "$ENTRY_FILE" >> "$CRATE_PATH/CHANGELOG.md" + fi + rm -f "$ENTRY_FILE" + + git add "$CRATE_PATH/CHANGELOG.md" + git commit -m "chore(release): update CHANGELOG.md for $NAME" + else + echo "No commits since last release for $NAME, skipping CHANGELOG generation" + fi + continue + fi + + # Build a tight range from commits already found by commits-since-release.sh. + # This will save some time analising unnecessary commits and prevent unrelated commits + # go through git-cliff filtering process. + NEWEST_COMMIT=$(echo "$COMMITS" | jq -r '.[0].hash // empty') + OLDEST_COMMIT=$(echo "$COMMITS" | jq -r '.[-1].hash // empty') + # --verify matters: plain `git rev-parse ^` exits non-zero but still + # echoes "^" on stdout, so the `|| echo ""` fallback to $RANGE would never + # fire and git-cliff would be handed a range start that does not resolve. + OLDEST_PARENT=$(git rev-parse --verify "${OLDEST_COMMIT}^" 2>/dev/null || echo "") + if [ -n "$OLDEST_PARENT" ] && [ -n "$NEWEST_COMMIT" ]; then + COMMITS_RANGE="$OLDEST_PARENT..$NEWEST_COMMIT" + else + COMMITS_RANGE="$RANGE" + fi + echo "Executing git cliff for $NAME since $COMMITS_RANGE (oldest: $OLDEST_COMMIT, newest: $NEWEST_COMMIT), next tag: $NEXT_TAG..." + + # git-cliff's --include-path uses cumulative tree diffs rather than per-commit + # diffs. This causes commits that don't touch the crate to pass the filter if an + # earlier commit in the range does touch it. In order to avoid that a first pass + # will generate the context inside the commit range and then a second step will + # filter the the commits according to the previously computed range stored in COMMITS. + CLIFF_CONTEXT_FILE=$(mktemp "${TMPDIR:-/tmp}/git-cliff-context-XXXXXX.json") + CLIFF_HASHES_FILE=$(mktemp "${TMPDIR:-/tmp}/git-cliff-hashes-XXXXXX.json") + CLIFF_FILTERED_FILE=$(mktemp "${TMPDIR:-/tmp}/git-cliff-filtered-XXXXXX.json") + + git cliff --context --tag "$NEXT_TAG" --ignore-tags ".*" -v "$COMMITS_RANGE" > "$CLIFF_CONTEXT_FILE" + echo "$COMMITS" | jq '[.[].hash]' > "$CLIFF_HASHES_FILE" + jq --slurpfile hashes "$CLIFF_HASHES_FILE" \ + --arg prev_tag "$TAG" \ + 'map(. + { + commits: [.commits[] | select(.id | IN($hashes[0][]))], + previous: (.previous + {"version": $prev_tag}) + })' \ + "$CLIFF_CONTEXT_FILE" > "$CLIFF_FILTERED_FILE" + git cliff --from-context "$CLIFF_FILTERED_FILE" -u -v --prepend "$CRATE_PATH/CHANGELOG.md" + rm -f "$CLIFF_CONTEXT_FILE" "$CLIFF_HASHES_FILE" "$CLIFF_FILTERED_FILE" + + git add "$CRATE_PATH/CHANGELOG.md" + git commit -m "chore(release): update CHANGELOG.md for $NAME" +done <<< "$RELEASE_ROWS" diff --git a/scripts/release-version-bumps.sh b/scripts/release-version-bumps.sh new file mode 100755 index 0000000000..9851f1f111 --- /dev/null +++ b/scripts/release-version-bumps.sh @@ -0,0 +1,203 @@ +#!/usr/bin/env bash + +# Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +# SPDX-License-Identifier: Apache-2.0 + +# Release Version Bumps Script +# Turns the output of commits-since-release.sh into api-changes.json, running +# `cargo release version` for each crate that is actually being released. +# +# Usage: ./release-version-bumps.sh --commits-by-crate FILE --out FILE --branch BRANCH +# [--hotfix] [--bypass-standard-checks] +# +# For every crate in the input, one of four things happens: +# +# deferred no commits of its own but a tag exists -- recorded with +# "pending_release": "true" and level "none", so the caller's libdd-* +# major-bump check can pull it back into the release, or drop it. +# skipped its tag is not the latest for that crate, so a newer release already +# exists elsewhere. Overridden by --hotfix and --bypass-standard-checks. +# released semver-level.sh picks the level, cargo-release applies it. +# initial no tag at all: released at 0.1.0, or the run fails. +# +# Diagnostics go to stdout (they are the caller's job log); the JSON result is written +# to --out. semver-level.sh is resolved next to this script, so it always comes from +# the same checkout as the caller. + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" + +COMMITS_BY_CRATE="" +OUT_FILE="" +BRANCH_NAME="" +IS_HOTFIX=false +BYPASS_STANDARD_CHECKS=false + +usage() { + echo "Usage: $0 --commits-by-crate FILE --out FILE --branch BRANCH [--hotfix] [--bypass-standard-checks]" + echo "" + echo "Options:" + echo " --commits-by-crate FILE Output of commits-since-release.sh (required)" + echo " --out FILE Where to write the api-changes JSON array (required)" + echo " --branch BRANCH Branch cargo-release is allowed to operate on (required)" + echo " --hotfix Release even when the crate's tag is not the latest" + echo " --bypass-standard-checks Same, for testing runs" + echo " --help, -h Show this message" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --commits-by-crate) COMMITS_BY_CRATE="${2:?--commits-by-crate needs a value}"; shift 2 ;; + --out) OUT_FILE="${2:?--out needs a value}"; shift 2 ;; + --branch) BRANCH_NAME="${2:?--branch needs a value}"; shift 2 ;; + --hotfix) IS_HOTFIX=true; shift ;; + --bypass-standard-checks) BYPASS_STANDARD_CHECKS=true; shift ;; + --help|-h) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage >&2; exit 1 ;; + esac +done + +[ -n "$COMMITS_BY_CRATE" ] || { echo "ERROR: --commits-by-crate is required" >&2; exit 1; } +[ -n "$OUT_FILE" ] || { echo "ERROR: --out is required" >&2; exit 1; } +[ -n "$BRANCH_NAME" ] || { echo "ERROR: --branch is required" >&2; exit 1; } +[ -f "$COMMITS_BY_CRATE" ] || { echo "ERROR: not a file: $COMMITS_BY_CRATE" >&2; exit 1; } +jq -e 'type == "array"' "$COMMITS_BY_CRATE" >/dev/null \ + || { echo "ERROR: $COMMITS_BY_CRATE is not a JSON array" >&2; exit 1; } + +echo "Release version bumps..." + +# Initialize results array. It holds one row per candidate crate: those released here, +# and those with no commits of their own, which carry "pending_release": "true" and are +# only released if the libdd-* major-bump check pulls them back in. +echo "[]" > "$OUT_FILE" + +append_row() { + # append_row JQ_ARGS... -- reads $OUT_FILE, appends, writes back. + local tmp="${OUT_FILE}.tmp" + jq "$@" "$OUT_FILE" > "$tmp" && mv "$tmp" "$OUT_FILE" +} + +# Materialize the rows before looping. `done < <(jq ...)` would run jq in a process +# substitution, whose exit status no shell option reports: set -e and pipefail both +# ignore it, so a jq that dies mid-stream would leave the loop with no input and this +# script would exit 0 having released nothing. +CRATE_ROWS=$(jq -c '.[]' "$COMMITS_BY_CRATE") + +# iterate over the commits and execute cargo release for each crate +while read -r crate; do + # $ROWS is empty when there are no candidates; <<< still feeds one blank line. + [ -n "$crate" ] || continue + NAME=$(echo "$crate" | jq -r '.name') + TAG=$(echo "$crate" | jq -r '.tag') + TAG_PREFIX="$NAME-v" + CRATE_PATH=$(echo "$crate" | jq -r '.path') + TAG_EXISTS=$(echo "$crate" | jq -r '.tag_exists') + COMMITS=$(echo "$crate" | jq -r '.commits') + INITIAL_RELEASE=false + TAG_COMMIT="" + RANGE="" + LEVEL="" + + # if there are no commits and there is an existing tag, do not release the crate here. + # but record it as a pending candidate + if [ "$COMMITS" = "[]" ] && [ "$TAG_EXISTS" = "true" ]; then + VERSION=$(echo "$crate" | jq -r '.version') + echo "No commits since last release for $NAME; deferring to the libdd-* major-bump check" + append_row --arg name "$NAME" \ + --arg tag "$TAG" \ + --arg version "$VERSION" \ + --arg path "$CRATE_PATH" \ + '. += [{"name": $name, "level": "none", "tag": $tag, "prev_tag": $tag, "version": $version, "range": "", "commits": [], "path": $path, "initial_release": "false", "pending_release": "true"}]' + continue + fi + + if [ "$TAG_EXISTS" = "true" ]; then + TAG_COMMIT=$(echo "$crate" | jq -r '.tag_commit') + RANGE=$(echo "$crate" | jq -r '.range') + if [ -z "$TAG_COMMIT" ] || [ -z "$RANGE" ]; then + echo "ERROR: Could not dereference tag $TAG to a commit" >&2 + exit 1 + fi + echo "Using $RANGE as range (tag: $TAG)" + + if [ "$(echo "$crate" | jq -r '.tag_in_local_branch')" != "true" ]; then + echo "Warning: tag $TAG (commit $TAG_COMMIT) is not in any local branch (normal for squash-merged releases)" + fi + + # if there is a tag more recent than $TAG, continue the loop. + LATEST_TAG=$(echo "$crate" | jq -r '.latest_tag') + if [ "$LATEST_TAG" != "$TAG" ]; then + echo "Tag $TAG is not the latest. Latest is: $LATEST_TAG. main branch has the latest release for $NAME" + + # do not skip the release for hotfix branches + if [ "$IS_HOTFIX" = "true" ]; then + echo "Continuing with the release for $NAME because it is a hotfix" + else + if [ "$BYPASS_STANDARD_CHECKS" = "false" ]; then + echo "Skipping release for $NAME" + continue + else + echo "Continuing with the release for $NAME because bypass_standard_checks is true" + fi + fi + fi + + echo "Executing semver-level.sh for $NAME since $RANGE (tag: $TAG)..." + # stderr is folded in so the reason travels with a failure; without this the + # capture swallows it and the run aborts with nothing to go on. + if ! SEMVER_LEVEL=$("${SCRIPT_DIR}/semver-level.sh" "$NAME" "refs/tags/$TAG" 2>&1); then + echo "ERROR: semver-level.sh failed for $NAME:" >&2 + echo "$SEMVER_LEVEL" >&2 + exit 1 + fi + echo "Semver level: $SEMVER_LEVEL" + + LEVEL=$(echo "$SEMVER_LEVEL" | jq -r '.level') + + echo "Executing cargo release for $NAME since $TAG with level $LEVEL..." + cargo release version -p "$NAME" --prev-tag-name "$TAG" --allow-branch "$BRANCH_NAME" -x "$LEVEL" --no-confirm + + else + echo "No previous release tag for $NAME, preparing initial release..." + + # Use the version from the crate metadata + VERSION=$(echo "$crate" | jq -r '.version') + LEVEL="major" + TAG="" + RANGE="" + + # fail when the version is not an initial release + if [ "$VERSION" != "0.1.0" ]; then + echo "Error: $NAME is not a 0.1.0 release" >&2 + exit 1 + fi + + INITIAL_RELEASE=true + + echo "Executing cargo release for $NAME with level $LEVEL..." + cargo release version -p "$NAME" --allow-branch "$BRANCH_NAME" -x "$LEVEL" --no-confirm + fi + + # Commit the changes + cargo release commit --no-confirm -x + + NEXT_VERSION=$(cargo metadata --format-version=1 --no-deps | jq -r --arg name "$NAME" '.packages[] | select(.name == $name) | .version') + NEXT_TAG="$TAG_PREFIX$NEXT_VERSION" + + # Add to results array + append_row --arg name "$NAME" \ + --arg level "$LEVEL" \ + --arg tag "$NEXT_TAG" \ + --arg prev_tag "$TAG" \ + --arg version "$NEXT_VERSION" \ + --arg range "$RANGE" \ + --argjson commits "$COMMITS" \ + --arg path "$CRATE_PATH" \ + --arg initial_release "$INITIAL_RELEASE" \ + '. += [{"name": $name, "level": $level, "tag": $tag, "prev_tag": $prev_tag, "version": $version, "range": $range, "commits": $commits, "path": $path, "initial_release": $initial_release}]' +done <<< "$CRATE_ROWS" + +# Output the results +echo "API changes summary:" +jq . "$OUT_FILE" diff --git a/scripts/release-version-major-bumps.sh b/scripts/release-version-major-bumps.sh new file mode 100755 index 0000000000..2c1a77d59a --- /dev/null +++ b/scripts/release-version-major-bumps.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash + +# Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +# SPDX-License-Identifier: Apache-2.0 + +# Release Version Major Bumps Script +# Audits every release candidate for direct libdd-* dependencies that went to a new +# major version, and promotes the crates that need it. +# +# Usage: ./release-version-major-bumps.sh --api-changes FILE --out FILE --branch BRANCH +# +# Input is the api-changes array produced by release-version-bumps.sh: the crates +# released there, plus the no-commit candidates carrying "pending_release": "true". +# Every row is audited the same way; only what happens to the result differs. +# +# released, no bump kept as it is +# released, already major kept as it is; nothing more to do +# released, below major promoted to major, version and tag updated +# pending, earns a bump promoted to major and pulled into the release +# pending, earns nothing dropped from the release entirely +# +# Diagnostics go to stdout (they are the caller's job log); the JSON result is written +# to --out. major-bumps-level.sh is resolved next to this script, so it always comes +# from the same checkout as the caller. + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" + +API_CHANGES="" +OUT_FILE="" +BRANCH_NAME="" + +usage() { + echo "Usage: $0 --api-changes FILE --out FILE --branch BRANCH" + echo "" + echo "Options:" + echo " --api-changes FILE Output of release-version-bumps.sh (required)" + echo " --out FILE Where to write the audited JSON array (required)" + echo " --branch BRANCH Branch cargo-release is allowed to operate on (required)" + echo " --help, -h Show this message" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --api-changes) API_CHANGES="${2:?--api-changes needs a value}"; shift 2 ;; + --out) OUT_FILE="${2:?--out needs a value}"; shift 2 ;; + --branch) BRANCH_NAME="${2:?--branch needs a value}"; shift 2 ;; + --help|-h) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage >&2; exit 1 ;; + esac +done + +[ -n "$API_CHANGES" ] || { echo "ERROR: --api-changes is required" >&2; exit 1; } +[ -n "$OUT_FILE" ] || { echo "ERROR: --out is required" >&2; exit 1; } +[ -n "$BRANCH_NAME" ] || { echo "ERROR: --branch is required" >&2; exit 1; } +[ -f "$API_CHANGES" ] || { echo "ERROR: not a file: $API_CHANGES" >&2; exit 1; } +jq -e 'type == "array"' "$API_CHANGES" >/dev/null \ + || { echo "ERROR: $API_CHANGES is not a JSON array" >&2; exit 1; } + +# Resolve to an absolute path while we are still in the caller's directory. The audit +# below runs in `cd "$MAJOR_BUMPS_WT"`, so a relative --api-changes would be looked up +# from the throwaway worktree and fail there, having validated fine here. +API_CHANGES="$(cd -- "$(dirname -- "$API_CHANGES")" && pwd)/$(basename -- "$API_CHANGES")" + +AUDITED=$(mktemp "${TMPDIR:-/tmp}/api-changes-with-major-bumps-pre-commit.XXXXXX.json") +cleanup() { rm -f "$AUDITED"; } +trap cleanup EXIT + +# Run the audit in a throwaway worktree so extra worktrees / cargo metadata do not touch +# the caller's checkout. Check it out at the proposal branch tip (HEAD) — the released ref +# plus this run's version bumps from the previous step. This is deliberate on both ends: +# - It includes the dependency-requirement rewrites cargo-release made in the previous +# step, so a dependency bumped to a new major IN THIS proposal propagates a major bump +# to its dependents (e.g. protobuf 3->4 forces its dependents major). +# - It is built from the released ref, NOT the workflow revision, so changes present only +# on current main (and absent from a hotfix/older-ref release) never trigger a +# spurious bump. +MAJOR_BUMPS_WT=$(mktemp -d "${RUNNER_TEMP:-${TMPDIR:-/tmp}}/major-bumps-wt.XXXXXX") +PROPOSAL_SHA=$(git rev-parse HEAD) + +git worktree add --detach "$MAJOR_BUMPS_WT" "$PROPOSAL_SHA" +set +e +( cd "$MAJOR_BUMPS_WT" && "${SCRIPT_DIR}/major-bumps-level.sh" "$API_CHANGES" ) > "$AUDITED" +MB_RC=$? +git worktree remove --force "$MAJOR_BUMPS_WT" || true +set -e +if [[ "$MB_RC" -ne 0 ]]; then + echo "Major bumps level script failed with code $MB_RC" + echo "Major bumps level script output:" + cat "$AUDITED" + exit "$MB_RC" +fi + +# Seed the result with every already-released crate. Pending crates are appended below +# only if they earn a major bump; those that do not stay out of the release entirely. +jq '[.[] | select(.pending_release != "true") | del(.pending_release)]' "$AUDITED" > "$OUT_FILE" + +# Materialize the rows before looping. `done < <(jq ...)` would run jq in a process +# substitution, whose exit status no shell option reports: set -e and pipefail both +# ignore it, so a jq that dies mid-stream would leave the loop with no input and this +# script would exit 0 having promoted nothing. +AUDIT_ROWS=$(jq -c '.[]' "$AUDITED") + +# iterate over the crates and, where a direct libdd-* dependency had a major bump, update the version +while read -r bump; do + # $ROWS is empty when there are no candidates; <<< still feeds one blank line. + [ -n "$bump" ] || continue + NAME=$(echo "$bump" | jq -r '.name') + LEVEL=$(echo "$bump" | jq -r '.level') + PREV_TAG=$(echo "$bump" | jq -r '.prev_tag') + TAG=$(echo "$bump" | jq -r '.tag') + VERSION=$(echo "$bump" | jq -r '.version') + PENDING=$(echo "$bump" | jq -r '.pending_release // "false"') + MAJOR_BUMPS=$(echo "$bump" | jq -c '.major_bumps') + + if [ "$MAJOR_BUMPS" = "[]" ]; then + if [ "$PENDING" = "true" ]; then + echo "No commits and no direct dependency major bumps for $NAME, keeping it out of the release" + fi + continue + fi + + # A crate already bumped to major in the previous step needs nothing more. Pending + # crates always have level "none" here, so this only short-circuits released crates. + if [ "$LEVEL" = "major" ]; then + echo "Skipping $NAME: already bumped at major level in the previous step (major_bumps: $MAJOR_BUMPS)" + continue + fi + + # Bump to major: either a pending (no-commit) crate whose direct dependency went major, + # or a released crate bumped below major in the previous step. Both are handled the same. + echo "Bumping $NAME to major due to direct dependency major bumps: $MAJOR_BUMPS" + cargo release version -p "$NAME" --prev-tag-name "$PREV_TAG" --allow-branch "$BRANCH_NAME" -x major --no-confirm + + git commit -am "chore(release): update version for $NAME with major bumps" + + NEXT_VERSION=$(cargo metadata --format-version=1 --no-deps | jq -r --arg name "$NAME" '.packages[] | select(.name == $name) | .version') + NEXT_TAG="$NAME-v$NEXT_VERSION" + + echo "Updating tag $TAG to $NEXT_TAG and version $VERSION to $NEXT_VERSION for $NAME" + + # Released crates are already in the result (seeded above): update them in place. Pending + # crates are not: append them. The row is derived from the audit entry either way. + ROW=$(echo "$bump" | jq --arg version "$NEXT_VERSION" --arg tag "$NEXT_TAG" \ + 'del(.pending_release) | . + {level: "major", version: $version, tag: $tag}') + jq --argjson row "$ROW" \ + 'if any(.[]; .name == $row.name) + then map(if .name == $row.name then $row else . end) + else . + [$row] end' \ + "$OUT_FILE" > "${OUT_FILE}.tmp" \ + && mv "${OUT_FILE}.tmp" "$OUT_FILE" +done <<< "$AUDIT_ROWS" + +# Output the results +echo "API changes with major bumps summary:" +jq . "$OUT_FILE" diff --git a/scripts/semver-level.sh b/scripts/semver-level.sh index abfd530616..069b1113a4 100755 --- a/scripts/semver-level.sh +++ b/scripts/semver-level.sh @@ -261,8 +261,17 @@ compute_semver_results() { '{"name": $name, "level": $level, "reason": $reason, "details": $details}')" } -# Run the computation and capture JSON output +# Run the computation and capture JSON output. +# compute_semver_results runs in a command substitution, so the `exit` calls in its +# error paths only terminate that subshell. Propagate the status explicitly, otherwise +# a tool failure leaves RESULT_JSON empty and the script still exits 0 — the caller +# then reads an empty semver level and bumps the crate with whatever cargo-release +# makes of it. RESULT_JSON=$(compute_semver_results "$CRATE" "$BASE_REF" "$CURRENT_REF") +COMPUTE_EXIT_CODE=$? +if [[ $COMPUTE_EXIT_CODE -ne 0 ]]; then + exit $COMPUTE_EXIT_CODE +fi # Output JSON to stdout (captured by workflow) echo "$RESULT_JSON"