From 8a74c2382b9cfb03538bc9433fb922e0ba169862 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 13 Jul 2026 18:45:12 -0700 Subject: [PATCH 01/28] ci: publish Codex Git release artifacts --- .github/workflows/codex-release.yml | 461 ++++++++++++++++++++++++++++ 1 file changed, 461 insertions(+) create mode 100644 .github/workflows/codex-release.yml diff --git a/.github/workflows/codex-release.yml b/.github/workflows/codex-release.yml new file mode 100644 index 00000000000000..025d3a1998d450 --- /dev/null +++ b/.github/workflows/codex-release.yml @@ -0,0 +1,461 @@ +name: Codex Git release + +on: + push: + branches: + - codex + +permissions: + contents: read + +concurrency: + group: codex-git-release-${{ github.sha }} + cancel-in-progress: false + +jobs: + version: + name: Determine version + runs-on: ubuntu-24.04 + outputs: + describe: ${{ steps.version.outputs.describe }} + upstream_tag: ${{ steps.version.outputs.upstream_tag }} + version: ${{ steps.version.outputs.version }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Derive OpenAI version from git describe + id: version + shell: bash + run: | + set -euo pipefail + describe="$(git describe \ + --match 'v[0-9]*' \ + --exclude 'v*-openai.*' \ + --long \ + --always \ + --abbrev=12 \ + "$GITHUB_SHA")" + + if [[ "$describe" =~ ^(.+)-([0-9]+)-g([0-9a-f]+)$ ]] + then + upstream_tag="${BASH_REMATCH[1]}" + version="$upstream_tag-openai.${BASH_REMATCH[2]}.g${BASH_REMATCH[3]}" + else + upstream_tag= + version="openai-$describe" + fi + git check-ref-format "refs/tags/$version" + printf 'describe=%s\n' "$describe" | tee -a "$GITHUB_OUTPUT" + printf 'upstream_tag=%s\n' "$upstream_tag" | tee -a "$GITHUB_OUTPUT" + printf 'version=%s\n' "$version" | tee -a "$GITHUB_OUTPUT" + + build: + name: ${{ matrix.name }} + needs: version + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - name: macOS arm64 + os: macos-15 + target_platform: macOS + asset_platform: macOS + arch: arm64 + binary: /tmp/build/git/bin/git + file_pattern: Mach-O 64-bit executable arm64 + has_gcm: true + - name: macOS x64 + os: macos-15-intel + target_platform: macOS + asset_platform: macOS + arch: x64 + binary: /tmp/build/git/bin/git + file_pattern: Mach-O 64-bit executable x86_64 + has_gcm: true + - name: Linux arm64 + os: ubuntu-22.04 + target_platform: ubuntu + asset_platform: ubuntu + arch: arm64 + binary: /tmp/build/git/bin/git + file_pattern: ELF 64-bit.*ARM aarch64 + has_gcm: false + - name: Linux x64 + os: ubuntu-22.04 + target_platform: ubuntu + asset_platform: ubuntu + arch: x64 + binary: /tmp/build/git/bin/git + file_pattern: ELF 64-bit.*x86-64 + has_gcm: true + - name: Windows arm64 + os: windows-2025 + target_platform: win32 + asset_platform: windows + arch: arm64 + binary: /tmp/build/git/clangarm64/bin/git.exe + file_pattern: PE32\+.*Aarch64 + has_gcm: true + sdk_arch: aarch64 + sdk_flavor: full + mingw_dir: clangarm64 + mingit_arch: arm64 + mingit_filename: MinGit-2.55.0.2-arm64.zip + mingit_url: https://github.com/git-for-windows/git/releases/download/v2.55.0.windows.2/MinGit-2.55.0.2-arm64.zip + mingit_sha256: 0b2b81fdce284efd174cbb51b886ccea2fd271679c4b5c21f07d9e03bae51413 + - name: Windows x64 + os: windows-2025 + target_platform: win32 + asset_platform: windows + arch: x64 + binary: /tmp/build/git/mingw64/bin/git.exe + file_pattern: PE32\+.*x86-64 + has_gcm: true + sdk_arch: x86_64 + sdk_flavor: minimal + mingw_dir: mingw64 + mingit_arch: amd64 + mingit_filename: MinGit-2.55.0.2-64-bit.zip + mingit_url: https://github.com/git-for-windows/git/releases/download/v2.55.0.windows.2/MinGit-2.55.0.2-64-bit.zip + mingit_sha256: e3ea2944cea4b3fabcd69c7c1669ef69b1b66c05ac7806d81224d0abad2dec31 + + steps: + # Keep the packaging contract, dependency pins, and platform build logic + # aligned with the artifacts already consumed by Codex and GitHub Desktop. + - name: Check out Dugite Native + uses: actions/checkout@v6 + with: + repository: desktop/dugite-native + ref: f97e50add48cdcff053a69d95aa343a4a4a258c2 + path: dugite-native + fetch-depth: 0 + persist-credentials: false + + - name: Check out this Git revision + uses: actions/checkout@v6 + with: + ref: ${{ github.sha }} + path: dugite-native/git + fetch-depth: 1 + persist-credentials: false + + - name: Give the source an immutable package version + shell: bash + working-directory: dugite-native/git + env: + VERSION: ${{ needs.version.outputs.version }} + run: | + git \ + -c 'user.name=github-actions[bot]' \ + -c 'user.email=41898282+github-actions[bot]@users.noreply.github.com' \ + tag -a "$VERSION" -m "$VERSION" + + # Match Dugite Native's compatibility choice for its macOS x64 build. + - name: Select Xcode 16.4 + if: matrix.target_platform == 'macOS' && matrix.arch == 'x64' + run: | + sudo xcode-select -s /Applications/Xcode_16.4.app/Contents/Developer/ + sudo rm -rf /Library/Developer/CommandLineTools + + - name: Install Linux build dependencies + if: matrix.target_platform == 'ubuntu' + run: | + sudo apt-get update + sudo apt-get install -y \ + autoconf \ + automake \ + build-essential \ + ca-certificates \ + curl \ + gettext \ + jq \ + lsb-release \ + pkg-config + + - name: Install Linux x64 build dependencies + if: matrix.target_platform == 'ubuntu' && matrix.arch == 'x64' + run: | + sudo apt-get install -y \ + libcurl4-gnutls-dev \ + libexpat1-dev \ + libssl-dev \ + zlib1g-dev + + - name: Install Linux arm64 build dependencies + if: matrix.target_platform == 'ubuntu' && matrix.arch == 'arm64' + run: | + sudo sed -i "s/^deb/deb [arch=amd64,i386]/g" /etc/apt/sources.list + release="$(lsb_release -s -c)" + echo "deb [arch=arm64,armhf] http://azure.ports.ubuntu.com/ ${release} main universe multiverse restricted" | sudo tee -a /etc/apt/sources.list + echo "deb [arch=arm64,armhf] http://azure.ports.ubuntu.com/ ${release}-updates main universe multiverse restricted" | sudo tee -a /etc/apt/sources.list + sudo dpkg --add-architecture arm64 + sudo apt-get update + sudo apt-get install -y \ + binutils-aarch64-linux-gnu \ + gcc-aarch64-linux-gnu \ + libcurl4-gnutls-dev:arm64 \ + libexpat1-dev:arm64 \ + libssl-dev:arm64 \ + zlib1g-dev:arm64 + + # Dugite Native currently pins MinGit 2.53. Keep its build script and + # dependency schema, but match the runtime to the Git series we compile. + - name: Select the matching MinGit runtime + if: matrix.target_platform == 'win32' + shell: bash + working-directory: dugite-native + env: + MINGIT_ARCH: ${{ matrix.mingit_arch }} + MINGIT_FILENAME: ${{ matrix.mingit_filename }} + MINGIT_SHA256: ${{ matrix.mingit_sha256 }} + MINGIT_URL: ${{ matrix.mingit_url }} + MINGIT_VERSION: v2.55.0 + SOURCE_UPSTREAM_TAG: ${{ needs.version.outputs.upstream_tag }} + run: | + set -euo pipefail + test "$SOURCE_UPSTREAM_TAG" = "$MINGIT_VERSION" + updated="$(mktemp)" + jq \ + --arg arch "$MINGIT_ARCH" \ + --arg checksum "$MINGIT_SHA256" \ + --arg filename "$MINGIT_FILENAME" \ + --arg url "$MINGIT_URL" \ + --arg version "$MINGIT_VERSION" \ + '.git.version = $version | + (.git.files[] | + select(.platform == "windows" and .arch == $arch)) |= + (.filename = $filename | + .url = $url | + .checksum = $checksum)' \ + dependencies.json >"$updated" + mv "$updated" dependencies.json + + - name: Set up Git for Windows SDK + if: matrix.target_platform == 'win32' + uses: git-for-windows/setup-git-for-windows-sdk@v2 + with: + architecture: ${{ matrix.sdk_arch }} + flavor: ${{ matrix.sdk_flavor }} + cache: false + + # Dugite cross-compiles several targets. Keep Git's optional Rust + # library disabled until its Makefile can direct Cargo at those targets. + - name: Build the Dugite Native distribution + shell: bash + working-directory: dugite-native + env: + NO_RUST: 1 + TARGET_PLATFORM: ${{ matrix.target_platform }} + TARGET_ARCH: ${{ matrix.arch }} + run: | + set -euo pipefail + if test "$TARGET_PLATFORM" = win32 + then + . /etc/profile + fi + script/build.sh + + # Dugite Native compiles its Git submodule on macOS and Linux. On + # Windows it starts from MinGit, so replace MinGit's Git programs with + # the build from this repository while retaining the portable runtime. + - name: Install this Git build into the Windows distribution + if: matrix.target_platform == 'win32' + shell: bash + working-directory: dugite-native/git + env: + MINGW_DIR: ${{ matrix.mingw_dir }} + run: | + set -euo pipefail + . /etc/profile + + make_args=( + "prefix=/$MINGW_DIR" + NO_PERL=YesPlease + NO_RUST=YesPlease + NO_TCLTK=YesPlease + NO_GETTEXT=YesPlease + NO_INSTALL_HARDLINKS=YesPlease + NO_CROSS_DIRECTORY_HARDLINKS=YesPlease + ) + jobs="$(nproc)" + make -j"$jobs" "${make_args[@]}" all + make "${make_args[@]}" DESTDIR=/tmp/build/git strip install + + - name: Verify distribution layout and provenance + shell: bash + env: + TARGET_PLATFORM: ${{ matrix.target_platform }} + MINGW_DIR: ${{ matrix.mingw_dir }} + GIT_BINARY: ${{ matrix.binary }} + FILE_PATTERN: ${{ matrix.file_pattern }} + HAS_GCM: ${{ matrix.has_gcm }} + run: | + set -euo pipefail + if test "$TARGET_PLATFORM" = win32 + then + . /etc/profile + test -f /tmp/build/git/cmd/git.exe + test -f "/tmp/build/git/$MINGW_DIR/libexec/git-core/git-lfs.exe" + test -d "/tmp/build/git/$MINGW_DIR/share/git-core/templates" + if test "$HAS_GCM" = true + then + test -f "/tmp/build/git/$MINGW_DIR/bin/git-credential-manager.exe" + fi + else + test -x /tmp/build/git/libexec/git-core/git-lfs + test -d /tmp/build/git/share/git-core/templates + if test "$HAS_GCM" = true + then + test -x /tmp/build/git/libexec/git-core/git-credential-manager + fi + fi + test -f /tmp/build/git/etc/gitconfig + + file "$GIT_BINARY" | tee /tmp/git-file-type + grep -E "$FILE_PATTERN" /tmp/git-file-type + strings "$GIT_BINARY" | grep -F "$GITHUB_SHA" + + - name: Smoke-test the native x64 distribution + if: matrix.arch == 'x64' + shell: bash + env: + TARGET_PLATFORM: ${{ matrix.target_platform }} + MINGW_DIR: ${{ matrix.mingw_dir }} + run: | + set -euo pipefail + smoke=/tmp/codex-git-smoke + mkdir -p "$smoke/home" + + if test "$TARGET_PLATFORM" = win32 + then + . /etc/profile + git_binary=/tmp/build/git/cmd/git.exe + git_env=( + "PATH=/tmp/build/git/cmd:/tmp/build/git/$MINGW_DIR/bin:/tmp/build/git/usr/bin:$PATH" + ) + else + git_binary=/tmp/build/git/bin/git + git_env=( + GIT_CONFIG_SYSTEM=/tmp/build/git/etc/gitconfig + GIT_EXEC_PATH=/tmp/build/git/libexec/git-core + GIT_TEMPLATE_DIR=/tmp/build/git/share/git-core/templates + ) + if test "$TARGET_PLATFORM" = ubuntu + then + git_env+=( + GIT_SSL_CAINFO=/tmp/build/git/ssl/cacert.pem + PREFIX=/tmp/build/git + ) + fi + fi + git_env+=("HOME=$smoke/home" GIT_TERMINAL_PROMPT=0) + + build_options="$(env "${git_env[@]}" "$git_binary" --version --build-options)" + printf '%s\n' "$build_options" + grep -F "built from commit: $GITHUB_SHA" <<<"$build_options" + env "${git_env[@]}" "$git_binary" lfs version + env "${git_env[@]}" "$git_binary" credential-manager --version + + env "${git_env[@]}" "$git_binary" init --quiet "$smoke/repo" + echo test >"$smoke/repo/file" + env "${git_env[@]}" "$git_binary" -C "$smoke/repo" add file + env "${git_env[@]}" "$git_binary" -C "$smoke/repo" \ + -c user.name='Codex Git CI' \ + -c user.email='codex-git-ci@openai.com' \ + commit --quiet -m initial + test -z "$(env "${git_env[@]}" "$git_binary" -C "$smoke/repo" status --porcelain)" + + - name: Package with Dugite Native + shell: bash + working-directory: dugite-native + env: + ASSET_PLATFORM: ${{ matrix.asset_platform }} + TARGET_PLATFORM: ${{ matrix.target_platform }} + TARGET_ARCH: ${{ matrix.arch }} + VERSION: ${{ needs.version.outputs.version }} + run: | + set -euo pipefail + if test "$TARGET_PLATFORM" = win32 + then + . /etc/profile + fi + script/package.sh + + for extension in tar.gz lzma + do + matches=( + output/dugite-native-"$VERSION"-*-"$ASSET_PLATFORM"-"$TARGET_ARCH.$extension" + ) + test "${#matches[@]}" -eq 1 + test -f "${matches[0]}" + + destination="output/git-$VERSION-$ASSET_PLATFORM-$TARGET_ARCH.$extension" + mv "${matches[0]}" "$destination" + mv "${matches[0]}.sha256" "$destination.sha256" + done + + for checksum in output/*.sha256 + do + archive="${checksum%.sha256}" + expected="$(tr -d '\r\n' <"$checksum")" + if command -v sha256sum >/dev/null 2>&1 + then + actual="$(sha256sum "$archive" | awk '{print $1}')" + else + actual="$(shasum -a 256 "$archive" | awk '{print $1}')" + fi + test "$actual" = "$expected" + done + + - name: Upload release assets + uses: actions/upload-artifact@v7 + with: + name: git-${{ matrix.asset_platform }}-${{ matrix.arch }} + path: dugite-native/output/git-* + if-no-files-found: error + retention-days: 7 + + release: + name: Publish GitHub prerelease + needs: + - version + - build + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: write + steps: + - name: Download release assets + uses: actions/download-artifact@v8 + with: + pattern: git-* + path: artifacts + merge-multiple: true + + - name: Publish immutable prerelease + env: + GH_TOKEN: ${{ github.token }} + SOURCE_DESCRIPTION: ${{ needs.version.outputs.describe }} + VERSION: ${{ needs.version.outputs.version }} + run: | + set -euo pipefail + assets=(artifacts/git-*) + + if gh release view "$VERSION" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1 + then + gh release upload "$VERSION" "${assets[@]}" \ + --repo "$GITHUB_REPOSITORY" \ + --clobber + else + gh release create "$VERSION" "${assets[@]}" \ + --repo "$GITHUB_REPOSITORY" \ + --target "$GITHUB_SHA" \ + --title "$VERSION" \ + --notes "OpenAI Git release artifacts for $SOURCE_DESCRIPTION, built from $GITHUB_SHA for Codex." \ + --prerelease + fi From f362c182b9f60846a05c515f4351e456281f1205 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 13 Jul 2026 19:37:49 -0700 Subject: [PATCH 02/28] ci: prepare Windows package before SDK setup --- .github/workflows/codex-release.yml | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/.github/workflows/codex-release.yml b/.github/workflows/codex-release.yml index 025d3a1998d450..389877044edf6c 100644 --- a/.github/workflows/codex-release.yml +++ b/.github/workflows/codex-release.yml @@ -235,14 +235,6 @@ jobs: dependencies.json >"$updated" mv "$updated" dependencies.json - - name: Set up Git for Windows SDK - if: matrix.target_platform == 'win32' - uses: git-for-windows/setup-git-for-windows-sdk@v2 - with: - architecture: ${{ matrix.sdk_arch }} - flavor: ${{ matrix.sdk_flavor }} - cache: false - # Dugite cross-compiles several targets. Keep Git's optional Rust # library disabled until its Makefile can direct Cargo at those targets. - name: Build the Dugite Native distribution @@ -254,12 +246,16 @@ jobs: TARGET_ARCH: ${{ matrix.arch }} run: | set -euo pipefail - if test "$TARGET_PLATFORM" = win32 - then - . /etc/profile - fi script/build.sh + - name: Set up Git for Windows SDK + if: matrix.target_platform == 'win32' + uses: git-for-windows/setup-git-for-windows-sdk@v2 + with: + architecture: ${{ matrix.sdk_arch }} + flavor: ${{ matrix.sdk_flavor }} + cache: false + # Dugite Native compiles its Git submodule on macOS and Linux. On # Windows it starts from MinGit, so replace MinGit's Git programs with # the build from this repository while retaining the portable runtime. From 3a68e4375a723d1ba0d5b6298d37670b51fdb09e Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 13 Jul 2026 19:39:37 -0700 Subject: [PATCH 03/28] ci: use Windows runner processor count --- .github/workflows/codex-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codex-release.yml b/.github/workflows/codex-release.yml index 389877044edf6c..636bc9471099dc 100644 --- a/.github/workflows/codex-release.yml +++ b/.github/workflows/codex-release.yml @@ -278,7 +278,7 @@ jobs: NO_INSTALL_HARDLINKS=YesPlease NO_CROSS_DIRECTORY_HARDLINKS=YesPlease ) - jobs="$(nproc)" + jobs="${NUMBER_OF_PROCESSORS:-2}" make -j"$jobs" "${make_args[@]}" all make "${make_args[@]}" DESTDIR=/tmp/build/git strip install From e7ad0f2c5a4d6ab9a3ef646e00386ae1820e2908 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 13 Jul 2026 19:46:35 -0700 Subject: [PATCH 04/28] ci: stabilize Windows SDK environment --- .github/workflows/codex-release.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/codex-release.yml b/.github/workflows/codex-release.yml index 636bc9471099dc..f3be531747d08d 100644 --- a/.github/workflows/codex-release.yml +++ b/.github/workflows/codex-release.yml @@ -117,7 +117,7 @@ jobs: file_pattern: PE32\+.*x86-64 has_gcm: true sdk_arch: x86_64 - sdk_flavor: minimal + sdk_flavor: full mingw_dir: mingw64 mingit_arch: amd64 mingit_filename: MinGit-2.55.0.2-64-bit.zip @@ -267,7 +267,6 @@ jobs: MINGW_DIR: ${{ matrix.mingw_dir }} run: | set -euo pipefail - . /etc/profile make_args=( "prefix=/$MINGW_DIR" @@ -294,7 +293,6 @@ jobs: set -euo pipefail if test "$TARGET_PLATFORM" = win32 then - . /etc/profile test -f /tmp/build/git/cmd/git.exe test -f "/tmp/build/git/$MINGW_DIR/libexec/git-core/git-lfs.exe" test -d "/tmp/build/git/$MINGW_DIR/share/git-core/templates" @@ -329,7 +327,6 @@ jobs: if test "$TARGET_PLATFORM" = win32 then - . /etc/profile git_binary=/tmp/build/git/cmd/git.exe git_env=( "PATH=/tmp/build/git/cmd:/tmp/build/git/$MINGW_DIR/bin:/tmp/build/git/usr/bin:$PATH" @@ -376,10 +373,6 @@ jobs: VERSION: ${{ needs.version.outputs.version }} run: | set -euo pipefail - if test "$TARGET_PLATFORM" = win32 - then - . /etc/profile - fi script/package.sh for extension in tar.gz lzma @@ -416,6 +409,15 @@ jobs: if-no-files-found: error retention-days: 7 + # The arm64 SDK puts its target Git first on PATH, but action cleanup + # runs on the x64 host and therefore needs the runner's native Git. + - name: Restore native Git for action cleanup + if: always() && matrix.target_platform == 'win32' + shell: pwsh + run: | + "C:\Program Files\Git\cmd" | + Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + release: name: Publish GitHub prerelease needs: From 466e67c3b9b0195f466574f47aea406f4c41d40e Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 13 Jul 2026 19:58:17 -0700 Subject: [PATCH 05/28] ci: build Windows arm64 on native runner --- .github/workflows/codex-release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codex-release.yml b/.github/workflows/codex-release.yml index f3be531747d08d..155d0d09cf544e 100644 --- a/.github/workflows/codex-release.yml +++ b/.github/workflows/codex-release.yml @@ -94,7 +94,7 @@ jobs: file_pattern: ELF 64-bit.*x86-64 has_gcm: true - name: Windows arm64 - os: windows-2025 + os: windows-11-arm target_platform: win32 asset_platform: windows arch: arm64 @@ -314,8 +314,8 @@ jobs: grep -E "$FILE_PATTERN" /tmp/git-file-type strings "$GIT_BINARY" | grep -F "$GITHUB_SHA" - - name: Smoke-test the native x64 distribution - if: matrix.arch == 'x64' + - name: Smoke-test the native distribution + if: matrix.arch == 'x64' || matrix.target_platform == 'win32' shell: bash env: TARGET_PLATFORM: ${{ matrix.target_platform }} From 4095be8ba018ae5a3727febb7ae4588cac8a8bbd Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 13 Jul 2026 20:06:08 -0700 Subject: [PATCH 06/28] ci: recognize Windows arm64 binaries --- .github/workflows/codex-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codex-release.yml b/.github/workflows/codex-release.yml index 155d0d09cf544e..076fe83c45dd4a 100644 --- a/.github/workflows/codex-release.yml +++ b/.github/workflows/codex-release.yml @@ -99,7 +99,7 @@ jobs: asset_platform: windows arch: arm64 binary: /tmp/build/git/clangarm64/bin/git.exe - file_pattern: PE32\+.*Aarch64 + file_pattern: PE32\+.*ARM64 has_gcm: true sdk_arch: aarch64 sdk_flavor: full From a31aedecab9cc703aa0ff9b3408dac6a1aefd876 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 14 Jul 2026 10:07:14 -0700 Subject: [PATCH 07/28] ci: slim Codex release artifacts --- .github/workflows/codex-release.yml | 44 ++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/.github/workflows/codex-release.yml b/.github/workflows/codex-release.yml index 076fe83c45dd4a..de2ac63f188a7d 100644 --- a/.github/workflows/codex-release.yml +++ b/.github/workflows/codex-release.yml @@ -68,7 +68,8 @@ jobs: arch: arm64 binary: /tmp/build/git/bin/git file_pattern: Mach-O 64-bit executable arm64 - has_gcm: true + has_gcm: false + max_tar_bytes: 33554432 - name: macOS x64 os: macos-15-intel target_platform: macOS @@ -76,7 +77,8 @@ jobs: arch: x64 binary: /tmp/build/git/bin/git file_pattern: Mach-O 64-bit executable x86_64 - has_gcm: true + has_gcm: false + max_tar_bytes: 33554432 - name: Linux arm64 os: ubuntu-22.04 target_platform: ubuntu @@ -85,6 +87,7 @@ jobs: binary: /tmp/build/git/bin/git file_pattern: ELF 64-bit.*ARM aarch64 has_gcm: false + max_tar_bytes: 41943040 - name: Linux x64 os: ubuntu-22.04 target_platform: ubuntu @@ -92,7 +95,8 @@ jobs: arch: x64 binary: /tmp/build/git/bin/git file_pattern: ELF 64-bit.*x86-64 - has_gcm: true + has_gcm: false + max_tar_bytes: 41943040 - name: Windows arm64 os: windows-11-arm target_platform: win32 @@ -101,6 +105,7 @@ jobs: binary: /tmp/build/git/clangarm64/bin/git.exe file_pattern: PE32\+.*ARM64 has_gcm: true + max_tar_bytes: 83886080 sdk_arch: aarch64 sdk_flavor: full mingw_dir: clangarm64 @@ -116,6 +121,7 @@ jobs: binary: /tmp/build/git/mingw64/bin/git.exe file_pattern: PE32\+.*x86-64 has_gcm: true + max_tar_bytes: 83886080 sdk_arch: x86_64 sdk_flavor: full mingw_dir: mingw64 @@ -235,6 +241,20 @@ jobs: dependencies.json >"$updated" mv "$updated" dependencies.json + # Codex does not configure or invoke GCM on macOS or Linux. The + # self-contained .NET payload accounts for most of those bundles, while + # Windows MinGit configures credential.helper=manager and must retain it. + - name: Omit unused GCM from POSIX bundles + if: matrix.target_platform != 'win32' + shell: bash + working-directory: dugite-native + run: | + set -euo pipefail + updated="$(mktemp)" + jq '."git-credential-manager".files = []' \ + dependencies.json >"$updated" + mv "$updated" dependencies.json + # Dugite cross-compiles several targets. Keep Git's optional Rust # library disabled until its Makefile can direct Cargo at those targets. - name: Build the Dugite Native distribution @@ -259,6 +279,8 @@ jobs: # Dugite Native compiles its Git submodule on macOS and Linux. On # Windows it starts from MinGit, so replace MinGit's Git programs with # the build from this repository while retaining the portable runtime. + # MinGit omits dashed builtin aliases; installing them as copies would + # add hundreds of redundant MiB to the archive. - name: Install this Git build into the Windows distribution if: matrix.target_platform == 'win32' shell: bash @@ -276,6 +298,7 @@ jobs: NO_GETTEXT=YesPlease NO_INSTALL_HARDLINKS=YesPlease NO_CROSS_DIRECTORY_HARDLINKS=YesPlease + SKIP_DASHED_BUILT_INS=YesPlease ) jobs="${NUMBER_OF_PROCESSORS:-2}" make -j"$jobs" "${make_args[@]}" all @@ -296,6 +319,7 @@ jobs: test -f /tmp/build/git/cmd/git.exe test -f "/tmp/build/git/$MINGW_DIR/libexec/git-core/git-lfs.exe" test -d "/tmp/build/git/$MINGW_DIR/share/git-core/templates" + test ! -e "/tmp/build/git/$MINGW_DIR/libexec/git-core/git-add.exe" if test "$HAS_GCM" = true then test -f "/tmp/build/git/$MINGW_DIR/bin/git-credential-manager.exe" @@ -306,6 +330,8 @@ jobs: if test "$HAS_GCM" = true then test -x /tmp/build/git/libexec/git-core/git-credential-manager + else + test ! -e /tmp/build/git/libexec/git-core/git-credential-manager fi fi test -f /tmp/build/git/etc/gitconfig @@ -320,6 +346,7 @@ jobs: env: TARGET_PLATFORM: ${{ matrix.target_platform }} MINGW_DIR: ${{ matrix.mingw_dir }} + HAS_GCM: ${{ matrix.has_gcm }} run: | set -euo pipefail smoke=/tmp/codex-git-smoke @@ -352,7 +379,10 @@ jobs: printf '%s\n' "$build_options" grep -F "built from commit: $GITHUB_SHA" <<<"$build_options" env "${git_env[@]}" "$git_binary" lfs version - env "${git_env[@]}" "$git_binary" credential-manager --version + if test "$HAS_GCM" = true + then + env "${git_env[@]}" "$git_binary" credential-manager --version + fi env "${git_env[@]}" "$git_binary" init --quiet "$smoke/repo" echo test >"$smoke/repo/file" @@ -371,6 +401,7 @@ jobs: TARGET_PLATFORM: ${{ matrix.target_platform }} TARGET_ARCH: ${{ matrix.arch }} VERSION: ${{ needs.version.outputs.version }} + MAX_TAR_BYTES: ${{ matrix.max_tar_bytes }} run: | set -euo pipefail script/package.sh @@ -401,6 +432,11 @@ jobs: test "$actual" = "$expected" done + tarball="output/git-$VERSION-$ASSET_PLATFORM-$TARGET_ARCH.tar.gz" + tar_bytes="$(wc -c <"$tarball")" + printf '%s bytes: %s\n' "$tar_bytes" "$tarball" + test "$tar_bytes" -le "$MAX_TAR_BYTES" + - name: Upload release assets uses: actions/upload-artifact@v7 with: From f90d09ac68aa37700d246c9f7acaa599adcc6eea Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 14 Jul 2026 10:21:40 -0700 Subject: [PATCH 08/28] ci: loosen artifact size guardrails --- .github/workflows/codex-release.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/codex-release.yml b/.github/workflows/codex-release.yml index de2ac63f188a7d..98ffccc6e23e19 100644 --- a/.github/workflows/codex-release.yml +++ b/.github/workflows/codex-release.yml @@ -69,7 +69,7 @@ jobs: binary: /tmp/build/git/bin/git file_pattern: Mach-O 64-bit executable arm64 has_gcm: false - max_tar_bytes: 33554432 + max_tar_bytes: 67108864 - name: macOS x64 os: macos-15-intel target_platform: macOS @@ -78,7 +78,7 @@ jobs: binary: /tmp/build/git/bin/git file_pattern: Mach-O 64-bit executable x86_64 has_gcm: false - max_tar_bytes: 33554432 + max_tar_bytes: 67108864 - name: Linux arm64 os: ubuntu-22.04 target_platform: ubuntu @@ -87,7 +87,7 @@ jobs: binary: /tmp/build/git/bin/git file_pattern: ELF 64-bit.*ARM aarch64 has_gcm: false - max_tar_bytes: 41943040 + max_tar_bytes: 67108864 - name: Linux x64 os: ubuntu-22.04 target_platform: ubuntu @@ -96,7 +96,7 @@ jobs: binary: /tmp/build/git/bin/git file_pattern: ELF 64-bit.*x86-64 has_gcm: false - max_tar_bytes: 41943040 + max_tar_bytes: 67108864 - name: Windows arm64 os: windows-11-arm target_platform: win32 @@ -105,7 +105,7 @@ jobs: binary: /tmp/build/git/clangarm64/bin/git.exe file_pattern: PE32\+.*ARM64 has_gcm: true - max_tar_bytes: 83886080 + max_tar_bytes: 134217728 sdk_arch: aarch64 sdk_flavor: full mingw_dir: clangarm64 @@ -121,7 +121,7 @@ jobs: binary: /tmp/build/git/mingw64/bin/git.exe file_pattern: PE32\+.*x86-64 has_gcm: true - max_tar_bytes: 83886080 + max_tar_bytes: 134217728 sdk_arch: x86_64 sdk_flavor: full mingw_dir: mingw64 From a41101ff8a274585173edffb28fa2e101aff8681 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 4 Aug 2026 13:05:32 -0700 Subject: [PATCH 09/28] ci: release only controller-published Codex commits Topic pull requests merge into codex before the controller rebuilds its recorded output. Treating every codex push as a release would run the full build matrix and publish artifacts for an intermediate tree. Compare each pushed commit with the output recorded in meta:codex.config before starting version detection. Skip the build and release jobs unless the controller published that exact commit; a later pending merge must not suppress its release. --- .github/workflows/codex-release.yml | 34 +++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/.github/workflows/codex-release.yml b/.github/workflows/codex-release.yml index 98ffccc6e23e19..fe3cdbc8a3abc6 100644 --- a/.github/workflows/codex-release.yml +++ b/.github/workflows/codex-release.yml @@ -13,8 +13,42 @@ concurrency: cancel-in-progress: false jobs: + publication: + name: Verify controller publication + runs-on: ubuntu-24.04 + outputs: + published: ${{ steps.verify.outputs.published }} + steps: + - name: Check the published controller output + id: verify + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + + meta=$(gh api \ + "repos/$GITHUB_REPOSITORY/git/ref/heads/meta" \ + --jq '.object.sha') + recorded=$(gh api \ + "repos/$GITHUB_REPOSITORY/contents/codex.config?ref=$meta" \ + -H 'Accept: application/vnd.github.raw+json' | + git config --no-includes --file /dev/stdin \ + --get codex.output-tip) + + if test "$GITHUB_SHA" = "$recorded" + then + printf 'published=true\n' >>"$GITHUB_OUTPUT" + printf 'Releasing controller-published commit %s.\n' "$GITHUB_SHA" + else + printf 'published=false\n' >>"$GITHUB_OUTPUT" + printf 'Skipping non-controller publication %s.\n' "$GITHUB_SHA" + fi + version: name: Determine version + needs: publication + if: needs.publication.outputs.published == 'true' runs-on: ubuntu-24.04 outputs: describe: ${{ steps.version.outputs.describe }} From a9de123b43efb58c53c99c71eb7e34f29e075071 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 4 Aug 2026 09:44:11 -0700 Subject: [PATCH 10/28] maintenance: account for promisor pack geometry Commit 9bc151850c (builtin/maintenance: introduce "geometric-repack" task, 2025-10-24) added a new maintenance task to perform either a geometric repack, or an all-into-one repack if the geometric repack would itself produce a single pack. Some time later, commit dcc9c7ef47 (builtin/repack: handle promisor packs with geometric repacking, 2026-01-05) taught the geometric repacking machinery to separate promisor packs from ordinary ones, but did not update the maintenance task accordingly. As a consequence, the geometric-repack maintenance task only considers the non-promisor pack progression. It falls back to all-into-one whenever a geometric repack would roll up all non-promisor packs into a single pack, even if the promisor progression would keep a large pack and roll up only smaller ones. Check both progressions before choosing the repack mode. If either leaves a pack above its split, geometric repack still avoids rewriting that pack, whereas the all-into-one fallback would rewrite it. Use the fallback only when neither progression leaves a pack behind. That preserves the reason for the fallback: let the all-into-one repack handle unreachable objects when it is not rewriting more packs than the geometric repack. Signed-off-by: Taylor Blau --- builtin/gc.c | 3 ++- t/t7900-maintenance.sh | 45 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/builtin/gc.c b/builtin/gc.c index 49c8474fade8ed..ed75c12c43eab9 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -1593,7 +1593,8 @@ static int maintenance_task_geometric_repack(struct maintenance_run_opts *opts, child.odb_to_close = the_repository->objects; strvec_pushl(&child.args, "repack", "-d", "-l", NULL); - if (geometry.split < geometry.pack_nr) + if (geometry.split < geometry.pack_nr || + geometry.promisor_split < geometry.promisor_pack_nr) strvec_pushf(&child.args, "--geometric=%d", geometry.split_factor); else diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index a8d691719da062..ba5b359e7786e4 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -659,6 +659,51 @@ test_expect_success 'geometric repacking task' ' ) ' +objdir=.git/objects +packdir=$objdir/pack + +pack_promisor () { + p="$(echo "$@" | git pack-objects --revs $packdir/pack)" && + touch "$packdir/pack-$p.promisor" && + echo "$p" +} + +test_expect_success 'geometric repacking task handles promisor packs' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + git config set maintenance.auto false && + git remote add promisor garbage && + git config set remote.promisor.promisor true && + + for n in $(test_seq 6) + do + test_commit $n || return 1 + done && + + A="$(pack_promisor 1)" && + B="$(pack_promisor 1..2)" && + C="$(pack_promisor 2..6)" && + git prune-packed && + + ls $packdir/pack-*.promisor | sort >promisors.before && + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \ + git maintenance run --quiet --task=geometric-repack && + ls $packdir/pack-*.promisor | sort >promisors.after && + + test_subcommand git repack -d -l --geometric=2 \ + --quiet --write-midx expect && + comm -23 promisors.before promisors.after >actual && + + test_cmp expect actual + ) +' + test_geometric_repack_needed () { NEEDED="$1" GEOMETRIC_CONFIG="$2" && From dc2fffc37cead551f8036c9ecab5e52a4cbee37b Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 4 Aug 2026 09:44:13 -0700 Subject: [PATCH 11/28] maintenance: trigger --auto for promisor rollups Commit 9bc151850c (builtin/maintenance: introduce "geometric-repack" task, 2025-10-24) added an auto condition for the geometric-repack task. It runs the task when ordinary packs need to be combined or when the number of loose objects crosses the configured threshold. Later on in commit dcc9c7ef47 (builtin/repack: handle promisor packs with geometric repacking, 2026-01-05), the geometric repack machinery started handling promisor packs separately, but did not correspondingly update the auto condition. As a result, a repository can have promisor packs ready to combine while its non-promisor packs and loose object count require no work. In that case, `--auto` skips the task even though a geometric repack would combine at least two promisor packs. Check `geometry.promisor_split` alongside `geometry.split`. There is some fallout in t5331: the new condition makes a filtered clone eligible for auto-maintenance before the test inspects its promisor packs. Disable auto-maintenance in that fixture so it continues to test `--stdin-packs`, not the maintenance task. Signed-off-by: Taylor Blau --- builtin/gc.c | 2 +- t/t5331-pack-objects-stdin.sh | 3 ++- t/t7900-maintenance.sh | 23 +++++++++++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/builtin/gc.c b/builtin/gc.c index ed75c12c43eab9..e9572940dcb22d 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -1649,7 +1649,7 @@ static int geometric_repack_auto_condition(struct gc_config *cfg UNUSED) * When we'd merge at least two packs with one another we always * perform the repack. */ - if (geometry.split) { + if (geometry.split || geometry.promisor_split) { ret = 1; goto out; } diff --git a/t/t5331-pack-objects-stdin.sh b/t/t5331-pack-objects-stdin.sh index c74b5861af322f..2a983e28ac43e7 100755 --- a/t/t5331-pack-objects-stdin.sh +++ b/t/t5331-pack-objects-stdin.sh @@ -368,7 +368,8 @@ test_expect_success '--stdin-packs does not perform backfill fetch' ' git -C remote config set --local uploadpack.allowfilter 1 && git -C remote config set --local uploadpack.allowanysha1inwant 1 && - git clone --filter=tree:0 "file://$(pwd)/remote" client && + git -c maintenance.auto=false clone --filter=tree:0 \ + "file://$(pwd)/remote" client && ( cd client && ls .git/objects/pack/*.promisor | sed "s|.*/||; s/\.promisor$/.pack/" >packs && diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index ba5b359e7786e4..fb5f2d89028738 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -759,6 +759,29 @@ test_expect_success 'geometric repacking with --auto' ' ) ' +test_expect_success 'geometric repacking with --auto handles promisor packs' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + git config set maintenance.auto false && + git remote add promisor garbage && + git config set remote.promisor.promisor true && + + for n in $(test_seq 6) + do + test_commit $n || return 1 + done && + + pack_promisor 1 >/dev/null && + pack_promisor 1..2 >/dev/null && + pack_promisor 2..6 >/dev/null && + git prune-packed && + + test_geometric_repack_needed true auto=9000 + ) +' + test_expect_success 'geometric repacking honors configured split factor' ' test_when_finished "rm -rf repo" && git init repo && From ba107e0ae8c7142238bb612e530d51d42f0280d3 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Wed, 5 Aug 2026 11:18:01 -0700 Subject: [PATCH 12/28] ci: release controller-published Codex unstable commits The release workflow currently listens only to codex and checks every push against codex.output-tip. The controller records and atomically publishes codex-unstable.output-tip separately, so preview output pushes need an independent gate. Listen to both generated branches, skip deletion events, and select the recorded output from the exact pushed ref. Unknown refs and missing state fail closed. The existing version, build, and release chain still runs only when the selected output matches GITHUB_SHA. --- .github/workflows/codex-release.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codex-release.yml b/.github/workflows/codex-release.yml index fe3cdbc8a3abc6..623c6e09aeb281 100644 --- a/.github/workflows/codex-release.yml +++ b/.github/workflows/codex-release.yml @@ -4,6 +4,7 @@ on: push: branches: - codex + - codex-unstable permissions: contents: read @@ -16,6 +17,7 @@ jobs: publication: name: Verify controller publication runs-on: ubuntu-24.04 + if: github.event.deleted == false outputs: published: ${{ steps.verify.outputs.published }} steps: @@ -26,6 +28,18 @@ jobs: GH_TOKEN: ${{ github.token }} run: | set -euo pipefail + case "$GITHUB_REF" in + refs/heads/codex) + output_key=codex.output-tip + ;; + refs/heads/codex-unstable) + output_key=codex-unstable.output-tip + ;; + *) + printf 'unexpected release ref: %s\n' "$GITHUB_REF" >&2 + exit 1 + ;; + esac meta=$(gh api \ "repos/$GITHUB_REPOSITORY/git/ref/heads/meta" \ @@ -34,7 +48,7 @@ jobs: "repos/$GITHUB_REPOSITORY/contents/codex.config?ref=$meta" \ -H 'Accept: application/vnd.github.raw+json' | git config --no-includes --file /dev/stdin \ - --get codex.output-tip) + --get "$output_key") if test "$GITHUB_SHA" = "$recorded" then From 40589b5333835ecd5e1b6187cbcec45d8382303e Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Wed, 5 Aug 2026 13:38:57 -0700 Subject: [PATCH 13/28] ci: stamp Codex release source refs Codex and codex-unstable releases currently share the same prerelease shape. Their target SHA differs, but the GitHub release object does not record which output ref triggered it. Consumers must therefore join release data with Actions runs to distinguish the lanes. Carry github.ref into the publish step and put source_ref and source_sha at the start of the release notes. This leaves tag and asset names unchanged while giving release-API consumers a machine-readable lane marker. --- .github/workflows/codex-release.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codex-release.yml b/.github/workflows/codex-release.yml index 623c6e09aeb281..0fe82d4455cf82 100644 --- a/.github/workflows/codex-release.yml +++ b/.github/workflows/codex-release.yml @@ -523,10 +523,18 @@ jobs: env: GH_TOKEN: ${{ github.token }} SOURCE_DESCRIPTION: ${{ needs.version.outputs.describe }} + SOURCE_REF: ${{ github.ref }} VERSION: ${{ needs.version.outputs.version }} run: | set -euo pipefail assets=(artifacts/git-*) + release_notes=$( + printf '%s\n' \ + "source_ref=$SOURCE_REF" \ + "source_sha=$GITHUB_SHA" \ + "" \ + "OpenAI Git release artifacts for $SOURCE_DESCRIPTION, built from $GITHUB_SHA for Codex." + ) if gh release view "$VERSION" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1 then @@ -538,6 +546,6 @@ jobs: --repo "$GITHUB_REPOSITORY" \ --target "$GITHUB_SHA" \ --title "$VERSION" \ - --notes "OpenAI Git release artifacts for $SOURCE_DESCRIPTION, built from $GITHUB_SHA for Codex." \ + --notes "$release_notes" \ --prerelease fi From 5cd3b3771c0aac541add6fc3a7a97bcfe1749cbb Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Thu, 6 Aug 2026 17:49:18 -0700 Subject: [PATCH 14/28] ci: scan approved Codex topics through pinned plans The default-branch trampoline must not start from pull_request_review: that event loads workflow code from the PR merge ref. Replace it with a trusted default-branch scanner. On a five-minute schedule, or an explicit scan dispatch, it pins meta, checks it out read-only, preflights approved candidates with propose-plan --no-push, and calls the App-backed producer only for a valid exact-head review. Stale or policy-invalid approvals are skipped instead of blocking later topics. Plan admission stays pull_request_target with trusted meta code; remove and reorder remain explicit dispatch-only policy operations. --- .github/workflows/codex.yml | 243 ++++++++++++++++++++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 .github/workflows/codex.yml diff --git a/.github/workflows/codex.yml b/.github/workflows/codex.yml new file mode 100644 index 00000000000000..650cfd55baef14 --- /dev/null +++ b/.github/workflows/codex.yml @@ -0,0 +1,243 @@ +name: Refresh codex + +on: + schedule: + - cron: '*/5 * * * *' + workflow_dispatch: + inputs: + operation: + description: Refresh, scan, remove, or reorder a pinned topic + type: choice + options: + - refresh + - scan + - remove + - reorder + default: refresh + lane: + description: codex or codex-unstable for a plan operation + required: false + type: string + topic: + description: Exact topic branch for a plan operation + required: false + type: string + after: + description: Existing topic or root for reorder + required: false + type: string + plan_branch: + description: Optional codex-plan/* branch name + required: false + type: string + pull_request_target: + branches: + - meta + types: + - opened + - reopened + - synchronize + - ready_for_review + +permissions: + actions: read + contents: read + pull-requests: read + +jobs: + refresh: + if: >- + github.event_name == 'workflow_dispatch' && + github.ref == 'refs/heads/codex' && + inputs.operation == 'refresh' + uses: openai/git/.github/workflows/codex.yml@meta + topic_plan_scan: + name: Find one approved topic plan + if: >- + github.event_name == 'schedule' || + (github.event_name == 'workflow_dispatch' && + github.ref == 'refs/heads/codex' && + inputs.operation == 'scan') + runs-on: ubuntu-24.04 + permissions: + contents: read + pull-requests: read + concurrency: + group: codex-topic-plan-scan + cancel-in-progress: false + outputs: + lane: ${{ steps.reviewed.outputs.lane }} + topic: ${{ steps.reviewed.outputs.topic }} + source_tip: ${{ steps.reviewed.outputs.source_tip }} + review_pr: ${{ steps.reviewed.outputs.review_pr }} + env: + GH_TOKEN: ${{ github.token }} + steps: + - name: Pin trusted meta + id: meta + run: | + set -euo pipefail + test "$GITHUB_REPOSITORY" = openai/git + test "$GITHUB_REF" = refs/heads/codex + sha=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/meta" \ + --jq .object.sha) + case "$sha" in + ''|*[!0-9a-f]*) exit 1 ;; + esac + test "${#sha}" = 40 + printf 'sha=%s\n' "$sha" >>"$GITHUB_OUTPUT" + + - name: Check out trusted meta + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: ${{ github.repository }} + ref: ${{ steps.meta.outputs.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Find one exact approved topic PR + id: reviewed + env: + META_SHA: ${{ steps.meta.outputs.sha }} + run: | + set -euo pipefail + + die () { + printf '%s\n' "$*" >&2 + exit 1 + } + + test "$GITHUB_REPOSITORY" = openai/git + test "$GITHUB_REF" = refs/heads/codex || + die "topic scan must run from the trusted default branch" + test "$(git rev-parse HEAD)" = "$META_SHA" || + die "trusted checkout does not match pinned meta" + gh auth setup-git + mkdir -p "$RUNNER_TEMP/codex-plan-scan" + for lane in codex codex-unstable + do + case "$lane" in + codex) plan=codex.plan ;; + codex-unstable) plan=codex-unstable.plan ;; + esac + test -f "$plan" || + die "trusted meta has no $plan" + gh pr list --repo "$GITHUB_REPOSITORY" --state open \ + --base "$lane" --limit 1000 \ + --json number,isDraft,headRefName,headRefOid,headRepository,reviewDecision | + jq -r --arg lane "$lane" ' + .[] | + select(.isDraft | not) | + select(.reviewDecision == "APPROVED") | + select(.headRepository.nameWithOwner == "openai/git") | + [$lane, .headRefName, .headRefOid, + (.number | tostring)] | @tsv + ' + done | sort -k4,4n >"$RUNNER_TEMP/codex-plan-scan/candidates" + + while IFS=$'\t' read -r lane topic source_tip review_pr + do + test -n "$review_pr" || continue + case "$review_pr" in + *[!0-9]*) die "approved topic PR has invalid number '$review_pr'" ;; + esac + case "$source_tip" in + *[!0-9a-f]*|'') die "approved topic PR has invalid source SHA" ;; + esac + test "${#source_tip}" = 40 || + die "approved topic PR has invalid source SHA" + git check-ref-format "refs/heads/$topic" >/dev/null 2>&1 || + die "approved topic PR has invalid branch '$topic'" + case "$topic" in + ??/codex/*) ;; + *) continue ;; + esac + suffix=${topic#??/codex/} + case "$suffix" in + ''|*/*|*-wip|*-stale) continue ;; + esac + case "$lane" in + codex) + case "$topic" in + *-unstable) continue ;; + esac + plan=codex.plan + ;; + codex-unstable) + case "$topic" in + *-unstable) ;; + *) continue ;; + esac + plan=codex-unstable.plan + ;; + *) die "approved topic PR has invalid lane '$lane'" ;; + esac + pinned=$(git config --no-includes \ + --file "$plan" \ + --get "branch.$topic.source-tip" || :) + test "$pinned" = "$source_tip" && continue + short=$(printf '%.12s' "$source_tip") + slug=${topic##*/} + plan_branch=codex-plan/$lane-$slug-$short + pending=$(gh pr list --repo "$GITHUB_REPOSITORY" \ + --state open --base meta --head "$plan_branch" \ + --json number --jq '.[0].number // empty') || + die "could not inspect pending Codex plan PR" + test -n "$pending" && continue + if ! sh .github/workflows/codex-branch.sh propose-plan \ + --remote origin --lane "$lane" --topic "$topic" \ + --action auto --source-tip "$source_tip" \ + --review-pr "$review_pr" --expected-meta "$META_SHA" \ + --no-push >/dev/null + then + printf 'skipping approved topic PR #%s: preflight failed\n' \ + "$review_pr" >&2 + continue + fi + { + printf 'lane=%s\n' "$lane" + printf 'topic=%s\n' "$topic" + printf 'source_tip=%s\n' "$source_tip" + printf 'review_pr=%s\n' "$review_pr" + } >>"$GITHUB_OUTPUT" + exit 0 + done <"$RUNNER_TEMP/codex-plan-scan/candidates" + topic_plan_propose: + name: Propose reviewed topic plan + needs: topic_plan_scan + if: needs.topic_plan_scan.outputs.review_pr != '' + permissions: + contents: read + pull-requests: read + uses: openai/git/.github/workflows/codex-plan-propose.yml@meta + with: + lane: ${{ needs.topic_plan_scan.outputs.lane }} + topic: ${{ needs.topic_plan_scan.outputs.topic }} + action: auto + source_tip: ${{ needs.topic_plan_scan.outputs.source_tip }} + review_pr: ${{ needs.topic_plan_scan.outputs.review_pr }} + policy_plan_propose: + name: Propose explicit plan policy + if: >- + github.event_name == 'workflow_dispatch' && + github.ref == 'refs/heads/codex' && + (inputs.operation == 'remove' || inputs.operation == 'reorder') + permissions: + contents: read + pull-requests: read + uses: openai/git/.github/workflows/codex-plan-propose.yml@meta + with: + lane: ${{ inputs.lane }} + topic: ${{ inputs.topic }} + action: ${{ inputs.operation }} + after: ${{ inputs.after }} + plan_branch: ${{ inputs.plan_branch }} + plan_admission: + name: Codex plan admission + if: >- + github.event_name == 'pull_request_target' && + github.event.pull_request.base.ref == 'meta' + permissions: + contents: read + pull-requests: write + uses: openai/git/.github/workflows/codex-plan-admission.yml@meta From 012e49a6be30d0a832475e0974132fb0a31fcf46 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 17:06:32 -0500 Subject: [PATCH 15/28] ci: run Codex Linux arm64 releases natively The release workflow cross-compiles Linux arm64 on an x64 runner and skips the smoke test for arm64 POSIX bundles. That prevents the workflow from executing the Linux artifact it just produced. Run Linux arm64 on GitHub's arm64 runner and install native development packages rather than configuring a foreign dpkg architecture. All matrix entries can then run the existing distribution smoke test. --- .github/workflows/codex-release.yml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/codex-release.yml b/.github/workflows/codex-release.yml index 0fe82d4455cf82..6e705e7f1ab23f 100644 --- a/.github/workflows/codex-release.yml +++ b/.github/workflows/codex-release.yml @@ -127,8 +127,9 @@ jobs: file_pattern: Mach-O 64-bit executable x86_64 has_gcm: false max_tar_bytes: 67108864 + # Keep arm64 builds native so release smoke tests can execute them. - name: Linux arm64 - os: ubuntu-22.04 + os: ubuntu-22.04-arm target_platform: ubuntu asset_platform: ubuntu arch: arm64 @@ -243,19 +244,13 @@ jobs: - name: Install Linux arm64 build dependencies if: matrix.target_platform == 'ubuntu' && matrix.arch == 'arm64' run: | - sudo sed -i "s/^deb/deb [arch=amd64,i386]/g" /etc/apt/sources.list - release="$(lsb_release -s -c)" - echo "deb [arch=arm64,armhf] http://azure.ports.ubuntu.com/ ${release} main universe multiverse restricted" | sudo tee -a /etc/apt/sources.list - echo "deb [arch=arm64,armhf] http://azure.ports.ubuntu.com/ ${release}-updates main universe multiverse restricted" | sudo tee -a /etc/apt/sources.list - sudo dpkg --add-architecture arm64 - sudo apt-get update sudo apt-get install -y \ binutils-aarch64-linux-gnu \ gcc-aarch64-linux-gnu \ - libcurl4-gnutls-dev:arm64 \ - libexpat1-dev:arm64 \ - libssl-dev:arm64 \ - zlib1g-dev:arm64 + libcurl4-gnutls-dev \ + libexpat1-dev \ + libssl-dev \ + zlib1g-dev # Dugite Native currently pins MinGit 2.53. Keep its build script and # dependency schema, but match the runtime to the Git series we compile. @@ -389,7 +384,6 @@ jobs: strings "$GIT_BINARY" | grep -F "$GITHUB_SHA" - name: Smoke-test the native distribution - if: matrix.arch == 'x64' || matrix.target_platform == 'win32' shell: bash env: TARGET_PLATFORM: ${{ matrix.target_platform }} From 3972a3e4bcb9349bdb097f16977a10f1b240b341 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 17:10:52 -0500 Subject: [PATCH 16/28] ci: build Codex Git releases with LTO Codex consumes Git release artifacts built with the Makefile's default -O2 flags. The release job compiles each artifact without link-time optimization. Add a release-only config.mak.openai and copy it into Git's ignored config.mak slot before building. Use thin LTO for Clang targets and automatic LTO for GCC targets, then check GIT-CFLAGS records the selected flag in every distribution job. Keeping the setting in config.mak.openai avoids carrying release-only policy in the upstream Makefile. --- .github/workflows/codex-release.yml | 15 +++++++++++++++ config.mak.openai | 10 ++++++++++ 2 files changed, 25 insertions(+) create mode 100644 config.mak.openai diff --git a/.github/workflows/codex-release.yml b/.github/workflows/codex-release.yml index 6e705e7f1ab23f..e40630af802e7d 100644 --- a/.github/workflows/codex-release.yml +++ b/.github/workflows/codex-release.yml @@ -117,6 +117,7 @@ jobs: binary: /tmp/build/git/bin/git file_pattern: Mach-O 64-bit executable arm64 has_gcm: false + lto: thin max_tar_bytes: 67108864 - name: macOS x64 os: macos-15-intel @@ -126,6 +127,7 @@ jobs: binary: /tmp/build/git/bin/git file_pattern: Mach-O 64-bit executable x86_64 has_gcm: false + lto: thin max_tar_bytes: 67108864 # Keep arm64 builds native so release smoke tests can execute them. - name: Linux arm64 @@ -136,6 +138,7 @@ jobs: binary: /tmp/build/git/bin/git file_pattern: ELF 64-bit.*ARM aarch64 has_gcm: false + lto: auto max_tar_bytes: 67108864 - name: Linux x64 os: ubuntu-22.04 @@ -145,6 +148,7 @@ jobs: binary: /tmp/build/git/bin/git file_pattern: ELF 64-bit.*x86-64 has_gcm: false + lto: auto max_tar_bytes: 67108864 - name: Windows arm64 os: windows-11-arm @@ -154,6 +158,7 @@ jobs: binary: /tmp/build/git/clangarm64/bin/git.exe file_pattern: PE32\+.*ARM64 has_gcm: true + lto: thin max_tar_bytes: 134217728 sdk_arch: aarch64 sdk_flavor: full @@ -170,6 +175,7 @@ jobs: binary: /tmp/build/git/mingw64/bin/git.exe file_pattern: PE32\+.*x86-64 has_gcm: true + lto: auto max_tar_bytes: 134217728 sdk_arch: x86_64 sdk_flavor: full @@ -210,6 +216,11 @@ jobs: -c 'user.email=41898282+github-actions[bot]@users.noreply.github.com' \ tag -a "$VERSION" -m "$VERSION" + - name: Install OpenAI build configuration + shell: bash + working-directory: dugite-native/git + run: cp config.mak.openai config.mak + # Match Dugite Native's compatibility choice for its macOS x64 build. - name: Select Xcode 16.4 if: matrix.target_platform == 'macOS' && matrix.arch == 'x64' @@ -305,6 +316,7 @@ jobs: working-directory: dugite-native env: NO_RUST: 1 + OPENAI_LTO: ${{ matrix.lto }} TARGET_PLATFORM: ${{ matrix.target_platform }} TARGET_ARCH: ${{ matrix.arch }} run: | @@ -330,6 +342,7 @@ jobs: working-directory: dugite-native/git env: MINGW_DIR: ${{ matrix.mingw_dir }} + OPENAI_LTO: ${{ matrix.lto }} run: | set -euo pipefail @@ -355,6 +368,7 @@ jobs: GIT_BINARY: ${{ matrix.binary }} FILE_PATTERN: ${{ matrix.file_pattern }} HAS_GCM: ${{ matrix.has_gcm }} + LTO: ${{ matrix.lto }} run: | set -euo pipefail if test "$TARGET_PLATFORM" = win32 @@ -382,6 +396,7 @@ jobs: file "$GIT_BINARY" | tee /tmp/git-file-type grep -E "$FILE_PATTERN" /tmp/git-file-type strings "$GIT_BINARY" | grep -F "$GITHUB_SHA" + grep -F -- "-flto=$LTO" dugite-native/git/GIT-CFLAGS - name: Smoke-test the native distribution shell: bash diff --git a/config.mak.openai b/config.mak.openai new file mode 100644 index 00000000000000..d75c9aee7d4903 --- /dev/null +++ b/config.mak.openai @@ -0,0 +1,10 @@ +# OpenAI release build settings. +# +# The Codex release workflow copies this file to config.mak before +# building. Keep release-only optimizations here instead of in the +# upstream Makefile. + +ifdef OPENAI_LTO +CFLAGS_APPEND += -flto=$(OPENAI_LTO) +LDFLAGS_APPEND += -flto=$(OPENAI_LTO) +endif From 6e6ec362f3c5e03f01bf1fd0fbdea5589cf4caec Mon Sep 17 00:00:00 2001 From: Daniel Reynaud Date: Mon, 20 Jul 2026 17:23:09 -0700 Subject: [PATCH 17/28] ci: build Codex Git from dugite fork --- .github/workflows/codex-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codex-release.yml b/.github/workflows/codex-release.yml index 0fe82d4455cf82..3652cf1d4b65f3 100644 --- a/.github/workflows/codex-release.yml +++ b/.github/workflows/codex-release.yml @@ -184,8 +184,8 @@ jobs: - name: Check out Dugite Native uses: actions/checkout@v6 with: - repository: desktop/dugite-native - ref: f97e50add48cdcff053a69d95aa343a4a4a258c2 + repository: dreynaud-oai/dugite-native + ref: b6f4473557acb85433fdf9deffe0854a34fd9cc5 path: dugite-native fetch-depth: 0 persist-credentials: false From f0ee69fd806fa486eb3d00e799cfe564a499f8ab Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 17:12:16 -0500 Subject: [PATCH 18/28] ci: train Codex Git releases with PGO LTO can optimize across translation units, but the release job has no execution profile for the status, diff, clone, fetch, and repack paths Codex invokes frequently. Git's built-in profile target runs the 1,048-script test suite serially. That is too expensive for every release target and weights test-harness paths more heavily than the local workload. Extend config.mak.openai with GCC and LLVM profile modes. Gate GIT-CFLAGS on an instrumented build, run a short offline trainer, merge LLVM raw profiles when needed, and rebuild with profile-use flags. Each matrix entry runs on its target architecture, so it can execute the instrumented binary. Check that final GIT-CFLAGS includes a profile-use flag and increase the timeout for the second compilation pass. The focused trainer took about 30 seconds locally; the full macOS build/install validation completed with thin LTO and LLVM profile-use enabled. --- .github/workflows/codex-pgo-training.sh | 60 +++++++++++++++++++++++ .github/workflows/codex-release.yml | 32 +++++++++++-- config.mak.openai | 64 +++++++++++++++++++++++++ 3 files changed, 151 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/codex-pgo-training.sh diff --git a/.github/workflows/codex-pgo-training.sh b/.github/workflows/codex-pgo-training.sh new file mode 100644 index 00000000000000..070d8b54cae486 --- /dev/null +++ b/.github/workflows/codex-pgo-training.sh @@ -0,0 +1,60 @@ +#!/bin/sh + +# Keep this workload short and biased toward the local Git operations Codex +# invokes frequently. The full Git test suite is too slow for each release +# target and would weight test-harness paths more heavily than status, diff, +# clone, fetch, and repository maintenance. + +set -eu + +git_bin="$PWD/bin-wrappers/git" +training_dir=$(mktemp -d "${TMPDIR:-/tmp}/codex-git-pgo.XXXXXX") +repo="$training_dir/repo" +clone="$training_dir/clone" + +cleanup () { + rm -rf "$training_dir" +} +trap cleanup EXIT HUP INT TERM + +mkdir -p "$training_dir/home" +export HOME="$training_dir/home" +export GIT_CONFIG_NOSYSTEM=1 +export GIT_TERMINAL_PROMPT=0 + +"$git_bin" clone --quiet --no-local "$PWD" "$repo" +"$git_bin" -C "$repo" config user.name "Codex Git PGO" +"$git_bin" -C "$repo" config user.email "codex-git-pgo@openai.com" + +i=0 +while test "$i" -lt 256 +do + dir="$repo/training/$((i % 16))" + mkdir -p "$dir" + printf '%s\n' "$i" >"$dir/file-$i" + i=$((i + 1)) +done + +"$git_bin" -C "$repo" status --porcelain=v2 --branch >/dev/null +"$git_bin" -C "$repo" status --porcelain=v2 --branch --untracked-files=all >/dev/null +"$git_bin" -C "$repo" ls-files --others --exclude-standard >/dev/null +"$git_bin" -C "$repo" add training +"$git_bin" -C "$repo" diff --cached --stat >/dev/null +"$git_bin" -C "$repo" commit --quiet -m "add training files" + +printf 'changed\n' >>"$repo/training/0/file-0" +rm "$repo/training/1/file-1" +mkdir -p "$repo/untracked" +printf 'new\n' >"$repo/untracked/file" + +"$git_bin" -C "$repo" status --porcelain=v2 --branch >/dev/null +"$git_bin" -C "$repo" diff --stat >/dev/null +"$git_bin" -C "$repo" diff --name-status >/dev/null +"$git_bin" -C "$repo" ls-files --stage >/dev/null +"$git_bin" -C "$repo" log --oneline --decorate -20 >/dev/null +"$git_bin" -C "$repo" rev-list --objects --all >/dev/null +"$git_bin" -C "$repo" for-each-ref --format='%(refname) %(objectname)' >/dev/null +"$git_bin" -C "$repo" repack -ad +"$git_bin" clone --quiet --no-local "$repo" "$clone" +"$git_bin" -C "$clone" status --porcelain=v2 --branch >/dev/null +"$git_bin" -C "$clone" fetch --quiet "$repo" diff --git a/.github/workflows/codex-release.yml b/.github/workflows/codex-release.yml index e40630af802e7d..aa58855d439351 100644 --- a/.github/workflows/codex-release.yml +++ b/.github/workflows/codex-release.yml @@ -104,7 +104,7 @@ jobs: name: ${{ matrix.name }} needs: version runs-on: ${{ matrix.os }} - timeout-minutes: 30 + timeout-minutes: 60 strategy: fail-fast: false matrix: @@ -118,6 +118,8 @@ jobs: file_pattern: Mach-O 64-bit executable arm64 has_gcm: false lto: thin + profile_format: LLVM + llvm_profdata: xcrun llvm-profdata max_tar_bytes: 67108864 - name: macOS x64 os: macos-15-intel @@ -128,6 +130,8 @@ jobs: file_pattern: Mach-O 64-bit executable x86_64 has_gcm: false lto: thin + profile_format: LLVM + llvm_profdata: xcrun llvm-profdata max_tar_bytes: 67108864 # Keep arm64 builds native so release smoke tests can execute them. - name: Linux arm64 @@ -139,6 +143,7 @@ jobs: file_pattern: ELF 64-bit.*ARM aarch64 has_gcm: false lto: auto + profile_format: GCC max_tar_bytes: 67108864 - name: Linux x64 os: ubuntu-22.04 @@ -149,6 +154,7 @@ jobs: file_pattern: ELF 64-bit.*x86-64 has_gcm: false lto: auto + profile_format: GCC max_tar_bytes: 67108864 - name: Windows arm64 os: windows-11-arm @@ -159,6 +165,8 @@ jobs: file_pattern: PE32\+.*ARM64 has_gcm: true lto: thin + profile_format: LLVM + llvm_profdata: llvm-profdata max_tar_bytes: 134217728 sdk_arch: aarch64 sdk_flavor: full @@ -176,6 +184,7 @@ jobs: file_pattern: PE32\+.*x86-64 has_gcm: true lto: auto + profile_format: GCC max_tar_bytes: 134217728 sdk_arch: x86_64 sdk_flavor: full @@ -309,14 +318,18 @@ jobs: dependencies.json >"$updated" mv "$updated" dependencies.json - # Dugite cross-compiles several targets. Keep Git's optional Rust - # library disabled until its Makefile can direct Cargo at those targets. + # Build an instrumented Git, run a representative local workload, then + # rebuild with its profile. Keep Git's optional Rust library disabled + # until its Makefile can direct Cargo at these targets. - name: Build the Dugite Native distribution shell: bash working-directory: dugite-native env: NO_RUST: 1 + OPENAI_LLVM_PROFDATA: ${{ matrix.llvm_profdata }} OPENAI_LTO: ${{ matrix.lto }} + OPENAI_PROFILE: BUILD + OPENAI_PROFILE_FORMAT: ${{ matrix.profile_format }} TARGET_PLATFORM: ${{ matrix.target_platform }} TARGET_ARCH: ${{ matrix.arch }} run: | @@ -342,7 +355,9 @@ jobs: working-directory: dugite-native/git env: MINGW_DIR: ${{ matrix.mingw_dir }} + OPENAI_LLVM_PROFDATA: ${{ matrix.llvm_profdata }} OPENAI_LTO: ${{ matrix.lto }} + OPENAI_PROFILE_FORMAT: ${{ matrix.profile_format }} run: | set -euo pipefail @@ -357,8 +372,8 @@ jobs: SKIP_DASHED_BUILT_INS=YesPlease ) jobs="${NUMBER_OF_PROCESSORS:-2}" - make -j"$jobs" "${make_args[@]}" all - make "${make_args[@]}" DESTDIR=/tmp/build/git strip install + make -j"$jobs" "${make_args[@]}" OPENAI_PROFILE=BUILD all + make "${make_args[@]}" OPENAI_PROFILE=USE DESTDIR=/tmp/build/git strip install - name: Verify distribution layout and provenance shell: bash @@ -369,6 +384,7 @@ jobs: FILE_PATTERN: ${{ matrix.file_pattern }} HAS_GCM: ${{ matrix.has_gcm }} LTO: ${{ matrix.lto }} + PROFILE_FORMAT: ${{ matrix.profile_format }} run: | set -euo pipefail if test "$TARGET_PLATFORM" = win32 @@ -397,6 +413,12 @@ jobs: grep -E "$FILE_PATTERN" /tmp/git-file-type strings "$GIT_BINARY" | grep -F "$GITHUB_SHA" grep -F -- "-flto=$LTO" dugite-native/git/GIT-CFLAGS + if test "$PROFILE_FORMAT" = LLVM + then + grep -F -- "-fprofile-instr-use=" dugite-native/git/GIT-CFLAGS + else + grep -F -- "-fprofile-use=" dugite-native/git/GIT-CFLAGS + fi - name: Smoke-test the native distribution shell: bash diff --git a/config.mak.openai b/config.mak.openai index d75c9aee7d4903..0ae133c8bb2ecf 100644 --- a/config.mak.openai +++ b/config.mak.openai @@ -4,7 +4,71 @@ # building. Keep release-only optimizations here instead of in the # upstream Makefile. +OPENAI_PROFILE_DIR := $(CURDIR) +OPENAI_PROFILE_RAW := $(OPENAI_PROFILE_DIR)/default_%m_%p.profraw +OPENAI_PROFILE_DATA := $(OPENAI_PROFILE_DIR)/default.profdata +OPENAI_PROFILE_TRAINING ?= .github/workflows/codex-pgo-training.sh +OPENAI_LLVM_PROFDATA ?= llvm-profdata + ifdef OPENAI_LTO CFLAGS_APPEND += -flto=$(OPENAI_LTO) LDFLAGS_APPEND += -flto=$(OPENAI_LTO) endif + +ifeq ("$(OPENAI_PROFILE_FORMAT)","LLVM") +ifeq ("$(OPENAI_PROFILE)","GEN") + BASIC_CFLAGS += -fprofile-instr-generate=$(OPENAI_PROFILE_RAW) + BASIC_CFLAGS += -DNO_NORETURN=1 + export CCACHE_DISABLE = t + V = 1 +else +ifneq ("$(OPENAI_PROFILE)","") + BASIC_CFLAGS += -fprofile-instr-use=$(OPENAI_PROFILE_DATA) + BASIC_CFLAGS += -Wno-profile-instr-unprofiled -DNO_NORETURN=1 + export CCACHE_DISABLE = t + V = 1 +endif +endif +else +ifeq ("$(OPENAI_PROFILE)","GEN") + BASIC_CFLAGS += -fprofile-generate=$(OPENAI_PROFILE_DIR) + BASIC_CFLAGS += -DNO_NORETURN=1 + EXTLIBS += -lgcov + export CCACHE_DISABLE = t + V = 1 +else +ifneq ("$(OPENAI_PROFILE)","") + BASIC_CFLAGS += -fprofile-use=$(OPENAI_PROFILE_DIR) + BASIC_CFLAGS += -fprofile-correction -DNO_NORETURN=1 + export CCACHE_DISABLE = t + V = 1 +endif +endif +endif + +# Every C object already waits for Git's forced GIT-CFLAGS target. Gate that +# target so "make strip install" finishes PGO before its normal prerequisites +# can start compiling with profile-use flags. +ifeq "$(OPENAI_PROFILE)" "BUILD" +GIT-CFLAGS: openai-profile +endif + +openai-profile: profile-clean openai-profile-clean + $(MAKE) OPENAI_PROFILE=GEN all + $(SHELL_PATH) $(OPENAI_PROFILE_TRAINING) + $(MAKE) OPENAI_PROFILE= openai-profile-merge + $(MAKE) OPENAI_PROFILE=USE all + +openai-profile-merge: +ifeq ("$(OPENAI_PROFILE_FORMAT)","LLVM") + $(OPENAI_LLVM_PROFDATA) merge \ + -output=$(OPENAI_PROFILE_DATA) \ + $(OPENAI_PROFILE_DIR)/*.profraw +endif + +clean: openai-profile-clean + +openai-profile-clean: + $(RM) $(OPENAI_PROFILE_DIR)/*.profraw $(OPENAI_PROFILE_DATA) + +.PHONY: openai-profile openai-profile-merge openai-profile-clean From da56896de86f524b287d5ed46063075b04967455 Mon Sep 17 00:00:00 2001 From: Ted Nyman Date: Sun, 26 Jul 2026 17:28:38 -0700 Subject: [PATCH 19/28] http-fetch: correct --index-pack-arg documentation The --packfile mode accepts one --index-pack-arg= option per argument passed to index-pack, but its documentation and option dependency errors still refer to the plural --index-pack-args form. Correct the spelling and describe the repeatable per-argument form. Signed-off-by: Ted Nyman Signed-off-by: Junio C Hamano --- Documentation/git-http-fetch.adoc | 9 +++++---- http-fetch.c | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Documentation/git-http-fetch.adoc b/Documentation/git-http-fetch.adoc index 2200f073c47120..12036e65e93651 100644 --- a/Documentation/git-http-fetch.adoc +++ b/Documentation/git-http-fetch.adoc @@ -50,11 +50,12 @@ commit-id:: URL and uses index-pack to generate corresponding .idx and .keep files. The hash is used to determine the name of the temporary file and is arbitrary. The output of index-pack is printed to stdout. Requires - --index-pack-args. + one or more --index-pack-arg options. ---index-pack-args=:: - For internal use only. The command to run on the contents of the - downloaded pack. Arguments are URL-encoded separated by spaces. +--index-pack-arg=:: + For internal use only. The first instance specifies the command run on + the contents of the downloaded pack. Subsequent instances specify its + arguments. --recover:: Verify that everything reachable from target is fetched. Used after diff --git a/http-fetch.c b/http-fetch.c index f9b6ecb0616fe0..601a77c3c10204 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -155,7 +155,7 @@ int cmd_main(int argc, const char **argv) if (packfile) { if (!index_pack_args.nr) - die(_("the option '%s' requires '%s'"), "--packfile", "--index-pack-args"); + die(_("the option '%s' requires '%s'"), "--packfile", "--index-pack-arg"); fetch_single_packfile(&packfile_hash, argv[arg], index_pack_args.v); @@ -164,7 +164,7 @@ int cmd_main(int argc, const char **argv) } if (index_pack_args.nr) - die(_("the option '%s' requires '%s'"), "--index-pack-args", "--packfile"); + die(_("the option '%s' requires '%s'"), "--index-pack-arg", "--packfile"); if (commits_on_stdin) { commits = walker_targets_stdin(&commit_id, &write_ref); From aae9a24f383734873ec6d02b3849bb1f23d508b3 Mon Sep 17 00:00:00 2001 From: Ted Nyman Date: Sun, 26 Jul 2026 17:28:39 -0700 Subject: [PATCH 20/28] http: avoid closing index-pack input twice finish_http_pack_request() passes its staging-file descriptor to index-pack through child_process.in. start_command() takes ownership of a supplied descriptor and closes it, even when starting the child fails. Do not close the descriptor again after run_command() returns. Signed-off-by: Ted Nyman Signed-off-by: Junio C Hamano --- http.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/http.c b/http.c index caccf2108e4479..89a1ccc6d290c6 100644 --- a/http.c +++ b/http.c @@ -2704,13 +2704,8 @@ int finish_http_pack_request(struct http_pack_request *preq) else ip.no_stdout = 1; - if (run_command(&ip)) { + if (run_command(&ip)) ret = -1; - goto cleanup; - } - -cleanup: - close(tmpfile_fd); unlink(preq->tmpfile.buf); return ret; } From a6d49d88d86659cd9eebd09d6ba81b1d941ca957 Mon Sep 17 00:00:00 2001 From: Ted Nyman Date: Sun, 26 Jul 2026 17:28:40 -0700 Subject: [PATCH 21/28] http: accept HTTP 416 for complete partial packs A resumed pack request may already have all bytes of the remote pack. A server can respond to the resulting Range request with HTTP 416 instead of returning an empty response. Accept that response in each pack-download caller and let index-pack validate the completed staging file. This can happen without concurrent downloads when a previous attempt completed the transfer but failed before indexing it. Add a regression test that seeds a complete partial pack and checks that http-fetch indexes it after the server returns HTTP 416. Signed-off-by: Ted Nyman Signed-off-by: Junio C Hamano --- http-fetch.c | 3 ++- http-push.c | 3 ++- http-walker.c | 3 ++- t/t5550-http-fetch-dumb.sh | 19 +++++++++++++++++++ 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/http-fetch.c b/http-fetch.c index 601a77c3c10204..05f68f306a5821 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -70,7 +70,8 @@ static void fetch_single_packfile(struct object_id *packfile_hash, if (start_active_slot(preq->slot)) { run_active_slot(preq->slot); - if (results.curl_result != CURLE_OK) { + if (results.curl_result != CURLE_OK && + results.http_code != 416) { struct url_info url; char *nurl = url_normalize(preq->url, &url); if (!nurl || !git_env_bool("GIT_TRACE_REDACT", 1)) { diff --git a/http-push.c b/http-push.c index 94a1fac9ab0fcd..786a2e9c0d0546 100644 --- a/http-push.c +++ b/http-push.c @@ -595,7 +595,8 @@ static void finish_request(struct transfer_request *request) } else if (request->state == RUN_FETCH_PACKED) { int fail = 1; - if (request->curl_result != CURLE_OK) { + if (request->curl_result != CURLE_OK && + request->http_code != 416) { fprintf(stderr, "Unable to get pack file %s\n%s", request->url, curl_errorstr); } else { diff --git a/http-walker.c b/http-walker.c index b58a3b2a92be38..abafca84d65441 100644 --- a/http-walker.c +++ b/http-walker.c @@ -451,7 +451,8 @@ static int http_fetch_pack(struct walker *walker, struct alt_base *repo, if (start_active_slot(preq->slot)) { run_active_slot(preq->slot); - if (results.curl_result != CURLE_OK) { + if (results.curl_result != CURLE_OK && + results.http_code != 416) { error("Unable to get pack file %s\n%s", preq->url, curl_errorstr); goto abort; diff --git a/t/t5550-http-fetch-dumb.sh b/t/t5550-http-fetch-dumb.sh index f00eeae48f8554..698bbb316036de 100755 --- a/t/t5550-http-fetch-dumb.sh +++ b/t/t5550-http-fetch-dumb.sh @@ -293,6 +293,25 @@ test_expect_success 'http-fetch --packfile' ' git -C packfileclient cat-file -e "$HASH" ' +test_expect_success 'http-fetch --packfile accepts an already complete partial' ' + git init packfileclient-complete && + p=$(cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git && + ls objects/pack/pack-*.pack) && + packhash=$(basename "$p" .pack) && + packhash=${packhash#pack-} && + tmpfile="packfileclient-complete/.git/objects/pack/pack-$packhash.pack.temp" && + cp "$HTTPD_DOCUMENT_ROOT_PATH/repo_pack.git/$p" "$tmpfile" && + chmod u+w "$tmpfile" && + GIT_TRACE_CURL="$TRASH_DIRECTORY/complete.trace" \ + git -C packfileclient-complete http-fetch --packfile="$packhash" \ + --index-pack-arg=index-pack \ + --index-pack-arg=--stdin --index-pack-arg=--keep \ + "$HTTPD_URL/dumb/repo_pack.git/$p" >out && + test_grep "416 Requested Range Not Satisfiable" complete.trace && + test_path_is_missing "$tmpfile" && + git -C packfileclient-complete cat-file -e "$HASH" +' + test_expect_success 'fetch notices corrupt pack' ' cp -R "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad1.git && (cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad1.git && From 7dfd028727cf593d1418992b8d23fa43262def6d Mon Sep 17 00:00:00 2001 From: Ted Nyman Date: Sun, 26 Jul 2026 17:28:41 -0700 Subject: [PATCH 22/28] http: avoid concurrent appends to partial packs Pack requests stage downloads in a predictable partial-pack file so an interrupted transfer can be resumed. Both packfile URI and ordinary dumb HTTP requests use this staging path. Opening it in append mode forces each write to the current end of the file, so concurrent responses can append duplicate data and corrupt the pack. Open the partial pack read-write without O_APPEND and seek once to its current end. Each downloader then retains the offset matching the Range it requested. Because the staging key must uniquely identify immutable pack contents, overlapping responses write the same bytes at the same offsets instead of extending the file with duplicate data. Duplicate the staging descriptor for index-pack instead of reopening the path after closing the stream. Another downloader may unlink the staging path before indexing begins, but index-pack can still read the retained descriptor. Exercise resumed transfers and overlapping 200 and 206 responses, and clarify the staging-key documentation. Signed-off-by: Ted Nyman Signed-off-by: Junio C Hamano --- Documentation/git-http-fetch.adoc | 5 +- http.c | 34 ++++--- t/t5550-http-fetch-dumb.sh | 164 ++++++++++++++++++++++++++++++ 3 files changed, 187 insertions(+), 16 deletions(-) diff --git a/Documentation/git-http-fetch.adoc b/Documentation/git-http-fetch.adoc index 12036e65e93651..45e0d3d07c73cf 100644 --- a/Documentation/git-http-fetch.adoc +++ b/Documentation/git-http-fetch.adoc @@ -48,8 +48,9 @@ commit-id:: line (which is not expected in this case), 'git http-fetch' fetches the packfile directly at the given URL and uses index-pack to generate corresponding .idx and .keep files. - The hash is used to determine the name of the temporary file and is - arbitrary. The output of index-pack is printed to stdout. Requires + The hash is used to determine the name of the temporary file. It need + not be the pack hash, but it must uniquely identify the pack contents + for resumption. The output of index-pack is printed to stdout. Requires one or more --index-pack-arg options. --index-pack-arg=:: diff --git a/http.c b/http.c index 89a1ccc6d290c6..ad07ef354902e5 100644 --- a/http.c +++ b/http.c @@ -2688,10 +2688,13 @@ int finish_http_pack_request(struct http_pack_request *preq) int tmpfile_fd; int ret = 0; + /* Another downloader may unlink the staging path while we index it. */ + tmpfile_fd = xdup(fileno(preq->packfile)); fclose(preq->packfile); preq->packfile = NULL; - - tmpfile_fd = xopen(preq->tmpfile.buf, O_RDONLY); + if (lseek(tmpfile_fd, 0, SEEK_SET) < 0) + die_errno("unable to seek local file %s for pack", + preq->tmpfile.buf); ip.git_cmd = 1; ip.in = tmpfile_fd; @@ -2733,22 +2736,30 @@ struct http_pack_request *new_http_pack_request( struct http_pack_request *new_direct_http_pack_request( const unsigned char *packed_git_hash, char *url) { - off_t prev_posn = 0; + off_t prev_posn; struct http_pack_request *preq; + int fd; CALLOC_ARRAY(preq, 1); strbuf_init(&preq->tmpfile, 0); - preq->url = url; odb_pack_name(the_repository, &preq->tmpfile, packed_git_hash, "pack"); strbuf_addstr(&preq->tmpfile, ".temp"); - preq->packfile = fopen(preq->tmpfile.buf, "a"); - if (!preq->packfile) { - error("Unable to open local file %s for pack", - preq->tmpfile.buf); + fd = open(preq->tmpfile.buf, O_RDWR | O_CREAT, 0666); + if (fd < 0) { + error_errno("unable to open local file %s for pack", + preq->tmpfile.buf); + goto abort; + } + prev_posn = lseek(fd, 0, SEEK_END); + if (prev_posn < 0) { + error_errno("unable to seek local file %s for pack", + preq->tmpfile.buf); + close(fd); goto abort; } + preq->packfile = xfdopen(fd, "w"); preq->slot = get_active_slot(); preq->headers = object_request_headers(); @@ -2757,12 +2768,7 @@ struct http_pack_request *new_direct_http_pack_request( curl_easy_setopt(preq->slot->curl, CURLOPT_URL, preq->url); curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER, preq->headers); - /* - * If there is data present from a previous transfer attempt, - * resume where it left off - */ - prev_posn = ftello(preq->packfile); - if (prev_posn>0) { + if (prev_posn > 0) { if (http_is_verbose) fprintf(stderr, "Resuming fetch of pack %s at byte %"PRIuMAX"\n", diff --git a/t/t5550-http-fetch-dumb.sh b/t/t5550-http-fetch-dumb.sh index 698bbb316036de..86b9d87ef5fd4c 100755 --- a/t/t5550-http-fetch-dumb.sh +++ b/t/t5550-http-fetch-dumb.sh @@ -312,6 +312,170 @@ test_expect_success 'http-fetch --packfile accepts an already complete partial' git -C packfileclient-complete cat-file -e "$HASH" ' +test_expect_success 'http-fetch --packfile resumes a partial download' ' + git init packfileclient-resume && + p=$(cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git && + ls objects/pack/pack-*.pack) && + tmpfile="packfileclient-resume/.git/objects/pack/pack-$ARBITRARY.pack.temp" && + test_copy_bytes 64 <"$HTTPD_DOCUMENT_ROOT_PATH/repo_pack.git/$p" >"$tmpfile" && + GIT_TRACE_CURL="$TRASH_DIRECTORY/resume.trace" \ + git -C packfileclient-resume http-fetch --packfile="$ARBITRARY" \ + --index-pack-arg=index-pack --index-pack-arg=--stdin \ + --index-pack-arg=--keep \ + "$HTTPD_URL/dumb/repo_pack.git/$p" >out && + test_grep "Range: bytes=64-" resume.trace && + test_path_is_missing "$tmpfile" && + git -C packfileclient-resume cat-file -e "$HASH" +' + +test_expect_success PERL,PIPE 'concurrent http-fetch --packfile cannot corrupt an overlapping download' ' + git init packfileclient-overlap && + blob=$(test-tool genrandom pack-overlap 2m | + git -C "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git \ + hash-object -w --stdin) && + packhash=$(printf "%s\n" "$blob" | + git -C "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git \ + pack-objects "$TRASH_DIRECTORY/overlap-pack") && + pack="$TRASH_DIRECTORY/overlap-pack-$packhash.pack" && + tmpfile="packfileclient-overlap/.git/objects/pack/pack-$packhash.pack.temp" && + mkfifo server-ready first-ready && + exec 7<>server-ready && + exec 8<>first-ready && + write_script slow-pack-server "$PERL_PATH" <<-\EOF && + use strict; + use warnings; + use IO::Socket::INET; + + my ($packfile, $server_ready, $first_ready) = @ARGV; + my $completed = 0; + END { + if (!$completed) { + signal_ready($server_ready, "failed"); + signal_ready($first_ready, "failed"); + } + } + + $SIG{ALRM} = sub { die "timed out serving concurrent pack requests\n" }; + alarm 60; + + open(my $in, "<:raw", $packfile) or die "open $packfile: $!"; + my $pack = do { local $/; <$in> }; + close($in) or die "close $packfile: $!"; + my $server = IO::Socket::INET->new(LocalAddr => "127.0.0.1", + LocalPort => 0, Proto => "tcp", Listen => 2, ReuseAddr => 1) + or die "listen: $!"; + + sub signal_ready { + my ($file, $value) = @_; + open(my $out, ">", $file) or die "open $file: $!"; + print $out "$value\n" or die "write $file: $!"; + close($out) or die "close $file: $!"; + } + + sub write_all { + my ($out, $data) = @_; + my $offset = 0; + while ($offset < length($data)) { + my $written = syswrite($out, $data, + length($data) - $offset, $offset); + defined($written) && $written or die "write response: $!"; + $offset += $written; + } + } + + sub start_response { + my $out = $server->accept() or die "accept: $!"; + <$out> or die "read request: $!"; + my $start = 0; + while (<$out>) { + last if /^\r?\n$/; + $start = $1 if /^Range: bytes=(\d+)-/i; + } + $start < length($pack) or die "invalid range $start"; + my $length = length($pack) - $start; + my $middle = int($length / 2); + my $status = $start ? "206 Partial Content" : "200 OK"; + my $headers = "HTTP/1.1 $status\r\n" . + "Content-Length: $length\r\n" . + ($start ? "Content-Range: bytes $start-" . + (length($pack) - 1) . "/" . length($pack) . "\r\n" : "") . + "Connection: close\r\n\r\n"; + write_all($out, $headers); + write_all($out, substr($pack, $start, $middle)); + return ($out, $start + $middle); + } + + signal_ready($server_ready, $server->sockport()); + my ($first, $first_pos) = start_response(); + signal_ready($first_ready, "ready"); + my ($second, $second_pos) = start_response(); + write_all($first, substr($pack, $first_pos)); + write_all($second, substr($pack, $second_pos)); + close($first) or die "close first response: $!"; + close($second) or die "close second response: $!"; + $completed = 1; + alarm 0; + EOF + { + "$TRASH_DIRECTORY/slow-pack-server" "$pack" \ + "$TRASH_DIRECTORY/server-ready" \ + "$TRASH_DIRECTORY/first-ready" >server.log 2>&1 & + server_pid=$! + } && + test_when_finished " + kill $server_pid 2>/dev/null || : + wait $server_pid 2>/dev/null || : + exec 7>&- + exec 8>&- + rm -f server-ready first-ready slow-pack-server + " && + read port <&7 && + url="http://127.0.0.1:$port/pack" && + { + ( + if ! GIT_TRACE_CURL="$TRASH_DIRECTORY/overlap-first.trace" \ + GIT_TRACE_CURL_NO_DATA=1 \ + git -C packfileclient-overlap http-fetch --packfile="$packhash" \ + --index-pack-arg=index-pack \ + --index-pack-arg=--stdin --index-pack-arg=--keep \ + "$url" >first.out + then + echo failed >"$TRASH_DIRECTORY/first-ready" && + exit 1 + fi + ) & + first_pid=$! + } && + test_when_finished " + kill $first_pid 2>/dev/null || : + wait $first_pid 2>/dev/null || : + " && + read ready <&8 && + test "$ready" = ready && + test_path_is_file "$tmpfile" && + { + GIT_TRACE_CURL="$TRASH_DIRECTORY/overlap-second.trace" \ + GIT_TRACE_CURL_NO_DATA=1 \ + git -C packfileclient-overlap http-fetch --packfile="$packhash" \ + --index-pack-arg=index-pack \ + --index-pack-arg=--stdin --index-pack-arg=--keep \ + "$url" >second.out & + second_pid=$! + } && + test_when_finished " + kill $second_pid 2>/dev/null || : + wait $second_pid 2>/dev/null || : + " && + wait "$second_pid" && + wait "$first_pid" && + wait "$server_pid" && + printf "keep\t%s\npack\t%s\n" "$packhash" "$packhash" | sort >expect && + sort first.out second.out >actual && + test_cmp expect actual && + test_path_is_missing "$tmpfile" && + git -C packfileclient-overlap cat-file -e "$blob" +' + test_expect_success 'fetch notices corrupt pack' ' cp -R "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad1.git && (cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad1.git && From 6f5e4ba49a969a4defefcd9787800319bf3c3761 Mon Sep 17 00:00:00 2001 From: Ted Nyman Date: Sun, 26 Jul 2026 17:28:42 -0700 Subject: [PATCH 23/28] http: permit unlinking partial packs on Windows On Windows, an open file must permit FILE_SHARE_DELETE before another process can unlink it. MinGW's non-append O_RDWR open enables that sharing mode only for an existing file; adding O_CREAT falls back to _wopen(), which cannot set it. First try opening the partial pack without O_CREAT. If it does not exist, create it exclusively, close that descriptor, and retry through the existing-file path. A racing creator retries after EEXIST. This ensures that every retained descriptor permits another downloader to unlink the staging path. Add an unlink-while-indexing test that does not require FIFOs and can therefore run on MinGW. Signed-off-by: Ted Nyman Signed-off-by: Junio C Hamano --- http.c | 17 ++++++++++++++++- t/t5550-http-fetch-dumb.sh | 21 +++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/http.c b/http.c index ad07ef354902e5..a0d399b2745ae6 100644 --- a/http.c +++ b/http.c @@ -2746,7 +2746,22 @@ struct http_pack_request *new_direct_http_pack_request( odb_pack_name(the_repository, &preq->tmpfile, packed_git_hash, "pack"); strbuf_addstr(&preq->tmpfile, ".temp"); - fd = open(preq->tmpfile.buf, O_RDWR | O_CREAT, 0666); + /* + * MinGW's non-append O_RDWR open grants FILE_SHARE_DELETE only for an + * existing file; reopen a newly created file so others may unlink it. + */ + for (;;) { + fd = open(preq->tmpfile.buf, O_RDWR); + if (fd >= 0 || errno != ENOENT) + break; + fd = open(preq->tmpfile.buf, O_RDWR | O_CREAT | O_EXCL, 0666); + if (fd >= 0) { + close(fd); + continue; + } + if (errno != EEXIST) + break; + } if (fd < 0) { error_errno("unable to open local file %s for pack", preq->tmpfile.buf); diff --git a/t/t5550-http-fetch-dumb.sh b/t/t5550-http-fetch-dumb.sh index 86b9d87ef5fd4c..b5758f1c9ca1c2 100755 --- a/t/t5550-http-fetch-dumb.sh +++ b/t/t5550-http-fetch-dumb.sh @@ -328,6 +328,27 @@ test_expect_success 'http-fetch --packfile resumes a partial download' ' git -C packfileclient-resume cat-file -e "$HASH" ' +test_expect_success 'http-fetch --packfile permits unlink while indexing' ' + git init packfileclient-unlink && + p=$(cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git && + ls objects/pack/pack-*.pack) && + tmpfile="packfileclient-unlink/.git/objects/pack/pack-$ARBITRARY.pack.temp" && + write_script git-unlink-index-pack <<-\EOF && + test -f "$GIT_TEST_PACK_TEMP" || exit 1 + rm "$GIT_TEST_PACK_TEMP" || exit 1 + exec git index-pack "$@" + EOF + test_when_finished "rm -f git-unlink-index-pack" && + PATH="$TRASH_DIRECTORY:$PATH" \ + GIT_TEST_PACK_TEMP="$TRASH_DIRECTORY/$tmpfile" \ + git -C packfileclient-unlink http-fetch --packfile="$ARBITRARY" \ + --index-pack-arg=unlink-index-pack \ + --index-pack-arg=--stdin --index-pack-arg=--keep \ + "$HTTPD_URL/dumb/repo_pack.git/$p" >out && + test_path_is_missing "$tmpfile" && + git -C packfileclient-unlink cat-file -e "$HASH" +' + test_expect_success PERL,PIPE 'concurrent http-fetch --packfile cannot corrupt an overlapping download' ' git init packfileclient-overlap && blob=$(test-tool genrandom pack-overlap 2m | From 4cc8b3223b233aa3b795b1265e6e7cf3d63c7170 Mon Sep 17 00:00:00 2001 From: Ted Nyman Date: Sun, 26 Jul 2026 17:28:43 -0700 Subject: [PATCH 24/28] fetch-pack: accept "pack" output for packfile URIs When index-pack finds an existing keep file it reports pack rather than keep. Accept either result from http-fetch, and only register a keep lockfile when this fetch created it. Read the pack/keep prefix and hash without consuming any following fsck output, validate the reported pack hash against the advertised hash, and exercise a packfile URI fetch with a pre-existing keep file. Signed-off-by: Ted Nyman Signed-off-by: Junio C Hamano --- fetch-pack.c | 33 ++++++++++++++++++--------------- t/t5702-protocol-v2.sh | 31 +++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/fetch-pack.c b/fetch-pack.c index 922a9b25812c68..626f799712ec2e 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -1854,9 +1854,10 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, } for (i = 0; i < packfile_uris.nr; i++) { + bool created_keep; int j; struct child_process cmd = CHILD_PROCESS_INIT; - char packname[GIT_MAX_HEXSZ + 1]; + char packhash[GIT_MAX_HEXSZ + 1]; const char *uri = packfile_uris.items[i].string + the_hash_algo->hexsz + 1; @@ -1874,16 +1875,17 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, if (start_command(&cmd)) die("fetch-pack: unable to spawn http-fetch"); - if (read_in_full(cmd.out, packname, 5) < 0 || - memcmp(packname, "keep\t", 5)) - die("fetch-pack: expected keep then TAB at start of http-fetch output"); + if (read_in_full(cmd.out, packhash, 5) != 5 || + (memcmp(packhash, "keep\t", 5) && + memcmp(packhash, "pack\t", 5))) + die("fetch-pack: expected pack or keep then TAB at start of http-fetch output"); + created_keep = !memcmp(packhash, "keep\t", 5); - if (read_in_full(cmd.out, packname, - the_hash_algo->hexsz + 1) < 0 || - packname[the_hash_algo->hexsz] != '\n') - die("fetch-pack: expected hash then LF at end of http-fetch output"); - - packname[the_hash_algo->hexsz] = '\0'; + if (read_in_full(cmd.out, packhash, + the_hash_algo->hexsz + 1) != the_hash_algo->hexsz + 1 || + packhash[the_hash_algo->hexsz] != '\n') + die("fetch-pack: expected hash then LF in http-fetch output"); + packhash[the_hash_algo->hexsz] = '\0'; parse_gitmodules_oids(cmd.out, &fsck_options.gitmodules_found); @@ -1892,16 +1894,17 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, if (finish_command(&cmd)) die("fetch-pack: unable to finish http-fetch"); - if (memcmp(packfile_uris.items[i].string, packname, + if (memcmp(packfile_uris.items[i].string, packhash, the_hash_algo->hexsz)) die("fetch-pack: pack downloaded from %s does not match expected hash %.*s", uri, (int) the_hash_algo->hexsz, packfile_uris.items[i].string); - string_list_append_nodup(pack_lockfiles, - xstrfmt("%s/pack/pack-%s.keep", - repo_get_object_directory(the_repository), - packname)); + if (created_keep) + string_list_append_nodup(pack_lockfiles, + xstrfmt("%s/pack/pack-%s.keep", + repo_get_object_directory(the_repository), + packhash)); } string_list_clear(&packfile_uris, 0); strvec_clear(&index_pack_args); diff --git a/t/t5702-protocol-v2.sh b/t/t5702-protocol-v2.sh index 74a2b7730bf3da..0f05286de8b4df 100755 --- a/t/t5702-protocol-v2.sh +++ b/t/t5702-protocol-v2.sh @@ -1291,6 +1291,37 @@ test_expect_success 'packfile URIs with fetch instead of clone' ' fetch "$HTTPD_URL/smart/http_parent" ' +test_expect_success 'packfile URI preserves an existing keep file' ' + P="$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + rm -rf "$P" http_child keep.expect && + + git init "$P" && + git -C "$P" config uploadpack.allowsidebandall true && + + echo my-blob >"$P/my-blob" && + git -C "$P" add my-blob && + git -C "$P" commit -m x && + configure_exclusion "$P" my-blob >h && + + git init http_child && + packhash=$(cat packh) && + keep="http_child/.git/objects/pack/pack-$packhash.keep" && + echo pre-existing >"$keep" && + cp "$keep" keep.expect && + + GIT_TEST_SIDEBAND_ALL=1 \ + git -C http_child -c protocol.version=2 \ + -c fetch.uriprotocols=http,https \ + fetch "$HTTPD_URL/smart/http_parent" && + + test_path_is_file \ + "http_child/.git/objects/pack/pack-$packhash.pack" && + test_path_is_file \ + "http_child/.git/objects/pack/pack-$packhash.idx" && + test_cmp keep.expect "$keep" && + git -C http_child cat-file -e "$(cat h)" +' + test_expect_success 'fetching with valid packfile URI but invalid hash fails' ' P="$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && rm -rf "$P" http_child log && From 091aa86a0f130f0c6858417eab75c4324f354b8f Mon Sep 17 00:00:00 2001 From: Friel Date: Sat, 15 Aug 2026 07:15:37 +0000 Subject: [PATCH 25/28] pack-objects: trace bytes written to stdout --- builtin/pack-objects.c | 6 ++++++ t/t5300-pack-object.sh | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 1ec5b6f206366e..f96ad42174fb2f 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -1337,6 +1337,7 @@ static void write_pack_file(void) uint32_t nr_remaining = nr_result; time_t last_mtime = 0; struct object_entry **write_order; + off_t bytes_written = 0; if (progress > pack_to_stdout) progress_state = start_progress(the_repository, @@ -1390,6 +1391,8 @@ static void write_pack_file(void) } if (pack_to_stdout) { + bytes_written += hashfile_total(f) + + the_repository->hash_algo->rawsz; /* * We never fsync when writing to stdout since we may * not be writing to an actual pack file. For instance, @@ -1510,6 +1513,9 @@ static void write_pack_file(void) written, nr_result); trace2_data_intmax("pack-objects", the_repository, "write_pack_file/wrote", nr_result); + if (pack_to_stdout) + trace2_data_intmax("pack-objects", the_repository, + "written/bytes", bytes_written); } static int no_try_delta(const char *path) diff --git a/t/t5300-pack-object.sh b/t/t5300-pack-object.sh index 9dabb3615aff56..101729f1f6a2a7 100755 --- a/t/t5300-pack-object.sh +++ b/t/t5300-pack-object.sh @@ -33,6 +33,16 @@ test_expect_success 'setup' ' } >expect ' +test_expect_success 'pack-object traces bytes written to stdout' ' + test_when_finished "rm -f pack.trace pack.pack" && + GIT_TRACE2_EVENT="$PWD/pack.trace" \ + git pack-objects --quiet --revs --stdout >pack.pack <<-EOF && + $commit + EOF + bytes=$(test_file_size pack.pack) && + test_grep "\"key\":\"written/bytes\",\"value\":\"$bytes\"" pack.trace +' + test_expect_success 'setup pack-object Date: Mon, 17 Aug 2026 03:54:18 +0000 Subject: [PATCH 26/28] pack-objects: trace bytes for all pack output --- builtin/pack-objects.c | 10 +++++----- t/t5300-pack-object.sh | 16 +++++++++++++++- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index f96ad42174fb2f..01a11da860c01b 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -1390,9 +1390,10 @@ static void write_pack_file(void) display_progress(progress_state, written); } + /* Every finalization path appends the pack checksum. */ + bytes_written += hashfile_total(f) + + the_repository->hash_algo->rawsz; if (pack_to_stdout) { - bytes_written += hashfile_total(f) + - the_repository->hash_algo->rawsz; /* * We never fsync when writing to stdout since we may * not be writing to an actual pack file. For instance, @@ -1513,9 +1514,8 @@ static void write_pack_file(void) written, nr_result); trace2_data_intmax("pack-objects", the_repository, "write_pack_file/wrote", nr_result); - if (pack_to_stdout) - trace2_data_intmax("pack-objects", the_repository, - "written/bytes", bytes_written); + trace2_data_intmax("pack-objects", the_repository, + "write_pack_file/wrote_bytes", bytes_written); } static int no_try_delta(const char *path) diff --git a/t/t5300-pack-object.sh b/t/t5300-pack-object.sh index 101729f1f6a2a7..aac139e6a096eb 100755 --- a/t/t5300-pack-object.sh +++ b/t/t5300-pack-object.sh @@ -40,7 +40,21 @@ test_expect_success 'pack-object traces bytes written to stdout' ' $commit EOF bytes=$(test_file_size pack.pack) && - test_grep "\"key\":\"written/bytes\",\"value\":\"$bytes\"" pack.trace + test_grep "\"key\":\"write_pack_file/wrote_bytes\",\"value\":\"$bytes\"" pack.trace +' + +test_expect_success 'pack-object traces bytes written to split pack files' ' + test_when_finished "rm -f split.trace traced-pack-*" && + GIT_TRACE2_EVENT="$PWD/split.trace" \ + git -c pack.packSizeLimit=3m pack-objects --quiet traced-pack Date: Mon, 17 Aug 2026 03:57:02 +0000 Subject: [PATCH 27/28] pack-objects: accumulate bytes after finalization --- builtin/pack-objects.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 01a11da860c01b..bbf1adb437dd8d 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -1348,6 +1348,7 @@ static void write_pack_file(void) do { unsigned char hash[GIT_MAX_RAWSZ]; char *pack_tmp_name = NULL; + off_t pack_bytes; if (pack_to_stdout) { /* @@ -1390,8 +1391,7 @@ static void write_pack_file(void) display_progress(progress_state, written); } - /* Every finalization path appends the pack checksum. */ - bytes_written += hashfile_total(f) + + pack_bytes = hashfile_total(f) + the_repository->hash_algo->rawsz; if (pack_to_stdout) { /* @@ -1423,6 +1423,7 @@ static void write_pack_file(void) write_bitmap_index = 0; } } + bytes_written += pack_bytes; if (!pack_to_stdout) { struct stat st; From 633e355546f268474504340a8cd494c7eebf4c76 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 21 Aug 2026 15:06:22 -0500 Subject: [PATCH 28/28] index-pack: hash full blobs in a bounded worker pool The first pass through a pack inflates and hashes each object on the main thread. `pack.threads` applies later while resolving deltas, so SHA1DC work for full blobs remains serial even when CPUs are idle. In a 99 Hz profile of an 844,020,252-byte pack dominated by full blobs, SHA1DC accounted for 72.14% of sampled user CPU. Add an opt-in worker pool for complete heap-backed blobs. The producer continues parsing, inflating, computing CRCs, and writing the pack. The workers use the normal object hashing backend, including SHA1DC, while the main thread performs the existing ODB, collision, and content checks as deferred results are retired in queue order. Bound retained data by both bytes and job count. Strict, fsck, promisor, non-threaded, streamed, and otherwise ineligible objects retain the serial path. The default is disabled. Across three runs of the original prototype on that pack, two workers reduced median wall time from 36.076 to 19.227 seconds. CPU changed from 50.121 to 50.955 seconds and peak RSS from 413.7 to 409.8 MiB. The object/delta-heavy control did not show a wall-time improvement. The fixed-size queue used here has not been rebenchmarked. t5352 covers output equivalence, queue limits, serial fallbacks, collision and duplicate handling, REF/OFS ordering, corrupt input, and configuration validation. p5352 compares serial, one-worker, and two-worker indexing on a reproducible full-blob pack. Existing SHA-1 and SHA-256 index-pack tests also pass. --- Documentation/config.adoc | 2 + Documentation/config/indexpack.adoc | 21 ++ Documentation/git-index-pack.adoc | 5 + builtin/index-pack.c | 238 ++++++++++++++++++++++- t/meson.build | 2 + t/perf/p5352-index-pack-hash-pipeline.sh | 49 +++++ t/t5352-index-pack-hash-pipeline.sh | 177 +++++++++++++++++ 7 files changed, 490 insertions(+), 4 deletions(-) create mode 100644 Documentation/config/indexpack.adoc create mode 100755 t/perf/p5352-index-pack-hash-pipeline.sh create mode 100755 t/t5352-index-pack-hash-pipeline.sh diff --git a/Documentation/config.adoc b/Documentation/config.adoc index 1ef72de62f2ba6..a6b83c014288cd 100644 --- a/Documentation/config.adoc +++ b/Documentation/config.adoc @@ -518,6 +518,8 @@ include::config/includeif.adoc[] include::config/index.adoc[] +include::config/indexpack.adoc[] + include::config/init.adoc[] include::config/instaweb.adoc[] diff --git a/Documentation/config/indexpack.adoc b/Documentation/config/indexpack.adoc new file mode 100644 index 00000000000000..a56beccd390f1a --- /dev/null +++ b/Documentation/config/indexpack.adoc @@ -0,0 +1,21 @@ +indexPack.hashThreads:: + Experimental number of workers used to hash full blobs during the + first pass of linkgit:git-index-pack[1]. The default, zero, disables + the workers. Values from 1 through 32 are accepted. Parsing and + inflation remain serial, and normal collision/content validation runs + on the main thread. This is independent of `pack.threads` and of + concurrent packfile-URI downloads. Strict, fsck, and promisor modes, + and builds without thread support, use the existing serial path. + +indexPack.hashBufferSize:: + Maximum retained blob-buffer bytes for `indexPack.hashThreads`, + including the terminating byte, the producer's reserved buffer, and + completed results awaiting validation. Defaults to 64 MiB. At most + twice the number of hash workers can be outstanding. Objects that do + not fit use the existing serial path; the limit is not a bound on + Git's total memory use. Usual `k`, `m`, and `g` suffixes are accepted. + +indexPack.hashMinSize:: + Minimum full-blob size eligible for `indexPack.hashThreads`. + Defaults to 64 KiB. Blobs above `core.bigFileThreshold` retain their + existing streaming path. Usual `k`, `m`, and `g` suffixes are accepted. diff --git a/Documentation/git-index-pack.adoc b/Documentation/git-index-pack.adoc index 18036953c06b22..0cf251ac9bb65d 100644 --- a/Documentation/git-index-pack.adoc +++ b/Documentation/git-index-pack.adoc @@ -148,6 +148,11 @@ accessible through promisor objects. + Requires to not be specified. +CONFIGURATION +------------- + +include::config/indexpack.adoc[] + NOTES ----- diff --git a/builtin/index-pack.c b/builtin/index-pack.c index bc86925ad04340..9b6a0abbb2b44f 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -31,6 +31,7 @@ #include "run-command.h" #include "setup.h" #include "strvec.h" +#include "trace2.h" static const char index_pack_usage[] = "git index-pack [-v] [-o ] [--keep | --keep=] [--[no-]rev-index] [--verify] [--strict[==...]] [--fsck-objects[==...]] ( | --stdin [--fix-thin] [])"; @@ -43,6 +44,43 @@ struct object_entry { signed char real_type; }; +/* + * Hash workers own no repository state. They only hash immutable full-blob + * buffers; the main thread retires their results and performs the usual + * collision/content checks before releasing those buffers. + */ +struct first_pass_hash_job { + struct object_entry *obj; + struct object_id oid; + void *data; + int done; +}; + +struct first_pass_hash_pool { + pthread_t *threads; + pthread_mutex_t mutex; + pthread_cond_t work_ready; + pthread_cond_t result_ready; + struct first_pass_hash_job *queue; + size_t nr_threads, queue_size; + size_t first, next_work, nr, pending; + size_t buffered, jobs; + int stop; +}; + +#define FIRST_PASS_HASH_MAX_THREADS 32 + +static int first_pass_hash_threads; +static size_t first_pass_hash_buffer_size = 64 * 1024 * 1024; +static size_t first_pass_hash_min_size = 64 * 1024; +static struct first_pass_hash_pool first_pass_hash_pool; + +static struct first_pass_hash_job *reserve_first_pass_hash(struct object_entry *obj); +static void submit_first_pass_hash(struct first_pass_hash_job *job, + struct object_entry *obj, void *data); +static void start_first_pass_hash(void); +static void finish_first_pass_hash(void); + struct object_stat { unsigned delta_depth; int base_object_no; @@ -479,7 +517,7 @@ static void *unpack_entry_data(off_t offset, size_t size, char hdr[32]; int hdrlen; - if (!is_delta_type(type)) { + if (!is_delta_type(type) && oid) { hdrlen = format_object_header(hdr, sizeof(hdr), type, size); git_hash_init(&c, the_hash_algo); git_hash_update(&c, hdr, hdrlen); @@ -520,7 +558,8 @@ static void *unpack_entry_data(off_t offset, size_t size, static void *unpack_raw_entry(struct object_entry *obj, off_t *ofs_offset, struct object_id *ref_oid, - struct object_id *oid) + struct object_id *oid, + struct first_pass_hash_job **hash_job) { unsigned char *p; size_t size, c; @@ -582,7 +621,9 @@ static void *unpack_raw_entry(struct object_entry *obj, } obj->hdr_size = consumed_bytes - obj->idx.offset; - data = unpack_entry_data(obj->idx.offset, obj->size, obj->type, oid); + *hash_job = reserve_first_pass_hash(obj); + data = unpack_entry_data(obj->idx.offset, obj->size, obj->type, + *hash_job ? NULL : oid); obj->idx.crc32 = input_crc32; return data; } @@ -978,6 +1019,171 @@ static void sha1_object(const void *data, struct object_entry *obj_entry, free(new_data); } +static void *first_pass_hash_worker(void *data UNUSED) +{ + struct first_pass_hash_pool *pool = &first_pass_hash_pool; + + trace2_thread_start("index-pack-hash"); + for (;;) { + struct first_pass_hash_job *job; + + pthread_mutex_lock(&pool->mutex); + while (!pool->pending && !pool->stop) + pthread_cond_wait(&pool->work_ready, &pool->mutex); + if (!pool->pending) { + pthread_mutex_unlock(&pool->mutex); + break; + } + job = &pool->queue[pool->next_work]; + pool->next_work = (pool->next_work + 1) % pool->queue_size; + pool->pending--; + pthread_mutex_unlock(&pool->mutex); + + /* This uses the same collision-detecting hash as the serial path. */ + hash_object_file(the_hash_algo, job->data, job->obj->size, + OBJ_BLOB, &job->oid); + + pthread_mutex_lock(&pool->mutex); + job->done = 1; + pthread_cond_signal(&pool->result_ready); + pthread_mutex_unlock(&pool->mutex); + } + trace2_thread_exit(); + return NULL; +} + +static void retire_first_pass_hash(void) +{ + struct first_pass_hash_pool *pool = &first_pass_hash_pool; + struct first_pass_hash_job *job; + size_t allocation; + + pthread_mutex_lock(&pool->mutex); + if (!pool->nr) + BUG("no queued first-pass hash to retire"); + job = &pool->queue[pool->first]; + while (!job->done) + pthread_cond_wait(&pool->result_ready, &pool->mutex); + pool->first = (pool->first + 1) % pool->queue_size; + pool->nr--; + pthread_mutex_unlock(&pool->mutex); + + /* All ODB and object-cache access stays on the main thread. */ + oidcpy(&job->obj->idx.oid, &job->oid); + sha1_object(job->data, NULL, job->obj->size, OBJ_BLOB, + &job->obj->idx.oid); + allocation = st_add(job->obj->size, 1); + free(job->data); + pool->buffered -= allocation; +} + +static struct first_pass_hash_job *reserve_first_pass_hash(struct object_entry *obj) +{ + struct first_pass_hash_pool *pool = &first_pass_hash_pool; + size_t allocation; + + if (!pool->nr_threads || obj->type != OBJ_BLOB || + obj->size < first_pass_hash_min_size || + obj->size >= first_pass_hash_buffer_size || + obj->size > repo_settings_get_big_file_threshold(the_repository)) + return NULL; + + allocation = st_add(obj->size, 1); + while (pool->nr == pool->queue_size || + allocation > first_pass_hash_buffer_size - pool->buffered) + retire_first_pass_hash(); + + /* Reserve before the producer allocates the inflated blob. */ + pool->buffered += allocation; + return &pool->queue[(pool->first + pool->nr) % pool->queue_size]; +} + +static void submit_first_pass_hash(struct first_pass_hash_job *job, + struct object_entry *obj, void *data) +{ + struct first_pass_hash_pool *pool = &first_pass_hash_pool; + + assert(pool->nr_threads && data && obj->type == OBJ_BLOB); + job->obj = obj; + job->data = data; + job->done = 0; + + pthread_mutex_lock(&pool->mutex); + pool->nr++; + pool->pending++; + pool->jobs++; + pthread_cond_signal(&pool->work_ready); + pthread_mutex_unlock(&pool->mutex); +} + +static void stop_first_pass_hash(size_t nr_threads) +{ + struct first_pass_hash_pool *pool = &first_pass_hash_pool; + size_t i; + + pthread_mutex_lock(&pool->mutex); + pool->stop = 1; + pthread_cond_broadcast(&pool->work_ready); + pthread_mutex_unlock(&pool->mutex); + for (i = 0; i < nr_threads; i++) + pthread_join(pool->threads[i], NULL); + pthread_cond_destroy(&pool->result_ready); + pthread_cond_destroy(&pool->work_ready); + pthread_mutex_destroy(&pool->mutex); + free(pool->queue); + free(pool->threads); +} + +static void start_first_pass_hash(void) +{ + struct first_pass_hash_pool *pool = &first_pass_hash_pool; + size_t i; + int ret; + + /* These modes share fsck/object-cache state; retain their serial path. */ + if (!HAVE_THREADS || !first_pass_hash_threads || strict || + do_fsck_object || record_outgoing_links || + first_pass_hash_min_size >= first_pass_hash_buffer_size) { + trace2_data_intmax("index-pack", the_repository, + "first_pass_hash/threads", 0); + return; + } + + pool->nr_threads = first_pass_hash_threads; + pool->queue_size = st_mult(pool->nr_threads, 2); + CALLOC_ARRAY(pool->threads, pool->nr_threads); + CALLOC_ARRAY(pool->queue, pool->queue_size); + pthread_mutex_init(&pool->mutex, NULL); + pthread_cond_init(&pool->work_ready, NULL); + pthread_cond_init(&pool->result_ready, NULL); + for (i = 0; i < pool->nr_threads; i++) { + ret = pthread_create(&pool->threads[i], NULL, + first_pass_hash_worker, NULL); + if (ret) { + stop_first_pass_hash(i); + die(_("unable to create index-pack hash thread: %s"), + strerror(ret)); + } + } + trace2_data_intmax("index-pack", the_repository, + "first_pass_hash/threads", pool->nr_threads); +} + +static void finish_first_pass_hash(void) +{ + struct first_pass_hash_pool *pool = &first_pass_hash_pool; + + if (!pool->nr_threads) + return; + while (pool->nr) + retire_first_pass_hash(); + + stop_first_pass_hash(pool->nr_threads); + trace2_data_intmax("index-pack", the_repository, + "first_pass_hash/jobs", pool->jobs); + pool->nr_threads = 0; +} + /* * Ensure that this node has been reconstructed and return its contents. * @@ -1255,6 +1461,8 @@ static void parse_pack_objects(unsigned char *hash) struct stat st; struct git_hash_ctx tmp_ctx; + start_first_pass_hash(); + if (verbose) progress = start_progress( the_repository, @@ -1263,9 +1471,10 @@ static void parse_pack_objects(unsigned char *hash) nr_objects); for (i = 0; i < nr_objects; i++) { struct object_entry *obj = &objects[i]; + struct first_pass_hash_job *hash_job; void *data = unpack_raw_entry(obj, &ofs_delta->offset, &ref_delta_oid, - &obj->idx.oid); + &obj->idx.oid, &hash_job); obj->real_type = obj->type; if (obj->type == OBJ_OFS_DELTA) { nr_ofs_deltas++; @@ -1276,6 +1485,9 @@ static void parse_pack_objects(unsigned char *hash) oidcpy(&ref_deltas[nr_ref_deltas].oid, &ref_delta_oid); ref_deltas[nr_ref_deltas].obj_no = i; nr_ref_deltas++; + } else if (hash_job) { + submit_first_pass_hash(hash_job, obj, data); + data = NULL; } else if (!data) { /* large blobs, check later */ obj->real_type = OBJ_BAD; @@ -1287,6 +1499,7 @@ static void parse_pack_objects(unsigned char *hash) display_progress(progress, i+1); } objects[i].idx.offset = consumed_bytes; + finish_first_pass_hash(); stop_progress(&progress); /* Check pack integrity */ @@ -1667,6 +1880,23 @@ static int git_index_pack_config(const char *k, const char *v, { struct pack_idx_option *opts = cb; + if (!strcmp(k, "indexpack.hashthreads")) { + first_pass_hash_threads = git_config_int(k, v, ctx->kvi); + if (first_pass_hash_threads < 0 || + first_pass_hash_threads > FIRST_PASS_HASH_MAX_THREADS) + die(_("%s must be between 0 and %d"), + k, FIRST_PASS_HASH_MAX_THREADS); + return 0; + } + if (!strcmp(k, "indexpack.hashbuffersize")) { + first_pass_hash_buffer_size = git_config_ulong(k, v, ctx->kvi); + return 0; + } + if (!strcmp(k, "indexpack.hashminsize")) { + first_pass_hash_min_size = git_config_ulong(k, v, ctx->kvi); + return 0; + } + if (!strcmp(k, "pack.indexversion")) { opts->version = git_config_int(k, v, ctx->kvi); if (opts->version > 2) diff --git a/t/meson.build b/t/meson.build index a25f37d2f5ae7d..65e26a371bd8f3 100644 --- a/t/meson.build +++ b/t/meson.build @@ -639,6 +639,7 @@ integration_tests = [ 't5334-incremental-multi-pack-index.sh', 't5335-compact-multi-pack-index.sh', 't5351-unpack-large-objects.sh', + 't5352-index-pack-hash-pipeline.sh', 't5400-send-pack.sh', 't5401-update-hooks.sh', 't5402-post-merge-hook.sh', @@ -1172,6 +1173,7 @@ benchmarks = [ 'perf/p5326-multi-pack-bitmaps.sh', 'perf/p5332-multi-pack-reuse.sh', 'perf/p5333-pseudo-merge-bitmaps.sh', + 'perf/p5352-index-pack-hash-pipeline.sh', 'perf/p5550-fetch-tags.sh', 'perf/p5551-fetch-rescan.sh', 'perf/p5600-partial-clone.sh', diff --git a/t/perf/p5352-index-pack-hash-pipeline.sh b/t/perf/p5352-index-pack-hash-pipeline.sh new file mode 100755 index 00000000000000..c9af7eafc5e6da --- /dev/null +++ b/t/perf/p5352-index-pack-hash-pipeline.sh @@ -0,0 +1,49 @@ +#!/bin/sh + +test_description='Test index-pack first-pass hash worker performance + +GIT_PERF_5352_NR_BLOBS controls the number of full blobs in the input pack. +GIT_PERF_5352_BLOB_SIZE controls the size of each blob in bytes. +Keep the blob size between 65536 and 67108863 to exercise the worker path. +' + +. ./perf-lib.sh + +test_perf_fresh_repo + +: ${GIT_PERF_5352_NR_BLOBS:=128} +: ${GIT_PERF_5352_BLOB_SIZE:=1048576} + +test_expect_success 'create a pack of full blobs' ' + for i in $(test_seq 1 "$GIT_PERF_5352_NR_BLOBS") + do + test-tool genrandom "index-pack-hash-$i" \ + "$GIT_PERF_5352_BLOB_SIZE" | + git hash-object -w --stdin || return 1 + done >oids && + git pack-objects --stdout --window=0 input.pack +' + +test_size 'pack size' ' + test_file_size input.pack +' + +test_perf 'index-pack, serial hash' \ + --setup 'rm -rf repo.git && git init --bare -q repo.git' ' + git -C repo.git -c indexPack.hashThreads=0 \ + index-pack --threads=1 --stdin /dev/null +' + +test_perf 'index-pack, one hash worker' --prereq PTHREADS \ + --setup 'rm -rf repo.git && git init --bare -q repo.git' ' + git -C repo.git -c indexPack.hashThreads=1 \ + index-pack --threads=1 --stdin /dev/null +' + +test_perf 'index-pack, two hash workers' --prereq PTHREADS \ + --setup 'rm -rf repo.git && git init --bare -q repo.git' ' + git -C repo.git -c indexPack.hashThreads=2 \ + index-pack --threads=1 --stdin /dev/null +' + +test_done diff --git a/t/t5352-index-pack-hash-pipeline.sh b/t/t5352-index-pack-hash-pipeline.sh new file mode 100755 index 00000000000000..6b1642c2a7f561 --- /dev/null +++ b/t/t5352-index-pack-hash-pipeline.sh @@ -0,0 +1,177 @@ +#!/bin/sh + +test_description='bounded first-pass index-pack hash workers' + +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-pack.sh + +run_index () { + name=$1 && + input=$2 && + shift 2 && + git init --bare "$name.git" && + GIT_TRACE2_EVENT="$TRASH_DIRECTORY/$name.trace" \ + git -C "$name.git" "$@" <"$input" >"$name.out" +} + +trace_value () { + key=$1 && + value=$2 && + file=$3 && + test_grep "\"key\":\"first_pass_hash/$key\",\"value\":\"$value\"" "$file" +} + +compare_pack_files () { + left=$1 && + right=$2 && + pack_hash=$(cut -f2 "$left.out") && + test_cmp "$left.out" "$right.out" && + for suffix in pack idx rev + do + test_cmp_bin "$left.git/objects/pack/pack-$pack_hash.$suffix" \ + "$right.git/objects/pack/pack-$pack_hash.$suffix" || return 1 + done +} + +test_expect_success 'make a pack of full blobs' ' + for i in $(test_seq 1 8) + do + test-tool genrandom "hash-pipeline-$i" 65536 >"blob-$i" && + git hash-object -w "blob-$i" || return 1 + done >oids && + git pack-objects --stdout --window=0 input.pack && + run_index serial input.pack index-pack --stdin --fix-thin && + trace_value threads 0 serial.trace +' + +test_expect_success PTHREADS 'workers preserve pack, index and reverse index' ' + run_index parallel input.pack \ + -c indexPack.hashThreads=2 \ + -c indexPack.hashBufferSize=131074 \ + index-pack --stdin --fix-thin && + compare_pack_files serial parallel && + trace_value threads 2 parallel.trace && + trace_value jobs 8 parallel.trace +' + +test_expect_success PTHREADS 'an object must fit including its trailing byte' ' + run_index too-small input.pack \ + -c indexPack.hashThreads=2 \ + -c indexPack.hashBufferSize=65536 \ + -c indexPack.hashMinSize=0 \ + index-pack --stdin --fix-thin && + compare_pack_files serial too-small && + trace_value jobs 0 too-small.trace +' + +test_expect_success PTHREADS 'streamed large blobs retain the serial path' ' + run_index streamed input.pack \ + -c indexPack.hashThreads=2 \ + -c indexPack.hashMinSize=0 \ + -c core.bigFileThreshold=1 \ + index-pack --stdin --fix-thin && + compare_pack_files serial streamed && + trace_value jobs 0 streamed.trace +' + +test_expect_success PTHREADS 'validation modes retain the serial path' ' + for mode in strict fsck-objects promisor + do + run_index "$mode" input.pack \ + -c indexPack.hashThreads=2 \ + -c indexPack.hashMinSize=0 \ + index-pack --stdin --fix-thin "--$mode" && + trace_value threads 0 "$mode.trace" || return 1 + done +' + +test_expect_success PTHREADS 'existing-object collision check is not skipped' ' + git init --bare collision.git && + a=$(git -C collision.git hash-object -w ../blob-1) && + b=$(git -C collision.git hash-object -w ../blob-2) && + a_path=$(echo "$a" | sed "s!^..!&/!") && + b_path=$(echo "$b" | sed "s!^..!&/!") && + cp -f "collision.git/objects/$b_path" "collision.git/objects/$a_path" && + test_env GIT_TRACE2_EVENT="$TRASH_DIRECTORY/collision.trace" \ + test_must_fail git -C collision.git \ + -c indexPack.hashThreads=2 \ + index-pack --stdin --fix-thin collision.err && + trace_value threads 2 collision.trace && + test_grep "SHA1 COLLISION FOUND" collision.err +' + +test_expect_success PTHREADS 'duplicate full blobs preserve native acceptance' ' + { + pack_header 2 && + pack_obj "$EMPTY_BLOB" && + pack_obj "$EMPTY_BLOB" + } >duplicates.pack && + pack_trailer duplicates.pack && + run_index duplicate-serial duplicates.pack index-pack --stdin && + run_index duplicate-parallel duplicates.pack \ + -c indexPack.hashThreads=2 \ + -c indexPack.hashBufferSize=2 \ + -c indexPack.hashMinSize=0 \ + index-pack --stdin && + compare_pack_files duplicate-serial duplicate-parallel && + trace_value jobs 2 duplicate-parallel.trace && + git init --bare duplicate-strict.git && + test_must_fail git -C duplicate-strict.git \ + -c indexPack.hashThreads=2 \ + -c indexPack.hashMinSize=0 \ + index-pack --strict --stdin ref.pack && + pack_trailer ref.pack && + pack_obj "$A" "$B" >ref-entry && + { + pack_header 2 && + pack_obj "$B" && + # The full B entry is eleven bytes; the delta data is five. + printf "\145\013" && + dd if=ref-entry bs=1 skip=$((1 + $(test_oid rawsz))) + } >ofs.pack && + pack_trailer ofs.pack && + for kind in ref ofs + do + run_index "$kind" "$kind.pack" \ + -c indexPack.hashThreads=2 \ + -c indexPack.hashMinSize=0 \ + index-pack --stdin --fix-thin && + trace_value jobs 1 "$kind.trace" && + git -C "$kind.git" cat-file blob "$A" >actual && + test "$(git hash-object actual)" = "$A" || return 1 + done +' + +test_expect_success PTHREADS 'bad input cannot publish an index' ' + length=$(wc -c actual && + test_must_be_empty actual || return 1 + done +' + +test_expect_success 'invalid worker counts are rejected' ' + test_must_fail git -c indexPack.hashThreads=-1 index-pack input.pack && + test_must_fail git -c indexPack.hashThreads=33 index-pack input.pack +' + +test_done