diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 94ca1885c..bc2392bd7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,11 @@ on: - "**.md" - "**.txt" workflow_dispatch: # e.g. to manually trigger on foreign PRs + inputs: + capture-corpus: + description: "Capture the raw envelopes and debug files the integration tests produce, as artifacts (see docs/envelope-capture.md). Slower, and the event assertions fail by design." + type: boolean + default: false env: DOTNET_CLI_TELEMETRY_OPTOUT: 1 @@ -142,6 +147,8 @@ jobs: build_platform: WebGL env: UNITY_PATH: docker exec unity unity-editor + # Empty unless a capture run was dispatched; see docs/envelope-capture.md. + SENTRY_CAPTURE_PATH: ${{ inputs.capture-corpus && format('test/IntegrationTest/capture/webgl-{0}', matrix.unity-version) || '' }} steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -301,6 +308,7 @@ jobs: uses: ./.github/workflows/test-build-android.yml with: unity-version: ${{ matrix.unity-version }} + capture: ${{ inputs.capture-corpus || false }} test-run-android: name: Run Android ${{ matrix.unity-version }} Integration Test @@ -314,6 +322,7 @@ jobs: unity-version: ${{ matrix.unity-version }} api-level: ${{ matrix.api-level }} init-type: ${{ matrix.init-type }} + capture: ${{ inputs.capture-corpus || false }} strategy: fail-fast: false matrix: @@ -336,6 +345,7 @@ jobs: uses: ./.github/workflows/test-build-ios.yml with: unity-version: ${{ matrix.unity-version }} + capture: ${{ inputs.capture-corpus || false }} test-compile-ios: name: Compile iOS ${{ matrix.unity-version }} Test @@ -350,6 +360,7 @@ jobs: with: unity-version: ${{ matrix.unity-version }} init-type: ${{ matrix.init-type }} + capture: ${{ inputs.capture-corpus || false }} test-run-ios: name: Run iOS ${{ matrix.unity-version }} Integration Test @@ -360,6 +371,7 @@ jobs: unity-version: ${{ matrix.unity-version }} ios-version: ${{ matrix.ios-version }} init-type: ${{ matrix.init-type }} + capture: ${{ inputs.capture-corpus || false }} secrets: SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} SENTRY_TEST_DSN: ${{ secrets.SENTRY_TEST_DSN }} @@ -394,6 +406,7 @@ jobs: uses: ./.github/workflows/test-run-webgl.yml with: unity-version: ${{ matrix.unity-version }} + capture: ${{ inputs.capture-corpus || false }} test-build-linux: name: Build Linux ${{ matrix.unity-version }} Integration Test @@ -410,6 +423,7 @@ jobs: uses: ./.github/workflows/test-build-linux.yml with: unity-version: ${{ matrix.unity-version }} + capture: ${{ inputs.capture-corpus || false }} test-build-windows: name: Build Windows ${{ matrix.unity-version }} Integration Test @@ -426,6 +440,7 @@ jobs: uses: ./.github/workflows/test-build-windows.yml with: unity-version: ${{ matrix.unity-version }} + capture: ${{ inputs.capture-corpus || false }} test-build-macos: name: Build macOS ${{ matrix.unity-version }} Integration Test @@ -442,6 +457,7 @@ jobs: uses: ./.github/workflows/test-build-macos.yml with: unity-version: ${{ matrix.unity-version }} + capture: ${{ inputs.capture-corpus || false }} test-run-linux: name: Run Linux ${{ matrix.backend }} ${{ matrix.unity-version }} Integration Test @@ -460,6 +476,7 @@ jobs: unity-version: ${{ matrix.unity-version }} platform: linux backend: ${{ matrix.backend }} + capture: ${{ inputs.capture-corpus || false }} test-run-windows: name: Run Windows ${{ matrix.backend }} ${{ matrix.unity-version }} Integration Test @@ -478,6 +495,7 @@ jobs: unity-version: ${{ matrix.unity-version }} platform: windows backend: ${{ matrix.backend }} + capture: ${{ inputs.capture-corpus || false }} test-run-macos: name: Run macOS ${{ matrix.backend }} ${{ matrix.unity-version }} Integration Test @@ -496,6 +514,7 @@ jobs: unity-version: ${{ matrix.unity-version }} platform: macos backend: ${{ matrix.backend }} + capture: ${{ inputs.capture-corpus || false }} build-size-summary: name: Build Size diff --git a/.github/workflows/test-build-android.yml b/.github/workflows/test-build-android.yml index 1524018ed..d15a853b1 100644 --- a/.github/workflows/test-build-android.yml +++ b/.github/workflows/test-build-android.yml @@ -5,6 +5,11 @@ on: unity-version: required: true type: string + capture: + required: false + type: boolean + default: false + description: "Capture the envelopes and debug files this job produces - see docs/envelope-capture.md" secrets: UNITY_LICENSE_SERVER_CONFIG: required: true @@ -25,6 +30,8 @@ jobs: GITHUB_ACTOR: ${{ github.actor }} UNITY_PATH: docker exec unity unity-editor UNITY_VERSION: ${{ inputs.unity-version }} + # Empty unless a capture run was dispatched; see docs/envelope-capture.md. + SENTRY_CAPTURE_PATH: ${{ inputs.capture && format('test/IntegrationTest/capture/android-{0}', inputs.unity-version) || '' }} steps: - name: Checkout @@ -176,3 +183,11 @@ jobs: !samples/IntegrationTest/Build/*_BackUpThisFolder_ButDontShipItWithYourGame retention-days: 14 # Lower retention period - we only need this to retry CI. + - name: Upload captured corpus + if: ${{ always() && inputs.capture }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: corpus-android-${{ inputs.unity-version }} + path: test/IntegrationTest/capture/ + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/test-build-ios.yml b/.github/workflows/test-build-ios.yml index 9dab592ec..729eb7784 100644 --- a/.github/workflows/test-build-ios.yml +++ b/.github/workflows/test-build-ios.yml @@ -5,6 +5,11 @@ on: unity-version: required: true type: string + capture: + required: false + type: boolean + default: false + description: "Capture the envelopes and debug files this job produces - see docs/envelope-capture.md" secrets: UNITY_LICENSE_SERVER_CONFIG: required: true @@ -31,6 +36,8 @@ jobs: GITHUB_ACTOR: ${{ github.actor }} UNITY_PATH: docker exec unity unity-editor UNITY_VERSION: ${{ inputs.unity-version }} + # Empty unless a capture run was dispatched; see docs/envelope-capture.md. + SENTRY_CAPTURE_PATH: ${{ inputs.capture && format('test/IntegrationTest/capture/ios-{0}', inputs.unity-version) || '' }} steps: - name: Checkout diff --git a/.github/workflows/test-build-linux.yml b/.github/workflows/test-build-linux.yml index 5535dc050..1065348c9 100644 --- a/.github/workflows/test-build-linux.yml +++ b/.github/workflows/test-build-linux.yml @@ -5,6 +5,11 @@ on: unity-version: required: true type: string + capture: + required: false + type: boolean + default: false + description: "Capture the envelopes and debug files this job produces - see docs/envelope-capture.md" secrets: UNITY_LICENSE_SERVER_CONFIG: required: true @@ -26,6 +31,8 @@ jobs: UNITY_PATH: docker exec unity unity-editor UNITY_VERSION: ${{ inputs.unity-version }} BUILD_PLATFORM: Linux + # Empty unless a capture run was dispatched; see docs/envelope-capture.md. + SENTRY_CAPTURE_PATH: ${{ inputs.capture && format('test/IntegrationTest/capture/linux-{0}', inputs.unity-version) || '' }} steps: - name: Checkout @@ -203,3 +210,12 @@ jobs: unity.log !samples/IntegrationTest/Build/*_BackUpThisFolder_ButDontShipItWithYourGame retention-days: 14 + + - name: Upload captured corpus + if: ${{ always() && inputs.capture }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: corpus-linux-${{ inputs.unity-version }} + path: test/IntegrationTest/capture/ + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/test-build-macos.yml b/.github/workflows/test-build-macos.yml index 318f71bd5..e06b567f8 100644 --- a/.github/workflows/test-build-macos.yml +++ b/.github/workflows/test-build-macos.yml @@ -5,6 +5,11 @@ on: unity-version: required: true type: string + capture: + required: false + type: boolean + default: false + description: "Capture the envelopes and debug files this job produces - see docs/envelope-capture.md" secrets: UNITY_LICENSE_SERVER_CONFIG: required: true @@ -24,6 +29,8 @@ jobs: env: UNITY_VERSION: ${{ inputs.unity-version }} BUILD_PLATFORM: MacOS + # Empty unless a capture run was dispatched; see docs/envelope-capture.md. + SENTRY_CAPTURE_PATH: ${{ inputs.capture && format('test/IntegrationTest/capture/macos-{0}', inputs.unity-version) || '' }} steps: - name: Checkout @@ -189,3 +196,12 @@ jobs: unity.log !samples/IntegrationTest/Build/*_BackUpThisFolder_ButDontShipItWithYourGame retention-days: 14 + + - name: Upload captured corpus + if: ${{ always() && inputs.capture }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: corpus-macos-${{ inputs.unity-version }} + path: test/IntegrationTest/capture/ + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/test-build-windows.yml b/.github/workflows/test-build-windows.yml index 89dedc474..f13ac179e 100644 --- a/.github/workflows/test-build-windows.yml +++ b/.github/workflows/test-build-windows.yml @@ -5,6 +5,11 @@ on: unity-version: required: true type: string + capture: + required: false + type: boolean + default: false + description: "Capture the envelopes and debug files this job produces - see docs/envelope-capture.md" secrets: UNITY_LICENSE_SERVER_CONFIG: required: true @@ -24,6 +29,8 @@ jobs: env: UNITY_VERSION: ${{ inputs.unity-version }} BUILD_PLATFORM: Windows + # Empty unless a capture run was dispatched; see docs/envelope-capture.md. + SENTRY_CAPTURE_PATH: ${{ inputs.capture && format('test/IntegrationTest/capture/windows-{0}', inputs.unity-version) || '' }} steps: - name: Checkout @@ -189,3 +196,12 @@ jobs: unity.log !samples/IntegrationTest/Build/*_BackUpThisFolder_ButDontShipItWithYourGame retention-days: 14 + + - name: Upload captured corpus + if: ${{ always() && inputs.capture }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: corpus-windows-${{ inputs.unity-version }} + path: test/IntegrationTest/capture/ + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/test-compile-ios.yml b/.github/workflows/test-compile-ios.yml index 74f58a129..5b29d6091 100644 --- a/.github/workflows/test-compile-ios.yml +++ b/.github/workflows/test-compile-ios.yml @@ -8,6 +8,11 @@ on: init-type: required: true type: string + capture: + required: false + type: boolean + default: false + description: "Capture the envelopes and debug files this job produces - see docs/envelope-capture.md" defaults: run: @@ -20,6 +25,8 @@ jobs: env: UNITY_VERSION: ${{ inputs.unity-version }} INIT_TYPE: ${{ inputs.init-type }} + # Empty unless a capture run was dispatched; see docs/envelope-capture.md. + SENTRY_CAPTURE_PATH: ${{ inputs.capture && format('test/IntegrationTest/capture/ios-{0}-{1}', inputs.unity-version, inputs.init-type) || '' }} steps: - name: Checkout @@ -122,3 +129,12 @@ jobs: name: build-size-iOS-${{ env.UNITY_VERSION }} path: build-size-measurements/*.json retention-days: 1 + + - name: Upload captured corpus + if: ${{ always() && inputs.capture }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: corpus-ios-${{ inputs.unity-version }}-${{ inputs.init-type }} + path: test/IntegrationTest/capture/ + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/test-run-android.yml b/.github/workflows/test-run-android.yml index 67b6471f6..6e1cf2ece 100644 --- a/.github/workflows/test-run-android.yml +++ b/.github/workflows/test-run-android.yml @@ -11,6 +11,11 @@ on: init-type: required: true type: string + capture: + required: false + type: boolean + default: false + description: "Capture the envelopes and debug files this job produces - see docs/envelope-capture.md" secrets: SENTRY_AUTH_TOKEN: required: true @@ -32,6 +37,8 @@ jobs: HOMEBREW_NO_INSTALL_CLEANUP: 1 SENTRY_DSN: ${{ secrets.SENTRY_TEST_DSN }} SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + # Empty unless a capture run was dispatched; see docs/envelope-capture.md. + SENTRY_CAPTURE_PATH: ${{ inputs.capture && format('test/IntegrationTest/capture/run-android-{0}-{1}-{2}', inputs.unity-version, inputs.api-level, inputs.init-type) || '' }} steps: - name: Checkout @@ -181,3 +188,12 @@ jobs: ${{ env.ARTIFACTS_PATH }} test/IntegrationTest/results/ retention-days: 14 + + - name: Upload captured corpus + if: ${{ always() && inputs.capture }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: corpus-run-android-${{ inputs.unity-version }}-${{ inputs.api-level }}-${{ inputs.init-type }} + path: test/IntegrationTest/capture/ + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/test-run-desktop.yml b/.github/workflows/test-run-desktop.yml index 979ee9a45..f08ebea64 100644 --- a/.github/workflows/test-run-desktop.yml +++ b/.github/workflows/test-run-desktop.yml @@ -14,6 +14,11 @@ on: type: string default: "" description: "macOS: native or cocoa. Windows: native or crashpad. Linux: native or breakpad." + capture: + required: false + type: boolean + default: false + description: "Capture the envelopes and debug files this job produces - see docs/envelope-capture.md" secrets: SENTRY_AUTH_TOKEN: required: true @@ -31,6 +36,8 @@ jobs: env: SENTRY_DSN: ${{ secrets.SENTRY_TEST_DSN }} SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + # Empty unless a capture run was dispatched; see docs/envelope-capture.md. + SENTRY_CAPTURE_PATH: ${{ inputs.capture && format('test/IntegrationTest/capture/run-{0}{1}-{2}', inputs.platform, inputs.backend && format('-{0}', inputs.backend) || '', inputs.unity-version) || '' }} steps: - name: Checkout @@ -94,3 +101,12 @@ jobs: path: | test/IntegrationTest/results/ retention-days: 14 + + - name: Upload captured corpus + if: ${{ always() && inputs.capture }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: corpus-run-${{ inputs.platform }}${{ inputs.backend && format('-{0}', inputs.backend) || '' }}-${{ inputs.unity-version }} + path: test/IntegrationTest/capture/ + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/test-run-ios.yml b/.github/workflows/test-run-ios.yml index 7d1e0bd89..1f6577cbe 100644 --- a/.github/workflows/test-run-ios.yml +++ b/.github/workflows/test-run-ios.yml @@ -11,6 +11,11 @@ on: init-type: required: true type: string + capture: + required: false + type: boolean + default: false + description: "Capture the envelopes and debug files this job produces - see docs/envelope-capture.md" # Map the workflow outputs to job outputs outputs: status: @@ -40,6 +45,8 @@ jobs: ARTIFACTS_PATH: samples/IntegrationTest/test-artifacts/ SENTRY_DSN: ${{ secrets.SENTRY_TEST_DSN }} SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + # Empty unless a capture run was dispatched; see docs/envelope-capture.md. + SENTRY_CAPTURE_PATH: ${{ inputs.capture && format('test/IntegrationTest/capture/run-ios-{0}-{1}-{2}', inputs.unity-version, inputs.ios-version, inputs.init-type) || '' }} steps: - name: Checkout @@ -85,3 +92,12 @@ jobs: ${{ env.ARTIFACTS_PATH }} test/IntegrationTest/results/ retention-days: 14 + + - name: Upload captured corpus + if: ${{ always() && inputs.capture }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: corpus-run-ios-${{ inputs.unity-version }}-${{ inputs.ios-version }}-${{ inputs.init-type }} + path: test/IntegrationTest/capture/ + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/test-run-webgl.yml b/.github/workflows/test-run-webgl.yml index cff15cb81..38c0fd204 100644 --- a/.github/workflows/test-run-webgl.yml +++ b/.github/workflows/test-run-webgl.yml @@ -5,6 +5,11 @@ on: unity-version: required: true type: string + capture: + required: false + type: boolean + default: false + description: "Capture the envelopes and debug files this job produces - see docs/envelope-capture.md" secrets: SENTRY_AUTH_TOKEN: required: true @@ -22,6 +27,8 @@ jobs: env: SENTRY_DSN: ${{ secrets.SENTRY_TEST_DSN }} SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + # Empty unless a capture run was dispatched; see docs/envelope-capture.md. + SENTRY_CAPTURE_PATH: ${{ inputs.capture && format('test/IntegrationTest/capture/run-webgl-{0}', inputs.unity-version) || '' }} steps: - name: Checkout @@ -59,3 +66,12 @@ jobs: path: | test/IntegrationTest/results/ retention-days: 14 + + - name: Upload captured corpus + if: ${{ always() && inputs.capture }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: corpus-run-webgl-${{ inputs.unity-version }} + path: test/IntegrationTest/capture/ + if-no-files-found: warn + retention-days: 14 diff --git a/docs/envelope-capture.md b/docs/envelope-capture.md new file mode 100644 index 000000000..af290a556 --- /dev/null +++ b/docs/envelope-capture.md @@ -0,0 +1,139 @@ +# Capturing an envelope + debug file corpus + +The integration tests exercise every managed error, native crash and app hang path the SDK has, on +every platform we ship. Capture mode records what those runs actually put on the wire - raw +envelopes, minidump uploads, and the debug files, source bundles and IL2CPP line mappings +sentry-cli uploads - so the whole lot can be replayed against a local Sentry to work on event +processing or symbolication. + +It is **off by default** and changes nothing about a normal CI run. + +## Running it + +Actions → **CI** → *Run workflow* → tick **capture-corpus**. + +Every build and run job then writes its corpus to an artifact: + +```bash +gh run download -p 'corpus-*' -D ./corpus +``` + +| Artifact | Contains | +|---|---| +| `corpus--` | debug files, source bundles and IL2CPP line mappings sentry-cli uploaded for that build | +| `corpus-run---` | the envelopes and minidump uploads that run produced | + +Debug files are large (IL2CPP `GameAssembly.pdb` and friends run to hundreds of MB per platform), so +they stay per-job rather than being merged into one download. + +Everything sentry-cli uploaded lands in `debug-files/`, named +`--` and suffixed by kind: + +| Suffix | Kind | | +|---|---|---| +| *(none)* | `debug-file` | the dSYM / PDB / ELF itself | +| `.src` | `source-bundle` | the sources, from `--include-sources` | +| `.il2cpp.json` | `il2cpp-line-mapping` | C++ → C# line mapping, from `--il2cpp-mapping` | +| `.proguard` | `proguard-mapping` | Android `mapping.txt`, from `upload-proguard` | + +`debug-files/index.jsonl` records the `kind` alongside the assemble request, and the server prints +a per-kind tally when it shuts down. + +**The event assertions fail by design in a capture run.** There is no backend to verify against, so +`Integration.Tests.ps1` skips the Sentry API lookups and every event assertion fails. The artifacts +are the deliverable; a red run is expected. + +Coverage per matrix entry: `message-capture`, `exception-capture`, `crash-capture` (+ the +`crash-send` relaunch that flushes the crash envelope) and `app-hang-capture`, each of which also +emits logs, metrics, sessions and a transaction. Windows/macOS/Linux run twice, once per crash +backend (`crashpad`/`breakpad`/`native`/`cocoa`), so the corpus covers each native payload shape. + +## Replaying into a local Sentry + +```bash +# events, crashes, sessions, logs +python3 scripts/replay-envelopes.py ./corpus --dsn http://@localhost:9000/1 + +# the debug files that symbolicate them +sentry-cli --url http://localhost:9000 --auth-token debug-files upload \ + -o -p ./corpus/corpus-macos-6000.5/macos-6000.5/debug-files +``` + +Each envelope is rewritten before it is posted: the DSN in the envelope header is swapped for the +target, `sent_at` is set to now, event ids are regenerated and all timestamps are shifted to now +while keeping their relative offsets (breadcrumbs, spans, session start). That keeps a corpus +replayable indefinitely without deduplicating against itself or falling outside the ingest window. +Pass `--keep-ids` / `--keep-timestamps` to replay the bytes as they were captured. + +Minidump uploads are replayed verbatim to `/api//minidump/` with only the ingest key +swapped - the event ids inside the multipart body are left alone. + +`debug-files upload` re-uploads the difs and source bundles, but **not** the `.il2cpp.json` +mappings: sentry-cli only picks up files it recognises as difs, and it recomputes mappings from the +generated C++ next to the object rather than from a mapping file. The C++ is not in the corpus, so +the captured `.il2cpp.json` is the only copy - read it directly, or POST it to the chunk-upload and +`files/difs/assemble/` endpoints the way sentry-cli does. + +## How it works + +Everything keys off one environment variable, **`SENTRY_CAPTURE_PATH`**. The workflows set it from +the `capture` input; unset, every hook below is a no-op. + +| Piece | Role | +|---|---| +| [`capture-corpus.ps1`](../test/Scripts.Integration.Test/capture-corpus.ps1) | `Test-CaptureEnabled` / `Start-CaptureServer` / `Set-CaptureLabel`, and the capture DSN and port | +| [`envelope-capture-server.py`](../test/Scripts.Integration.Test/envelope-capture-server.py) | Stands in for both Sentry endpoints: envelope ingest, and the chunk-upload API sentry-cli uses for debug files | +| [`replay-envelopes.py`](../scripts/replay-envelopes.py) | Posts a captured corpus to a DSN of your choice | +| `configure-sentry.ps1` | Bakes the capture DSN into the test app | +| `build-project.ps1`, `compile-xcode-project.ps1` | Start the server and point sentry-cli at it for the build | +| `Integration.Tests.ps1` | Starts the server for the test run, labels each action, skips API verification | +| `ci-docker.sh` | In capture mode only, shares the host network so the in-container sentry-cli can reach the server | + +Details worth knowing if any of this regresses: + +- The server is started **inside** the build or test step that needs it, never in a step of its own: + a server started earlier does not survive the gap - the runner leaves it suspended, holding the + port without answering, which surfaces as "Empty reply from server". `Start-CaptureServer` clears + such a leftover before binding, and is safe to call repeatedly within a job. +- `SENTRY_URL` is what redirects sentry-cli. **sentry-cli 3.x ignores `defaults.url` in + `sentry.properties`**, which is the only knob the SDK offers + ([`SentryCli.UrlOverride`](../src/Sentry.Unity.Editor/SentryCli.cs)) - so without it, symbol upload + silently goes to sentry.io and the build still reports success. That also means self-hosted users + currently upload their symbols to sentry.io; worth fixing upstream. +- The iOS Xcode phase additionally needs `SENTRY_AUTH_TOKEN` from the environment, because sentry-cli + refuses to combine a URL from the environment with a token from `sentry.properties`. +- IL2CPP line mappings need no wiring of their own: `--il2cpp-mapping` is part of the same + `debug-files upload` the difs go through ([`BuildPostProcess`](../src/Sentry.Unity.Editor/Native/BuildPostProcess.cs), + [`DebugSymbolUpload`](../src/Sentry.Unity.Editor/Android/DebugSymbolUpload.cs), + [`SentryXcodeProject`](../src/Sentry.Unity.Editor.iOS/SentryXcodeProject.cs)), and they are chunked + and assembled like everything else. They are told apart **by content**: assemble carries only a + name, a debug id and the chunks, and a mapping inherits name and debug id from the object it was + computed from, so only the payload distinguishes them (`SYSB` magic, or a leading `{` for the + mapping JSON). +- Android proguard mappings ride the same path despite being a separate `upload-proguard` + invocation: sentry-cli chunk-uploads them and assembles them through `files/difs/assemble/` like + everything else. They are the one kind identifiable by metadata - sentry-cli names them + `/proguard/.txt` and sends no debug id, hence the `unknown-` prefix in the corpus. +- A proguard mapping only exists when minification is on: `sentryUploadProguardMapping` is + registered from [`AndroidUtils.ShouldUploadMapping`](../src/Sentry.Unity.Editor/Android/AndroidUtils.cs), + which reads `PlayerSettings.Android.minifyRelease` (release, because the test builds set + `EditorUserBuildSettings.development = false`). The integration test turns both minify flags on + in [`Builder.cs`](../test/Scripts.Integration.Test/Editor/Builder.cs), so an Android capture run + is expected to show a `proguard-mapping` in the tally. If it does not, check that flag first. +- A capture run where the tally shows difs but no `il2cpp-line-mapping` means IL2CPP line numbers + regressed upstream of the upload - either `--emit-source-mapping` never reached il2cpp + ([`Il2CppBuildPreProcess`](../src/Sentry.Unity.Editor/Il2CppBuildPreProcess.cs), gated on + `Il2CppLineNumberSupportEnabled`), or the generated C++ was gone by the time sentry-cli ran, since + it reads the `source_info` comments back out of those files. +- Capture listens on **8787**; `webgl-server.py` already serves the WebGL build on 8000. +- macOS ATS blocks plain HTTP to an IP literal, so the test app's `Info.plist` gets + `NSAllowsArbitraryLoads` ([`AllowInsecureHttp.cs`](../test/Scripts.Integration.Test/Editor/AllowInsecureHttp.cs)). + +## Capturing locally + +```bash +SENTRY_CAPTURE_PATH=$PWD/corpus/macos \ + ./test/Scripts.Integration.Test/dev-integration-test.ps1 -UnityVersion 6000.5 -Platform MacOS +``` + +The same hooks apply, so a local run produces the same corpus layout as CI. diff --git a/scripts/ci-docker.sh b/scripts/ci-docker.sh index a9330c0b1..ebbeb511c 100755 --- a/scripts/ci-docker.sh +++ b/scripts/ci-docker.sh @@ -31,10 +31,19 @@ uniqueHostname="${GITHUB_JOB:-local}-${imageVariant}-${GITHUB_RUN_ID:-0}" # Sanitize hostname: replace underscores and spaces with hyphens, ensure lowercase uniqueHostname=$(echo "$uniqueHostname" | tr '[:upper:]_ ' '[:lower:]--' | tr -s '-') +# Capture mode (see docs/envelope-capture.md): sentry-cli runs inside this container but the capture +# server runs on the host, so the container shares the host network to reach it. `--hostname` and +# `--network host` are mutually exclusive, hence the either/or. Port matches capture-corpus.ps1. +if [ -n "${SENTRY_CAPTURE_PATH:-}" ]; then + networkArgs=(--network host -e SENTRY_URL="http://127.0.0.1:8787" -e SENTRY_CAPTURE_PATH="${SENTRY_CAPTURE_PATH}") +else + networkArgs=(--hostname "$uniqueHostname") +fi + # We use the host dotnet installation - it's much faster than installing inside the docker container. set -x docker run -td --name $container \ - --hostname $uniqueHostname \ + "${networkArgs[@]}" \ --user $uid:$gid \ -v "$cwd":/sentry-unity \ -v $ANDROID_HOME:$ANDROID_HOME \ diff --git a/scripts/compile-xcode-project.ps1 b/scripts/compile-xcode-project.ps1 index 891117b0b..5fa37e564 100644 --- a/scripts/compile-xcode-project.ps1 +++ b/scripts/compile-xcode-project.ps1 @@ -3,6 +3,16 @@ param ( ) . $PSScriptRoot/../test/Scripts.Integration.Test/common.ps1 +. $PSScriptRoot/../test/Scripts.Integration.Test/capture-corpus.ps1 + +# Capture mode: sentry-cli refuses to combine a URL from the environment with the auth token +# baked into sentry.properties, so both come from here. The capture server ignores the token. +if (Test-CaptureEnabled) +{ + Start-CaptureServer + $env:SENTRY_URL = $Global:CaptureUrl + $env:SENTRY_AUTH_TOKEN = "capture-mode" +} $ProjectName = "Unity-iPhone" $repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path diff --git a/scripts/replay-envelopes.py b/scripts/replay-envelopes.py new file mode 100644 index 000000000..9ab2141a3 --- /dev/null +++ b/scripts/replay-envelopes.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +"""Replays a captured envelope corpus into a Sentry instance. + +Takes the output of test/Scripts.Integration.Test/envelope-capture-server.py (envelopes and +crashpad minidump uploads produced by the Unity integration tests on every platform) and posts +it to the DSN of your choice - typically a local Sentry. + +By default every replay gets fresh event ids and timestamps shifted to now, so the same corpus +can be replayed repeatedly without events deduplicating or falling outside the ingest window. + +Usage: + replay-envelopes.py --dsn http://@localhost:9000/1 + replay-envelopes.py --dsn ... --include '*crash*' --dry-run +""" + +import argparse +import fnmatch +import json +import sys +import urllib.error +import urllib.request +import uuid +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import parse_qsl, urlencode, urlparse + +TIMESTAMP_KEYS = {"timestamp", "start_timestamp", "started", "received", "time"} +JSON_ITEM_TYPES = {"event", "transaction", "session", "sessions", "check_in", "log", "feedback", + "user_report", "replay_event", "profile", "client_report"} + + +def parse_envelope(data): + """Splits envelope bytes into (header, [(item_header, payload)]).""" + newline = data.find(b"\n") + if newline == -1: + raise ValueError("no envelope header") + header = json.loads(data[:newline]) + items = [] + pos = newline + 1 + while pos < len(data): + if data[pos:pos + 1] == b"\n": + pos += 1 + continue + newline = data.find(b"\n", pos) + if newline == -1: + break + item_header = json.loads(data[pos:newline]) + pos = newline + 1 + if "length" in item_header: + end = pos + int(item_header["length"]) + else: + end = data.find(b"\n", pos) + if end == -1: + end = len(data) + items.append((item_header, data[pos:end])) + pos = end + return header, items + + +def serialize_envelope(header, items): + out = [json.dumps(header, separators=(",", ":")).encode(), b"\n"] + for item_header, payload in items: + item_header = dict(item_header, length=len(payload)) + out += [json.dumps(item_header, separators=(",", ":")).encode(), b"\n", payload, b"\n"] + return b"".join(out) + + +def to_epoch(value): + if isinstance(value, (int, float)): + return float(value) + if isinstance(value, str): + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() + except ValueError: + return None + return None + + +def from_epoch(epoch, template): + if isinstance(template, (int, float)): + return epoch + return datetime.fromtimestamp(epoch, timezone.utc).isoformat().replace("+00:00", "Z") + + +def collect_timestamps(node, found): + if isinstance(node, dict): + for key, value in node.items(): + if key in TIMESTAMP_KEYS: + epoch = to_epoch(value) + if epoch: + found.append(epoch) + collect_timestamps(value, found) + elif isinstance(node, list): + for value in node: + collect_timestamps(value, found) + + +def shift_timestamps(node, delta): + if isinstance(node, dict): + for key, value in node.items(): + if key in TIMESTAMP_KEYS: + epoch = to_epoch(value) + if epoch: + node[key] = from_epoch(epoch + delta, value) + continue + shift_timestamps(value, delta) + elif isinstance(node, list): + for value in node: + shift_timestamps(value, delta) + + +def rewrite(data, dsn, new_ids, fresh_timestamps): + header, items = parse_envelope(data) + header["dsn"] = dsn.url + header["sent_at"] = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + parsed = [] + for item_header, payload in items: + if item_header.get("type") in JSON_ITEM_TYPES: + try: + parsed.append((item_header, json.loads(payload), True)) + continue + except (ValueError, UnicodeDecodeError): + pass + parsed.append((item_header, payload, False)) + + if fresh_timestamps: + found = [] + for _, payload, is_json in parsed: + if is_json: + collect_timestamps(payload, found) + if found: + delta = datetime.now(timezone.utc).timestamp() - max(found) + for _, payload, is_json in parsed: + if is_json: + shift_timestamps(payload, delta) + + if new_ids: + event_id = uuid.uuid4().hex + if "event_id" in header: + header["event_id"] = event_id + for _, payload, is_json in parsed: + if is_json and isinstance(payload, dict) and "event_id" in payload: + payload["event_id"] = event_id + + rebuilt = [ + (item_header, json.dumps(payload, separators=(",", ":")).encode() if is_json else payload) + for item_header, payload, is_json in parsed + ] + return serialize_envelope(header, rebuilt) + + +class Dsn: + def __init__(self, url): + parsed = urlparse(url) + if not parsed.username or not parsed.hostname or len(parsed.path) < 2: + raise ValueError(f"not a valid DSN: {url}") + self.url = url + self.key = parsed.username + self.project = parsed.path.strip("/") + port = f":{parsed.port}" if parsed.port else "" + self.base = f"{parsed.scheme}://{parsed.hostname}{port}/api/{self.project}" + + def endpoint(self, name): + return f"{self.base}/{name}/" + + +def post(url, body, content_type, dsn, timeout): + auth = f"Sentry sentry_version=7, sentry_client=replay-envelopes/1.0, sentry_key={dsn.key}" + request = urllib.request.Request( + url, data=body, method="POST", + headers={"Content-Type": content_type, "X-Sentry-Auth": auth}) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return response.status, response.read(200).decode("utf-8", "replace") + except urllib.error.HTTPError as error: + return error.code, error.read(400).decode("utf-8", "replace") + except urllib.error.URLError as error: + return None, str(error.reason) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("corpus", help="directory with captured .envelope / .multipart.bin files") + parser.add_argument("--dsn", required=True, help="target DSN, e.g. http://key@localhost:9000/1") + parser.add_argument("--include", default="*", help="glob filter on the file name") + parser.add_argument("--keep-ids", action="store_true", help="replay original event ids") + parser.add_argument("--keep-timestamps", action="store_true", help="do not shift timestamps to now") + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--timeout", type=float, default=30) + args = parser.parse_args() + + dsn = Dsn(args.dsn) + corpus = Path(args.corpus) + files = sorted(path for path in corpus.rglob("*") + if path.suffix in (".envelope", ".bin") and fnmatch.fnmatch(path.name, args.include)) + if not files: + print(f"no envelopes matching '{args.include}' under {corpus}", file=sys.stderr) + return 1 + + failures = 0 + for path in files: + data = path.read_bytes() + + if path.name.endswith(".multipart.bin"): + # crashpad minidump upload - replayed verbatim, only the ingest key is swapped + meta = json.loads(path.with_name(path.name[:-len(".multipart.bin")] + ".meta.json").read_text()) + content_type = meta["headers"].get("Content-Type", "multipart/form-data") + query = dict(parse_qsl(meta.get("query", ""))) + query["sentry_key"] = dsn.key + url = f"{dsn.endpoint('minidump')}?{urlencode(query)}" + else: + try: + data = rewrite(data, dsn, not args.keep_ids, not args.keep_timestamps) + except Exception as error: + print(f"SKIP {path.name}: cannot rewrite ({error})", file=sys.stderr) + failures += 1 + continue + content_type = "application/x-sentry-envelope" + url = dsn.endpoint("envelope") + + if args.dry_run: + print(f"DRY {path.name} -> {url} ({len(data)} bytes)") + continue + + status, body = post(url, data, content_type, dsn, args.timeout) + ok = status is not None and 200 <= status < 300 + failures += 0 if ok else 1 + print(f"{'OK ' if ok else 'FAIL'} {status if status else 'ERR'} {path.name} {body.strip()[:120]}") + + print(f"\n{len(files) - failures}/{len(files)} replayed to {dsn.base}") + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/IntegrationTest/Integration.Tests.ps1 b/test/IntegrationTest/Integration.Tests.ps1 index 9dec4f8eb..602579cd7 100644 --- a/test/IntegrationTest/Integration.Tests.ps1 +++ b/test/IntegrationTest/Integration.Tests.ps1 @@ -24,6 +24,11 @@ $ErrorActionPreference = "Stop" . $PSScriptRoot/CommonTestCases.ps1 BeforeAll { + # Opt-in capture (no-op unless SENTRY_CAPTURE_PATH is set). Dot-sourced here rather than at + # script level because Pester runs this block in a scope that does not see script-level + # functions. + . $PSScriptRoot/../Scripts.Integration.Test/capture-corpus.ps1 + # Build app arguments for a given test action function Get-AppArguments { param([string]$Action) @@ -140,6 +145,7 @@ BeforeAll { ) Write-Host "Running $Action..." + Set-CaptureLabel -Label $Action if ($script:Platform -eq "WebGL") { return Invoke-WebGLTestAction -Action $Action @@ -160,6 +166,7 @@ BeforeAll { # Launch app again to ensure crash report is sent if ($Action -eq "crash-capture") { Write-Host "Running crash-send to ensure crash report is sent..." + Set-CaptureLabel -Label "crash-send" $sendArgs = Get-AppArguments -Action "crash-send" $sendResult = Invoke-DeviceApp -ExecutablePath $script:ExecutablePath -Arguments $sendArgs @@ -197,7 +204,18 @@ BeforeAll { if ([string]::IsNullOrEmpty($env:SENTRY_DSN)) { throw "SENTRY_DSN environment variable is not set." } - if ([string]::IsNullOrEmpty($env:SENTRY_AUTH_TOKEN)) { + + # Capture mode: the app sends its envelopes to the local capture server instead of Sentry, so the + # test actions still run (and their payloads get recorded) but there is no backend to verify + # against. The event assertions below fail by design in this mode - the corpus is the artifact. + # The server is started here rather than in a workflow step: one started earlier does not + # reliably survive the gap between steps. + if (Test-CaptureEnabled) { + Write-Host "Capture mode: recording the corpus, skipping Sentry API verification." -ForegroundColor Yellow + Start-CaptureServer + } + + if (-not (Test-CaptureEnabled) -and [string]::IsNullOrEmpty($env:SENTRY_AUTH_TOKEN)) { throw "SENTRY_AUTH_TOKEN environment variable is not set." } if ([string]::IsNullOrEmpty($env:SENTRY_TEST_APP)) { @@ -214,6 +232,7 @@ BeforeAll { Connect-Device -Platform "Adb" Install-DeviceApp -Path $env:SENTRY_TEST_APP + Connect-CaptureToDevice # Detect the launcher activity from the installed package $dumpOutput = & adb shell dumpsys package $script:PackageName 2>&1 | Out-String @@ -277,14 +296,21 @@ BeforeAll { AuthToken = $env:SENTRY_AUTH_TOKEN } - Connect-SentryApi ` - -ApiToken $script:TestSetup.AuthToken ` - -DSN $script:TestSetup.Dsn + if (-not (Test-CaptureEnabled)) { + Connect-SentryApi ` + -ApiToken $script:TestSetup.AuthToken ` + -DSN $script:TestSetup.Dsn + } } AfterAll { - Disconnect-SentryApi + . $PSScriptRoot/../Scripts.Integration.Test/capture-corpus.ps1 + Stop-CaptureServer + + if (-not (Test-CaptureEnabled)) { + Disconnect-SentryApi + } if ($script:Platform -ne "WebGL") { Disconnect-Device } @@ -299,7 +325,7 @@ Describe "Unity $($env:SENTRY_TEST_PLATFORM) Integration Tests" { $script:runResult = Invoke-TestAction -Action "message-capture" $eventId = Get-EventIds -AppOutput $script:runResult.Output -ExpectedCount 1 - if ($eventId) { + if ($eventId -and -not (Test-CaptureEnabled)) { Write-Host "::group::Getting event content" $script:runEvent = Get-SentryTestEvent -EventId "$eventId" Write-Host "::endgroup::" @@ -325,7 +351,7 @@ Describe "Unity $($env:SENTRY_TEST_PLATFORM) Integration Tests" { $script:runResult = Invoke-TestAction -Action "exception-capture" $eventId = Get-EventIds -AppOutput $script:runResult.Output -ExpectedCount 1 - if ($eventId) { + if ($eventId -and -not (Test-CaptureEnabled)) { Write-Host "::group::Getting event content" $script:runEvent = Get-SentryTestEvent -EventId "$eventId" Write-Host "::endgroup::" @@ -387,7 +413,7 @@ if ($env:SENTRY_TEST_PLATFORM -ne "WebGL") { } $eventId = Get-EventIds -AppOutput $script:runResult.Output -ExpectedCount 1 - if ($eventId) { + if ($eventId -and -not (Test-CaptureEnabled)) { Write-Host "::group::Getting event content" $script:runEvent = Get-SentryTestEvent -TagName "test.crash_id" -TagValue "$eventId" -TimeoutSeconds 300 Write-Host "::endgroup::" @@ -443,7 +469,7 @@ if ($env:SENTRY_TEST_PLATFORM -in "Desktop", "Android" -and -not $isCocoaBackend # The native app-hang event is captured in-proc (same run, no relaunch). Its event ID # is generated natively, so look it up by the unique scope tag the app sets instead. $hangId = Get-EventIds -AppOutput $script:runResult.Output -ExpectedCount 1 - if ($hangId) { + if ($hangId -and -not (Test-CaptureEnabled)) { Write-Host "::group::Getting event content" $script:runEvent = Get-SentryTestEvent -TagName "test.app_hang_id" -TagValue "$hangId" -TimeoutSeconds 300 Write-Host "::endgroup::" diff --git a/test/Scripts.Integration.Test/Editor/AllowInsecureHttp.cs b/test/Scripts.Integration.Test/Editor/AllowInsecureHttp.cs index 4482aada7..d2e687617 100644 --- a/test/Scripts.Integration.Test/Editor/AllowInsecureHttp.cs +++ b/test/Scripts.Integration.Test/Editor/AllowInsecureHttp.cs @@ -1,6 +1,8 @@ using System; using System.IO; using System.Reflection; +using System.Text.RegularExpressions; +using System.Xml; using UnityEditor; using UnityEditor.Build; using UnityEditor.Build.Reporting; @@ -21,6 +23,14 @@ public void OnPreprocessBuild(BuildReport report) public void OnPostprocessBuild(BuildReport report) { var pathToBuiltProject = report.summary.outputPath; + if (report.summary.platform == BuildTarget.StandaloneOSX) + { + // ATS applies to macOS players too and blocks plain HTTP to an IP literal, which is what + // the envelope capture server is. The iOS module isn't available on macOS build agents, + // so patch the plist as plain XML instead of going through PlistDocument. + AllowArbitraryLoadsInMacPlist(Path.Combine(pathToBuiltProject, "Contents", "Info.plist")); + } + if (report.summary.platform == BuildTarget.iOS) { var plistPath = Path.Combine(pathToBuiltProject, "Info.plist"); @@ -51,4 +61,56 @@ public void OnPostprocessBuild(BuildReport report) File.WriteAllText(plistPath, contents); } } + + private static void AllowArbitraryLoadsInMacPlist(string plistPath) + { + if (!File.Exists(plistPath)) + { + Debug.LogError($"Failed to find the plist at {plistPath}."); + return; + } + + var document = new XmlDocument { XmlResolver = null }; + // Parse (not Ignore) keeps the DOCTYPE in the document; the null resolver keeps us from + // fetching the external DTD Apple references. + using (var reader = XmlReader.Create(plistPath, new XmlReaderSettings { DtdProcessing = DtdProcessing.Parse, XmlResolver = null })) + { + document.Load(reader); + } + + var root = document.SelectSingleNode("/plist/dict"); + if (root is null) + { + Debug.LogError("Failed to find the root in the plist."); + return; + } + + foreach (XmlNode child in root.ChildNodes) + { + if (child.Name == "key" && child.InnerText == "NSAppTransportSecurity") + { + Debug.Log("AllowInsecureHttp: plist already contains NSAppTransportSecurity, nothing to do."); + return; + } + } + + var key = document.CreateElement("key"); + key.InnerText = "NSAppTransportSecurity"; + var value = document.CreateElement("dict"); + var allowKey = document.CreateElement("key"); + allowKey.InnerText = "NSAllowsArbitraryLoads"; + value.AppendChild(allowKey); + value.AppendChild(document.CreateElement("true")); + + root.AppendChild(key); + root.AppendChild(value); + document.Save(plistPath); + + // XmlDocument serializes the DOCTYPE with an empty internal subset (`...PropertyList-1.0.dtd"[]>`) + // which Apple's plist parser rejects. Drop it again. + var patched = Regex.Replace(File.ReadAllText(plistPath), @"(\[]*)\[\]>", "$1>"); + File.WriteAllText(plistPath, patched); + + Debug.Log("AllowInsecureHttp: added NSAllowsArbitraryLoads to the macOS plist."); + } } diff --git a/test/Scripts.Integration.Test/build-project.ps1 b/test/Scripts.Integration.Test/build-project.ps1 index 35a287637..3ded48a89 100644 --- a/test/Scripts.Integration.Test/build-project.ps1 +++ b/test/Scripts.Integration.Test/build-project.ps1 @@ -11,6 +11,16 @@ if (-not $Global:NewProjectPathCache) } . $PSScriptRoot/common.ps1 +. $PSScriptRoot/capture-corpus.ps1 + +# Capture mode: sentry-cli uploads the debug files during the build, so the server has to be up +# for its duration. SENTRY_URL is what redirects it - sentry-cli 3.x ignores the `defaults.url` +# the SDK writes into sentry.properties. +if (Test-CaptureEnabled) +{ + Start-CaptureServer + $env:SENTRY_URL = $Global:CaptureUrl +} $unityPath = FormatUnityPath $UnityPath $buildMethod = BuildMethodFor $Platform diff --git a/test/Scripts.Integration.Test/capture-corpus.ps1 b/test/Scripts.Integration.Test/capture-corpus.ps1 new file mode 100644 index 000000000..cd1b89372 --- /dev/null +++ b/test/Scripts.Integration.Test/capture-corpus.ps1 @@ -0,0 +1,151 @@ +#!/usr/bin/env pwsh +# +# Opt-in capture of the raw envelopes and debug files the integration tests produce, so they can be +# replayed against a local Sentry. See docs/envelope-capture.md. +# +# Everything keys off one environment variable: when SENTRY_CAPTURE_PATH points at a directory, the +# integration scripts route the SDK (envelopes, at run time) and sentry-cli (debug files, at build +# time) to a local capture server writing into it. Unset, every function here is a no-op and the +# tests behave exactly as they always have. +# +# Dot-source this file to use it: +# . $PSScriptRoot/capture-corpus.ps1 + +$Global:CapturePort = 8787 +$Global:CaptureUrl = "http://127.0.0.1:$Global:CapturePort" +# The key is irrelevant - the capture server accepts anything - but the DSN has to parse. +$Global:CaptureDsn = "http://capture@127.0.0.1:$Global:CapturePort/1" + +function Test-CaptureEnabled +{ + return -not [string]::IsNullOrEmpty($env:SENTRY_CAPTURE_PATH) +} + +# Starts the capture server unless one is already serving. Call this from whatever step actually +# needs it: a server started in an earlier CI step does not reliably survive the gap - the runner +# leaves it suspended, holding the port without answering, which surfaces as an empty reply. +function Start-CaptureServer +{ + if (-not (Test-CaptureEnabled)) + { + return + } + + if (Test-CaptureServerHealthy) + { + Write-Host "Capture server already running on port $Global:CapturePort" + return + } + + Clear-CapturePort + + $python = if (Get-Command python3 -ErrorAction SilentlyContinue) { "python3" } else { "python" } + $server = Join-Path $PSScriptRoot "envelope-capture-server.py" + $output = $env:SENTRY_CAPTURE_PATH + + New-Item -ItemType Directory -Force -Path $output | Out-Null + Start-Process -FilePath $python ` + -ArgumentList @($server, "--output", $output, "--port", $Global:CapturePort, + "--platform", (Split-Path $output -Leaf)) ` + -RedirectStandardError (Join-Path $output "capture-server.log") -NoNewWindow + + for ($i = 1; $i -le 30; $i++) + { + if (Test-CaptureServerHealthy) + { + Write-Host "Capture server is up on port $Global:CapturePort (writing to $output)" + return + } + Start-Sleep -Seconds 1 + } + + Get-Content (Join-Path $output "capture-server.log") -ErrorAction SilentlyContinue | Write-Host + throw "Capture server did not come up on port $Global:CapturePort" +} + +# Android runs the app on a device or emulator, where 127.0.0.1 is the device itself. Tunnel the +# capture port back to this host so the SDK's envelopes reach the server. Safe to call repeatedly. +function Connect-CaptureToDevice +{ + if (-not (Test-CaptureEnabled)) + { + return + } + + & adb reverse "tcp:$Global:CapturePort" "tcp:$Global:CapturePort" 2>&1 | Write-Host + # `adb` is a native command; don't let its exit code become the calling step's. + $global:LASTEXITCODE = 0 +} + +# Tags the files captured next with the test action they belong to, so the corpus is browsable. +function Set-CaptureLabel +{ + param([Parameter(Mandatory = $true)][string] $Label) + + if (-not (Test-CaptureEnabled)) + { + return + } + + try + { + Invoke-WebRequest -Uri "$Global:CaptureUrl/MARK?label=$Label" -TimeoutSec 5 -UseBasicParsing | Out-Null + } + catch + { + Write-Host "Failed to mark capture label '$Label': $_" + } +} + +function Stop-CaptureServer +{ + if (-not (Test-CaptureEnabled)) + { + return + } + + try + { + Invoke-WebRequest -Uri "$Global:CaptureUrl/STOP" -TimeoutSec 5 -UseBasicParsing | Out-Null + } + catch + { + Write-Host "Capture server already gone" + } +} + +function Test-CaptureServerHealthy +{ + try + { + Invoke-WebRequest -Uri "$Global:CaptureUrl/HEALTH" -TimeoutSec 2 -UseBasicParsing | Out-Null + return $true + } + catch + { + return $false + } +} + +# A suspended server from an earlier step keeps the port bound, which would make the new one fail +# with "Address already in use". +function Clear-CapturePort +{ + if ($IsWindows) + { + Get-NetTCPConnection -LocalPort $Global:CapturePort -State Listen -ErrorAction SilentlyContinue | + ForEach-Object { Stop-Process -Id $_.OwningProcess -Force -ErrorAction SilentlyContinue } + } + else + { + foreach ($processId in (& lsof -ti "tcp:$Global:CapturePort" 2>$null)) + { + Write-Host "Killing stale listener on port $Global:CapturePort (pid $processId)" + & kill -9 $processId 2>$null + } + + # `lsof` exits 1 when nothing matches, which is the normal case here. Left alone that + # becomes the exit code of the whole calling step, failing a build that actually succeeded. + $global:LASTEXITCODE = 0 + } +} diff --git a/test/Scripts.Integration.Test/configure-sentry.ps1 b/test/Scripts.Integration.Test/configure-sentry.ps1 index 21c04aa26..b0c6e746f 100644 --- a/test/Scripts.Integration.Test/configure-sentry.ps1 +++ b/test/Scripts.Integration.Test/configure-sentry.ps1 @@ -9,6 +9,13 @@ if (-not $Global:NewProjectPathCache) } . $PSScriptRoot/common.ps1 +. $PSScriptRoot/capture-corpus.ps1 + +# Capture mode: the app sends its envelopes to the local capture server instead of Sentry. +if (Test-CaptureEnabled) +{ + $env:SENTRY_DSN = $Global:CaptureDsn +} $UnityPath = FormatUnityPath $UnityPath diff --git a/test/Scripts.Integration.Test/envelope-capture-server.py b/test/Scripts.Integration.Test/envelope-capture-server.py new file mode 100644 index 000000000..1485bd101 --- /dev/null +++ b/test/Scripts.Integration.Test/envelope-capture-server.py @@ -0,0 +1,393 @@ +#!/usr/bin/env python3 +"""Captures raw Sentry envelopes sent by the integration test app. + +Stands in for Sentry's ingest endpoint: accepts every request, writes the body to disk +and answers 200 so the SDK considers the payload delivered. Point the DSN of the test +build at this server (host 127.0.0.1) and the run produces a corpus of real envelopes - +including native crash envelopes and crashpad minidump uploads - that can be replayed +against a local Sentry via scripts/replay-envelopes.py. + +Usage: + envelope-capture-server.py --output DIR [--host 0.0.0.0] [--port 8787] [--platform NAME] + +Control endpoints: + GET /HEALTH 200 once the server is serving + GET /MARK?label=foo tags subsequently captured files with `foo` (the test action) + GET /STOP shuts the server down +""" + +import argparse +import gzip +import json +import re +import shutil +import sys +import tempfile +import threading +import traceback +import uuid +import zlib +from datetime import datetime, timezone +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import parse_qs, urlparse + +state_lock = threading.Lock() +sequence = 0 +label = "startup" +output_dir = Path(".") +chunk_dir = Path(".") +symbol_dir = Path(".") +platform_name = "unknown" +assembled = set() +kinds = {} + + +def parse_envelope(data): + """Splits envelope bytes into (header, [(item_header, payload)]).""" + newline = data.find(b"\n") + if newline == -1: + raise ValueError("no envelope header") + header = json.loads(data[:newline]) + items = [] + pos = newline + 1 + while pos < len(data): + if data[pos:pos + 1] == b"\n": + pos += 1 + continue + newline = data.find(b"\n", pos) + if newline == -1: + break + item_header = json.loads(data[pos:newline]) + pos = newline + 1 + if "length" in item_header: + end = pos + int(item_header["length"]) + else: + end = data.find(b"\n", pos) + if end == -1: + end = len(data) + items.append((item_header, data[pos:end])) + pos = end + return header, items + + +def parse_multipart(body, boundary): + """Yields (headers, payload) for each part of a multipart/form-data body.""" + for segment in body.split(b"--" + boundary): + if segment in (b"", b"--", b"--\r\n", b"\r\n"): + continue + segment = segment[2:] if segment.startswith(b"\r\n") else segment + head, _, payload = segment.partition(b"\r\n\r\n") + if payload.endswith(b"\r\n"): + payload = payload[:-2] + headers = {} + for line in head.decode("utf-8", "replace").splitlines(): + key, sep, value = line.partition(":") + if sep: + headers[key.strip().lower()] = value.strip() + yield headers, payload + + +def decode_body(body, encoding): + if not encoding: + return body + encoding = encoding.lower() + try: + if encoding == "gzip": + return gzip.decompress(body) + if encoding in ("deflate", "zlib"): + return zlib.decompress(body) + except Exception as error: + print(f"failed to decompress {encoding} body: {error}", file=sys.stderr) + return body + + +def safe(value): + return re.sub(r"[^A-Za-z0-9_.-]", "_", value)[:60] or "unknown" + + +def classify(name, magic): + """Names the kind of file being assembled, and the suffix that marks it in the corpus. + + Assemble tells us only name, debug id and chunks. `upload-proguard` announces itself in the + name, but a source bundle and an IL2CPP line mapping inherit both name and debug id from the + object they were computed from, so their kind has to come from the content. + """ + # sentry-cli assembles proguard mappings as `/proguard/.txt`, with no debug id. + if name.startswith("/proguard/"): + return "proguard-mapping", ".proguard" + if magic.startswith(b"SYSB"): + return "source-bundle", ".src" + # `--il2cpp-mapping` uploads the line mapping as a plain JSON object: + # {"": {"": {"": }}, "__debug-id__": {...}}. + # No debug file format starts with a brace, so that alone tells them apart. + if magic.startswith(b"{"): + return "il2cpp-line-mapping", ".il2cpp.json" + return "debug-file", "" + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, fmt, *args): + print(f"{self.address_string()} - {fmt % args}", file=sys.stderr) + + def cors(self): + self.send_header("Access-Control-Allow-Origin", "*") + self.send_header("Access-Control-Allow-Methods", "POST, GET, OPTIONS") + self.send_header("Access-Control-Allow-Headers", "*") + self.send_header("Access-Control-Max-Age", "86400") + + def respond(self, code, payload=b"", content_type="application/json"): + self.send_response(code) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(payload))) + # One request per connection. Keep-alive sockets that the server later drops surface as + # "the network connection was lost" in NSURLSession and cost us envelopes. + self.send_header("Connection", "close") + self.close_connection = True + self.cors() + self.end_headers() + if payload: + self.wfile.write(payload) + + def do_OPTIONS(self): + self.respond(200) + + def do_GET(self): + global label + url = urlparse(self.path) + if url.path == "/HEALTH": + self.respond(200, b'{"ok":true}') + elif url.path == "/MARK": + new_label = parse_qs(url.query).get("label", ["unlabeled"])[0] + with state_lock: + label = safe(new_label) + print(f"--- mark: {label} ---", file=sys.stderr) + self.respond(200, b'{"ok":true}') + elif url.path.endswith("/chunk-upload/"): + # sentry-cli asks what the server accepts before uploading debug files. Advertising + # uncompressed chunks keeps the upload handler trivial. + options = { + "url": f"http://{self.headers.get('Host', '127.0.0.1')}{url.path}", + "chunkSize": 8 * 1024 * 1024, + "chunksPerRequest": 64, + "maxFileSize": 2 * 1024 * 1024 * 1024, + "maxRequestSize": 32 * 1024 * 1024, + "concurrency": 1, + "hashAlgorithm": "sha1", + "compression": [], + "accept": ["debug_files", "sources", "pdbs", "portablepdbs", "il2cpp", + "bcsymbolmaps", "proguard"], + } + self.respond(200, json.dumps(options).encode()) + elif url.path == "/STOP": + self.respond(200, b'{"ok":true}') + threading.Thread(target=self.server.shutdown).start() + else: + self.respond(200, b"{}") + + def read_body(self): + if self.headers.get("Transfer-Encoding", "").lower() == "chunked": + chunks = [] + while True: + size = int(self.rfile.readline().split(b";")[0], 16) + if size == 0: + self.rfile.readline() + break + chunks.append(self.rfile.read(size)) + self.rfile.readline() + return b"".join(chunks) + return self.rfile.read(int(self.headers.get("Content-Length", 0))) + + def handle_chunk_upload(self, body): + """Stores each uploaded chunk under its sha1 so assemble can stitch the file back.""" + boundary = re.search(r"boundary=([^;]+)", self.headers.get("Content-Type", "")) + if not boundary: + self.respond(400, b'{"detail":"missing boundary"}') + return + + count = 0 + for headers, payload in parse_multipart(body, boundary.group(1).strip('"').encode()): + name = re.search(r'filename="([^"]*)"', headers.get("content-disposition", "")) + if not name: + continue + (chunk_dir / name.group(1)).write_bytes(payload) + count += 1 + + print(f"stored {count} chunks", file=sys.stderr) + self.respond(200, b"{}") + + def handle_assemble(self, body): + """Reassembles uploaded chunks into the debug files sentry-cli meant to upload.""" + try: + request = json.loads(body) + except ValueError as error: + self.respond(400, json.dumps({"detail": str(error)}).encode()) + return + + response = {} + for checksum, entry in request.items(): + requested_name = entry.get("name") or checksum + name = Path(requested_name).name + + # sentry-cli polls assemble until every file reports `ok`. Once assembled we drop the + # chunks, so answer from this set rather than re-checking them - otherwise the next + # poll reports the file as missing and sentry-cli fails the upload. + with state_lock: + if checksum in assembled: + response[checksum] = {"state": "ok", "missingChunks": [], "detail": None} + continue + + missing = [c for c in entry.get("chunks", []) if not (chunk_dir / c).exists()] + if missing: + response[checksum] = {"state": "not_found", "missingChunks": missing, "detail": None} + continue + + # A dif, its source bundle and its IL2CPP line mapping all share debug id and name, + # so the checksum keeps them from overwriting each other. + target = symbol_dir / f"{entry.get('debug_id', 'unknown')}-{checksum[:8]}-{safe(name)}" + with target.open("wb") as out: + for chunk in entry["chunks"]: + out.write((chunk_dir / chunk).read_bytes()) + # The handle has to be closed before renaming - Windows refuses to rename an open file. + with target.open("rb") as probe: + kind, suffix = classify(requested_name, probe.read(4)) + if suffix: + # replace(), not rename(): a second build re-uploads the same files and Windows + # refuses to rename onto an existing file. + target = target.replace(target.with_name(target.name + suffix)) + + # Chunks deliberately stay until shutdown: they are deduplicated by hash, so deleting + # them here breaks any other file that shares one and makes sentry-cli fail the upload + # with "Some uploaded files are now missing on the server". + print(f"assembled {kind} {target.name} ({target.stat().st_size} bytes)", file=sys.stderr) + + with state_lock: + assembled.add(checksum) + kinds[kind] = kinds.get(kind, 0) + 1 + with (symbol_dir / "index.jsonl").open("a") as index: + index.write(json.dumps({"file": target.name, "kind": kind, + "platform": platform_name, "checksum": checksum, + "size": target.stat().st_size, "request": entry}) + "\n") + + response[checksum] = {"state": "ok", "missingChunks": [], "detail": None} + + self.respond(200, json.dumps(response).encode()) + + def handle_one_request(self): + # An exception escaping a handler closes the connection with no response, which surfaces to + # sentry-cli as "Empty reply from server" and hides the real cause. Answer 500 instead. + try: + super().handle_one_request() + except Exception: + traceback.print_exc() + try: + self.respond(500, json.dumps({"detail": traceback.format_exc()}).encode()) + except Exception: + pass + + def do_POST(self): + global sequence + url = urlparse(self.path) + raw = self.read_body() + body = decode_body(raw, self.headers.get("Content-Encoding")) + + if url.path.endswith("/chunk-upload/"): + self.handle_chunk_upload(body) + return + if url.path.endswith("/assemble/"): + self.handle_assemble(body) + return + + with state_lock: + sequence += 1 + seq, current_label = sequence, label + + meta = { + "sequence": seq, + "label": current_label, + "platform": platform_name, + "received": datetime.now(timezone.utc).isoformat(), + "method": self.command, + "path": url.path, + "query": url.query, + "headers": dict(self.headers), + "raw_bytes": len(raw), + "decoded_bytes": len(body), + } + + content_type = self.headers.get("Content-Type", "") + event_id = None + if "multipart/form-data" in content_type: + # crashpad uploads the minidump to /api//minidump/ as multipart + extension = "multipart.bin" + kind = "minidump" + else: + extension = "envelope" + kind = "envelope" + try: + header, items = parse_envelope(body) + meta["envelope_header"] = header + meta["items"] = [ + { + "type": item_header.get("type"), + "length": len(payload), + "filename": item_header.get("filename"), + "content_type": item_header.get("content_type"), + } + for item_header, payload in items + ] + event_id = header.get("event_id") + types = [i.get("type") or "unknown" for i, _ in items] + if types: + kind = "+".join(dict.fromkeys(types)) + except Exception as error: + meta["parse_error"] = str(error) + + name = f"{seq:03d}-{safe(platform_name)}-{safe(current_label)}-{safe(kind)}" + (output_dir / f"{name}.{extension}").write_bytes(body) + (output_dir / f"{name}.meta.json").write_text(json.dumps(meta, indent=2)) + with state_lock: + with (output_dir / "index.jsonl").open("a") as index: + index.write(json.dumps({"file": f"{name}.{extension}", **meta}) + "\n") + + print(f"captured {name}.{extension} ({len(body)} bytes) {url.path}", file=sys.stderr) + self.respond(200, json.dumps({"id": event_id or uuid.uuid4().hex}).encode()) + + +def main(): + global output_dir, chunk_dir, symbol_dir, platform_name + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--host", default="0.0.0.0") + parser.add_argument("--port", type=int, default=8787) + parser.add_argument("--output", required=True) + parser.add_argument("--platform", default="unknown") + args = parser.parse_args() + + output_dir = Path(args.output) + output_dir.mkdir(parents=True, exist_ok=True) + platform_name = args.platform + + # Debug files sentry-cli uploads land next to the envelopes; the chunks they are stitched + # from are scratch and get cleaned up on shutdown. + symbol_dir = output_dir / "debug-files" + symbol_dir.mkdir(exist_ok=True) + chunk_dir = Path(tempfile.mkdtemp(prefix="sentry-chunks-")) + + server = ThreadingHTTPServer((args.host, args.port), Handler) + print(f"envelope capture listening on {args.host}:{args.port} -> {output_dir}", file=sys.stderr) + server.serve_forever() + shutil.rmtree(chunk_dir, ignore_errors=True) + print(f"envelope capture stopped after {sequence} requests", file=sys.stderr) + for kind, count in sorted(kinds.items()): + print(f" {kind}: {count}", file=sys.stderr) + # A build that uploaded difs but no mapping means the IL2CPP line numbers regressed: either + # `--emit-source-mapping` never reached il2cpp, or the generated C++ was gone by upload time. + if kinds and "il2cpp-line-mapping" not in kinds: + print(" WARNING: no IL2CPP line mappings were uploaded", file=sys.stderr) + + +if __name__ == "__main__": + main()