diff --git a/package.json b/package.json index e4c0a475..fe9465ee 100644 --- a/package.json +++ b/package.json @@ -104,7 +104,9 @@ "picomatch@4": "^4.0.4", "picomatch@3": "^3.0.2", "picomatch@2": "^2.3.2", - "tar@7": "^7.5.4" + "form-data@2": "^2.5.4", + "tar@6": "^7.5.19", + "tar@7": "^7.5.19" } }, "dependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 64107f4a..2774322d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -19,7 +19,9 @@ overrides: picomatch@4: ^4.0.4 picomatch@3: ^3.0.2 picomatch@2: ^2.3.2 - tar@7: ^7.5.4 + form-data@2: ^2.5.4 + tar@6: ^7.5.19 + tar@7: ^7.5.19 importers: @@ -5168,8 +5170,8 @@ packages: forever-agent@0.6.1: resolution: {integrity: sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==} - form-data@2.3.3: - resolution: {integrity: sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==} + form-data@2.5.6: + resolution: {integrity: sha512-Ogz/E85h9tlfJzpI6TuFpGcHZFhLrb9Gw8wq9v40CxSCPnv7ahKr6Xgtkn0KYCDQJ8DNn5VoMO8EXr9V5PadyA==} engines: {node: '>= 0.12'} form-data@3.0.4: @@ -6534,10 +6536,6 @@ packages: resolution: {integrity: sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==} engines: {node: '>=8'} - minipass@5.0.0: - resolution: {integrity: sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==} - engines: {node: '>=8'} - minipass@7.1.2: resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==} engines: {node: '>=16 || 14 >=14.17'} @@ -8120,13 +8118,8 @@ packages: resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} engines: {node: '>=6'} - tar@6.2.1: - resolution: {integrity: sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==} - engines: {node: '>=10'} - deprecated: Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me - - tar@7.5.16: - resolution: {integrity: sha512-56adEpPMouktRlBLXiaYFFzZ/3+JXa8P9n7WbR+ibIjtviN55mEaOkiysCnPnWm+7kkui1Dn8J9l+g6zV8731w==} + tar@7.5.22: + resolution: {integrity: sha512-MFO/QzvtAOmJbkhOaCTvbGcFN9L9b+JunIsDwaKljSOdcLMea3NJ1k9Usz/rjdfSXTq4dfzfeS7W4p4YOAAHeA==} engines: {node: '>=18'} temp-dir@2.0.0: @@ -10444,7 +10437,7 @@ snapshots: ora: 5.4.1 read-binary-file-arch: 1.0.6 semver: 7.8.5 - tar: 6.2.1 + tar: 7.5.22 yargs: 17.7.3 transitivePeerDependencies: - bluebird @@ -10464,7 +10457,7 @@ snapshots: ora: 5.4.1 read-binary-file-arch: 1.0.6 semver: 7.7.3 - tar: 6.2.1 + tar: 7.5.22 yargs: 17.7.2 transitivePeerDependencies: - supports-color @@ -10765,7 +10758,7 @@ snapshots: source-map-support: 0.5.21 stacktrace-parser: 0.1.11 structured-headers: 0.4.1 - tar: 6.2.1 + tar: 7.5.22 temp-dir: 2.0.0 tempy: 0.7.1 terminal-link: 2.1.1 @@ -12744,7 +12737,7 @@ snapshots: resedit: 1.7.2 sanitize-filename: 1.6.4 semver: 7.8.5 - tar: 6.2.1 + tar: 7.5.22 temp-file: 3.4.0 transitivePeerDependencies: - bluebird @@ -12784,7 +12777,7 @@ snapshots: plist: 3.1.0 resedit: 1.7.2 semver: 7.7.3 - tar: 6.2.1 + tar: 7.5.22 temp-file: 3.4.0 tiny-async-pool: 1.3.0 which: 5.0.0 @@ -13318,7 +13311,7 @@ snapshots: promise-inflight: 1.0.1 rimraf: 3.0.2 ssri: 9.0.1 - tar: 6.2.1 + tar: 7.5.22 unique-filename: 2.0.1 transitivePeerDependencies: - bluebird @@ -13335,7 +13328,7 @@ snapshots: minipass-pipeline: 1.2.4 p-map: 4.0.0 ssri: 10.0.6 - tar: 6.2.1 + tar: 7.5.22 unique-filename: 3.0.0 cacache@19.0.1: @@ -13350,7 +13343,7 @@ snapshots: minipass-pipeline: 1.2.4 p-map: 7.0.4 ssri: 12.0.0 - tar: 7.5.16 + tar: 7.5.22 unique-filename: 4.0.0 cacheable-lookup@5.0.4: {} @@ -14727,11 +14720,14 @@ snapshots: forever-agent@0.6.1: optional: true - form-data@2.3.3: + form-data@2.5.6: dependencies: asynckit: 0.4.0 combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 + hasown: 2.0.4 mime-types: 2.1.35 + safe-buffer: 5.2.1 optional: true form-data@3.0.4: @@ -16316,8 +16312,6 @@ snapshots: dependencies: yallist: 4.0.0 - minipass@5.0.0: {} - minipass@7.1.2: {} minizlib@2.1.2: @@ -16447,7 +16441,7 @@ snapshots: nopt: 8.1.0 proc-log: 5.0.0 semver: 7.8.5 - tar: 7.5.16 + tar: 7.5.22 tinyglobby: 0.2.15 which: 5.0.0 transitivePeerDependencies: @@ -16463,7 +16457,7 @@ snapshots: request: 2.88.2 rimraf: 3.0.2 semver: 7.8.5 - tar: 6.2.1 + tar: 7.5.22 which: 2.0.2 optional: true @@ -16478,7 +16472,7 @@ snapshots: npmlog: 6.0.2 rimraf: 3.0.2 semver: 7.8.5 - tar: 6.2.1 + tar: 7.5.22 which: 2.0.2 transitivePeerDependencies: - bluebird @@ -17313,7 +17307,7 @@ snapshots: combined-stream: 1.0.8 extend: 3.0.2 forever-agent: 0.6.1 - form-data: 2.3.3 + form-data: 2.5.6 har-validator: 5.1.5 http-signature: 1.2.0 is-typedarray: 1.0.0 @@ -18053,16 +18047,7 @@ snapshots: inherits: 2.0.4 readable-stream: 3.6.2 - tar@6.2.1: - dependencies: - chownr: 2.0.0 - fs-minipass: 2.1.0 - minipass: 5.0.0 - minizlib: 2.1.2 - mkdirp: 1.0.4 - yallist: 4.0.0 - - tar@7.5.16: + tar@7.5.22: dependencies: '@isaacs/fs-minipass': 4.0.1 chownr: 3.0.0 diff --git a/scripts/lib/package-managers/aur-submit.test.ts b/scripts/lib/package-managers/aur-submit.test.ts new file mode 100644 index 00000000..7c155095 --- /dev/null +++ b/scripts/lib/package-managers/aur-submit.test.ts @@ -0,0 +1,210 @@ +/** + * AUR submission resilience tests + * + * aur.archlinux.org goes into maintenance without notice. When it does, it + * still accepts the SSH connection but refuses the git operation, which used + * to fail the entire release even though every other package manager had + * already published. + * + * These live in their own file because they mock node:child_process, which the + * shared package-managers suite deliberately does not. + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { execSync } from 'node:child_process'; +import type { ReleaseInfo, Logger } from './types.js'; +import { AURPackageManager } from './aur.js'; + +vi.mock('node:child_process', () => ({ execSync: vi.fn() })); + +vi.mock('node:fs', () => ({ + writeFileSync: vi.fn(), + mkdirSync: vi.fn(), + rmSync: vi.fn(), + existsSync: vi.fn(() => false), +})); + +const mockExecSync = vi.mocked(execSync); +const mockFetch = vi.fn(); +global.fetch = mockFetch as unknown as typeof fetch; + +const MAINTENANCE_STDERR = + 'The AUR is down due to maintenance. We will be back soon.\n' + + 'fatal: Could not read from remote repository.'; + +const CONNECTION_CLOSED_STDERR = + 'Connection closed by 209.126.35.78 port 22\n' + 'fatal: Could not read from remote repository.'; + +function createMockLogger(): Logger { + return { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + success: vi.fn(), + debug: vi.fn(), + }; +} + +/** Mimic what execSync throws when stdio is piped: message hides the detail. */ +function execFailure(command: string, stderr: string): Error { + const error = new Error(`Command failed: ${command}`) as Error & { stderr: Buffer }; + error.stderr = Buffer.from(stderr); + return error; +} + +function createRelease(version = '1.0.0'): ReleaseInfo { + return { + version, + tagName: `v${version}`, + assets: [ + { + name: `PairUX-${version}-x86_64.AppImage`, + downloadUrl: `https://github.com/profullstack/pairux.com/releases/download/v${version}/PairUX-${version}-x86_64.AppImage`, + size: 95000000, + contentType: 'application/octet-stream', + sha256: 'JKL012ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF012345', + }, + ], + checksums: new Map(), + releaseUrl: 'https://github.com/profullstack/pairux.com/releases/tag/v1.0.0', + } as ReleaseInfo; +} + +/** Commands issued to git, in order, ignoring the temp-dir noise. */ +function gitCalls(): string[] { + return mockExecSync.mock.calls.map(([command]) => command); +} + +function callsMatching(fragment: string): string[] { + return gitCalls().filter((command) => command.includes(fragment)); +} + +describe('AURPackageManager.submit resilience', () => { + let aur: AURPackageManager; + let logger: Logger; + + beforeEach(() => { + vi.useFakeTimers(); + logger = createMockLogger(); + aur = new AURPackageManager({ enabled: true }, logger); + + mockExecSync.mockReset(); + mockExecSync.mockImplementation(() => Buffer.from('')); + + mockFetch.mockReset(); + // Package is not yet at this version, so submit proceeds. + mockFetch.mockResolvedValue({ + ok: true, + json: () => Promise.resolve({ results: [] }), + }); + + process.env.AUR_SSH_KEY = Buffer.from('fake-key').toString('base64'); + }); + + afterEach(() => { + vi.useRealTimers(); + delete process.env.AUR_SSH_KEY; + }); + + async function submit(release = createRelease()) { + const pending = aur.submit(release); + await vi.runAllTimersAsync(); + return pending; + } + + it('skips rather than fails when AUR is in maintenance', async () => { + mockExecSync.mockImplementation((command) => { + if (command.startsWith('git clone')) { + throw execFailure(command, MAINTENANCE_STDERR); + } + return Buffer.from(''); + }); + + const result = await submit(); + + expect(result.status).toBe('skipped'); + expect(result.message).toContain('AUR unreachable'); + // The release must not be reported as failed over a third-party outage. + expect(result.status).not.toBe('failed'); + }); + + it('does not mistake an outage for a package that does not exist yet', async () => { + mockExecSync.mockImplementation((command) => { + if (command.startsWith('git clone')) { + throw execFailure(command, MAINTENANCE_STDERR); + } + return Buffer.from(''); + }); + + await submit(); + + // Creating an empty repo here would have pushed a fresh history over the + // real package once AUR came back. + expect(callsMatching('git init')).toHaveLength(0); + expect(callsMatching('git push')).toHaveLength(0); + }); + + it('retries a dropped push and succeeds when AUR recovers', async () => { + let pushAttempts = 0; + mockExecSync.mockImplementation((command) => { + if (command.startsWith('git push')) { + pushAttempts += 1; + if (pushAttempts < 3) { + throw execFailure(command, CONNECTION_CLOSED_STDERR); + } + } + return Buffer.from(''); + }); + + const result = await submit(); + + expect(pushAttempts).toBe(3); + expect(result.status).toBe('success'); + expect(logger.warn).toHaveBeenCalled(); + }); + + it('gives up and skips after exhausting push retries', async () => { + mockExecSync.mockImplementation((command) => { + if (command.startsWith('git push')) { + throw execFailure(command, CONNECTION_CLOSED_STDERR); + } + return Buffer.from(''); + }); + + const result = await submit(createRelease('2.5.0')); + + expect(callsMatching('git push')).toHaveLength(3); + expect(result.status).toBe('skipped'); + expect(result.message).toContain('2.5.0'); + }); + + it('still fails hard on a real error, without retrying', async () => { + mockExecSync.mockImplementation((command) => { + if (command.startsWith('git push')) { + throw execFailure(command, 'Permission denied (publickey).'); + } + return Buffer.from(''); + }); + + const result = await submit(); + + // A bad key is our problem and must not be quietly skipped. + expect(result.status).toBe('failed'); + expect(callsMatching('git push')).toHaveLength(1); + expect(result.message).toContain('Permission denied'); + }); + + it('surfaces the stderr that execSync hides behind "Command failed"', async () => { + mockExecSync.mockImplementation((command) => { + if (command.startsWith('git push')) { + throw execFailure(command, 'Permission denied (publickey).'); + } + return Buffer.from(''); + }); + + const result = await submit(); + + expect(result.message).toContain('Command failed: git push origin master'); + expect(result.message).toContain('Permission denied (publickey).'); + }); +}); diff --git a/scripts/lib/package-managers/aur.ts b/scripts/lib/package-managers/aur.ts index 5b816f7b..663734fc 100644 --- a/scripts/lib/package-managers/aur.ts +++ b/scripts/lib/package-managers/aur.ts @@ -14,6 +14,59 @@ import type { ReleaseInfo, SubmissionResult } from './types.js'; const AUR_SSH_HOST = 'aur@aur.archlinux.org'; const PACKAGE_NAME = 'pairux-bin'; +/** How many times to try a git command that failed because AUR was unreachable. */ +const AUR_ATTEMPTS = 3; +/** Gap between those attempts. AUR maintenance windows are usually short. */ +const AUR_RETRY_DELAY_MS = 15_000; + +/** + * aur.archlinux.org drops into maintenance without notice, and when it does it + * still accepts the SSH connection but refuses the git operation. None of these + * say anything about our key or our package, so they must not be reported as a + * submission failure that fails the whole release. + */ +const TRANSIENT_AUR_ERRORS = [ + 'down due to maintenance', + 'connection closed by', + 'connection reset by peer', + 'connection timed out', + 'kex_exchange_identification', + 'the remote end hung up unexpectedly', + 'early eof', +]; + +function isTransientAURError(message: string): boolean { + const haystack = message.toLowerCase(); + return TRANSIENT_AUR_ERRORS.some((needle) => haystack.includes(needle)); +} + +/** + * execSync hides the interesting part in `stderr` when stdio is piped, so the + * bare message is only ever "Command failed: git push origin master". + */ +function describeCommandError(error: unknown): string { + if (!(error instanceof Error)) { + return typeof error === 'string' ? error : JSON.stringify(error); + } + + const { stderr } = error as Error & { stderr?: Buffer | string }; + const details = stderr?.toString().trim() ?? ''; + + return details && !error.message.includes(details) + ? `${error.message}\n${details}` + : error.message; +} + +class AURCommandError extends Error { + readonly transient: boolean; + + constructor(command: string, details: string) { + super(`Command failed: ${command}\n${details}`); + this.name = 'AURCommandError'; + this.transient = isTransientAURError(details); + } +} + export class AURPackageManager extends BasePackageManager { readonly name = 'aur'; readonly displayName = 'AUR'; @@ -147,6 +200,37 @@ pkgname = ${PACKAGE_NAME} `; } + private sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); + } + + /** + * Run a git command, retrying while AUR itself is the thing that is broken. + * Anything else (a bad key, a rejected ref) fails on the first attempt. + */ + private async runGit( + command: string, + options: { cwd?: string; env: NodeJS.ProcessEnv }, + attempts = AUR_ATTEMPTS + ): Promise { + for (let attempt = 1; ; attempt++) { + try { + execSync(command, { ...options, stdio: 'pipe' }); + return; + } catch (error) { + const failure = new AURCommandError(command, describeCommandError(error)); + + if (!failure.transient || attempt >= attempts) throw failure; + + this.logger.warn( + `AUR is unreachable (attempt ${String(attempt)}/${String(attempts)}), ` + + `retrying in ${String(AUR_RETRY_DELAY_MS / 1000)}s...` + ); + await this.sleep(AUR_RETRY_DELAY_MS); + } + } + } + async submit(release: ReleaseInfo, dryRun = false): Promise { // Check if already exists if (await this.checkExisting(release.version)) { @@ -213,12 +297,13 @@ pkgname = ${PACKAGE_NAME} // Clone the AUR repo this.logger.info('Cloning AUR repository...'); try { - execSync(`git clone ${AUR_SSH_HOST}:${PACKAGE_NAME}.git ${repoDir}`, { - env, - stdio: 'pipe', - }); - } catch { - // Package doesn't exist yet, create it + await this.runGit(`git clone ${AUR_SSH_HOST}:${PACKAGE_NAME}.git ${repoDir}`, { env }); + } catch (error) { + // Only treat a clone failure as "package doesn't exist yet" when AUR + // actually answered. If AUR is down, creating an empty repo here just + // turns an outage into a confusing push error further down. + if (error instanceof AURCommandError && error.transient) throw error; + this.logger.info('Creating new AUR package...'); mkdirSync(repoDir, { recursive: true }); execSync('git init -b master', { cwd: repoDir, env, stdio: 'pipe' }); @@ -269,7 +354,7 @@ pkgname = ${PACKAGE_NAME} // Push to AUR this.logger.info('Pushing to AUR...'); - execSync('git push origin master', { cwd: repoDir, env, stdio: 'pipe' }); + await this.runGit('git push origin master', { cwd: repoDir, env }); return { packageManager: this.name, @@ -278,6 +363,21 @@ pkgname = ${PACKAGE_NAME} }; } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); + + // AUR being down is not a release failure. Every other package manager + // has already published, so report it as skipped and let the release + // stand; the next release re-submits the version AUR missed. + if (error instanceof AURCommandError && error.transient) { + this.logger.warn(`AUR is unavailable, skipping submission: ${errorMessage}`); + return { + packageManager: this.name, + status: 'skipped', + message: + `AUR unreachable after ${String(AUR_ATTEMPTS)} attempts ` + + `(version ${release.version} not submitted): ${errorMessage}`, + }; + } + return { packageManager: this.name, status: 'failed',