From 94dfc7715d3dec57eface5a0acf146c8000f8734 Mon Sep 17 00:00:00 2001 From: Long Ho Date: Tue, 28 Jul 2026 09:49:34 -0400 Subject: [PATCH 1/6] ci: shard live functional tests --- .github/workflows/functional-tests.yml | 8 +- e2e/BUILD.bazel | 39 +++- e2e/live.spec.ts | 289 +++++++++++++------------ 3 files changed, 198 insertions(+), 138 deletions(-) diff --git a/.github/workflows/functional-tests.yml b/.github/workflows/functional-tests.yml index 69aea5d..7cbf942 100644 --- a/.github/workflows/functional-tests.yml +++ b/.github/workflows/functional-tests.yml @@ -47,4 +47,10 @@ jobs: - name: Run functional tests env: PPLX_API_TOKEN: ${{ secrets.PPLX_API_TOKEN }} - run: bazel test //e2e:live --test_env=PPLX_API_TOKEN + run: | + bazel test \ + //e2e:live_async \ + //e2e:live_fast \ + //e2e:live_responses \ + //e2e:live_streaming_responses \ + --test_env=PPLX_API_TOKEN diff --git a/e2e/BUILD.bazel b/e2e/BUILD.bazel index 65ebfac..34f9178 100644 --- a/e2e/BUILD.bazel +++ b/e2e/BUILD.bazel @@ -40,13 +40,50 @@ js_test( ) js_test( - name = "live", + name = "live_fast", data = [ ":typecheck", "//:node_modules/@perplexity-ai/perplexity_ai", "//:node_modules/published-sdk", ], entry_point = "live.spec.ts", + env = {"LIVE_TEST_SHARD": "fast"}, + tags = ["manual"], +) + +js_test( + name = "live_responses", + data = [ + ":typecheck", + "//:node_modules/@perplexity-ai/perplexity_ai", + "//:node_modules/published-sdk", + ], + entry_point = "live.spec.ts", + env = {"LIVE_TEST_SHARD": "responses"}, + tags = ["manual"], +) + +js_test( + name = "live_streaming_responses", + data = [ + ":typecheck", + "//:node_modules/@perplexity-ai/perplexity_ai", + "//:node_modules/published-sdk", + ], + entry_point = "live.spec.ts", + env = {"LIVE_TEST_SHARD": "streaming-responses"}, + tags = ["manual"], +) + +js_test( + name = "live_async", + data = [ + ":typecheck", + "//:node_modules/@perplexity-ai/perplexity_ai", + "//:node_modules/published-sdk", + ], + entry_point = "live.spec.ts", + env = {"LIVE_TEST_SHARD": "async"}, tags = ["manual"], ) diff --git a/e2e/live.spec.ts b/e2e/live.spec.ts index 025051e..251c871 100644 --- a/e2e/live.spec.ts +++ b/e2e/live.spec.ts @@ -264,146 +264,163 @@ function createClients() { }; } -describe('live API parity', () => { - it('matches published chat completion response contract', async () => { - const { candidate, published } = createClients(); - const request = { - max_tokens: 16, - messages: [{ content: 'Reply with only the word pong.', role: 'user' as const }], - model: 'sonar', - temperature: 0, - }; - - await expectMatchingContracts( - () => candidate.chat.completions.create(request), - () => published.chat.completions.create(request), - completionContract, - ); - }); - - it('matches published streaming chat contract', async () => { - const { candidate, published } = createClients(); - const request = { - max_tokens: 16, - messages: [{ content: 'Reply with only the word pong.', role: 'user' as const }], - model: 'sonar', - stream: true as const, - temperature: 0, - }; - - await expectMatchingContracts( - async () => collect(await candidate.chat.completions.create(request)), - async () => collect(await published.chat.completions.create(request)), - (chunks) => { - assert.ok(Array.isArray(chunks)); - return streamingCompletionContract(chunks); - }, - ); - }); - - it('matches published search response contract', async () => { - const { candidate, published } = createClients(); - const request = { - max_results: 1, - query: 'Perplexity AI', - }; - - await expectMatchingContracts( - () => candidate.search.create(request), - () => published.search.create(request), - searchContract, - ); - }); - - it('matches published embeddings response contract', async () => { - const { candidate, published } = createClients(); - const request = { - dimensions: 128, - input: 'Perplexity answers questions.', - model: 'pplx-embed-v1-0.6b' as const, - }; - - await expectMatchingContracts( - () => candidate.embeddings.create(request), - () => published.embeddings.create(request), - embeddingContract, - ); - }); - - it('matches published contextualized embeddings response contract', async () => { - const { candidate, published } = createClients(); - const request = { - dimensions: 128, - input: [['Perplexity answers questions.', 'Its answers include citations.']], - model: 'pplx-embed-context-v1-0.6b' as const, - }; - - await expectMatchingContracts( - () => candidate.contextualizedEmbeddings.create(request), - () => published.contextualizedEmbeddings.create(request), - contextualizedEmbeddingContract, - ); +const liveTestShard = process.env['LIVE_TEST_SHARD']; +const runsShard = (shard: string) => !liveTestShard || liveTestShard === shard; + +if (runsShard('fast')) { + describe('live API parity: fast APIs', () => { + it('matches published chat completion response contract', async () => { + const { candidate, published } = createClients(); + const request = { + max_tokens: 16, + messages: [{ content: 'Reply with only the word pong.', role: 'user' as const }], + model: 'sonar', + temperature: 0, + }; + + await expectMatchingContracts( + () => candidate.chat.completions.create(request), + () => published.chat.completions.create(request), + completionContract, + ); + }); + + it('matches published streaming chat contract', async () => { + const { candidate, published } = createClients(); + const request = { + max_tokens: 16, + messages: [{ content: 'Reply with only the word pong.', role: 'user' as const }], + model: 'sonar', + stream: true as const, + temperature: 0, + }; + + await expectMatchingContracts( + async () => collect(await candidate.chat.completions.create(request)), + async () => collect(await published.chat.completions.create(request)), + (chunks) => { + assert.ok(Array.isArray(chunks)); + return streamingCompletionContract(chunks); + }, + ); + }); + + it('matches published search response contract', async () => { + const { candidate, published } = createClients(); + const request = { + max_results: 1, + query: 'Perplexity AI', + }; + + await expectMatchingContracts( + () => candidate.search.create(request), + () => published.search.create(request), + searchContract, + ); + }); + + it('matches published embeddings response contract', async () => { + const { candidate, published } = createClients(); + const request = { + dimensions: 128, + input: 'Perplexity answers questions.', + model: 'pplx-embed-v1-0.6b' as const, + }; + + await expectMatchingContracts( + () => candidate.embeddings.create(request), + () => published.embeddings.create(request), + embeddingContract, + ); + }); + + it('matches published contextualized embeddings response contract', async () => { + const { candidate, published } = createClients(); + const request = { + dimensions: 128, + input: [['Perplexity answers questions.', 'Its answers include citations.']], + model: 'pplx-embed-context-v1-0.6b' as const, + }; + + await expectMatchingContracts( + () => candidate.contextualizedEmbeddings.create(request), + () => published.contextualizedEmbeddings.create(request), + contextualizedEmbeddingContract, + ); + }); }); +} - it('matches published responses API contract', async () => { - const { candidate, published } = createClients(); - const request = { - input: 'Reply with only the word pong.', - max_output_tokens: 16, - preset: 'pro-search', - }; - - await expectMatchingContracts( - () => candidate.responses.create(request), - () => published.responses.create(request), - responseContract, - ); +if (runsShard('responses')) { + describe('live API parity: Responses API', () => { + it('matches published responses API contract', async () => { + const { candidate, published } = createClients(); + const request = { + input: 'Reply with only the word pong.', + max_output_tokens: 16, + preset: 'pro-search', + }; + + await expectMatchingContracts( + () => candidate.responses.create(request), + () => published.responses.create(request), + responseContract, + ); + }); }); +} - it('matches published streaming responses API contract', async () => { - const { candidate, published } = createClients(); - const request = { - input: 'Reply with only the word pong.', - max_output_tokens: 16, - preset: 'pro-search', - stream: true as const, - }; - - await expectMatchingContracts( - async () => collect(await candidate.responses.create(request)), - async () => collect(await published.responses.create(request)), - (events) => { - assert.ok(Array.isArray(events)); - return streamingResponseContract(events); - }, - ); +if (runsShard('streaming-responses')) { + describe('live API parity: streaming Responses API', () => { + it('matches published streaming responses API contract', async () => { + const { candidate, published } = createClients(); + const request = { + input: 'Reply with only the word pong.', + max_output_tokens: 16, + preset: 'pro-search', + stream: true as const, + }; + + await expectMatchingContracts( + async () => collect(await candidate.responses.create(request)), + async () => collect(await published.responses.create(request)), + (events) => { + assert.ok(Array.isArray(events)); + return streamingResponseContract(events); + }, + ); + }); }); +} - it('matches published async chat completion lifecycle', { timeout: 180_000 }, async () => { - const { candidate, published } = createClients(); - const request = { - request: { - max_tokens: 16, - messages: [{ content: 'Reply with only the word pong.', role: 'user' as const }], - model: 'sonar-deep-research', - }, - }; - const [candidateInitial, publishedInitial] = await Promise.all([ - candidate.async.chat.completions.create(request), - published.async.chat.completions.create(request), - ]); - assert.deepStrictEqual( - asyncCompletionContract(candidateInitial), - asyncCompletionContract(publishedInitial), - ); - - const [candidateCompleted, publishedCompleted] = await Promise.all([ - pollAsyncCompletion(candidateInitial, (id) => candidate.async.chat.completions.get(id)), - pollAsyncCompletion(publishedInitial, (id) => published.async.chat.completions.get(id)), - ]); - assert.deepStrictEqual( - asyncCompletionContract(candidateCompleted), - asyncCompletionContract(publishedCompleted), - ); +if (runsShard('async')) { + describe('live API parity: async API', () => { + it('matches published async chat completion lifecycle', { timeout: 180_000 }, async () => { + const { candidate, published } = createClients(); + const request = { + request: { + max_tokens: 16, + messages: [{ content: 'Reply with only the word pong.', role: 'user' as const }], + model: 'sonar-deep-research', + }, + }; + const [candidateInitial, publishedInitial] = await Promise.all([ + candidate.async.chat.completions.create(request), + published.async.chat.completions.create(request), + ]); + assert.deepStrictEqual( + asyncCompletionContract(candidateInitial), + asyncCompletionContract(publishedInitial), + ); + + const [candidateCompleted, publishedCompleted] = await Promise.all([ + pollAsyncCompletion(candidateInitial, (id) => candidate.async.chat.completions.get(id)), + pollAsyncCompletion(publishedInitial, (id) => published.async.chat.completions.get(id)), + ]); + assert.deepStrictEqual( + asyncCompletionContract(candidateCompleted), + asyncCompletionContract(publishedCompleted), + ); + }); }); -}); +} From 9b1b0d41ba01c4ba68680dd6e3d24d5cb1115e71 Mon Sep 17 00:00:00 2001 From: Long Ho Date: Tue, 28 Jul 2026 10:02:01 -0400 Subject: [PATCH 2/6] refactor: let Bazel shard live tests --- .github/workflows/functional-tests.yml | 12 +- e2e/BUILD.bazel | 53 +----- e2e/live/BUILD.bazel | 147 ++++++++++++++++ e2e/live/async-chat-completion.spec.ts | 31 ++++ e2e/live/chat-completion.spec.ts | 18 ++ e2e/live/contextualized-embeddings.spec.ts | 17 ++ e2e/live/embeddings.spec.ts | 17 ++ e2e/{live.spec.ts => live/helpers.ts} | 186 ++------------------- e2e/live/responses.spec.ts | 17 ++ e2e/live/search.spec.ts | 16 ++ e2e/live/streaming-chat.spec.ts | 23 +++ e2e/live/streaming-responses.spec.ts | 22 +++ 12 files changed, 329 insertions(+), 230 deletions(-) create mode 100644 e2e/live/BUILD.bazel create mode 100644 e2e/live/async-chat-completion.spec.ts create mode 100644 e2e/live/chat-completion.spec.ts create mode 100644 e2e/live/contextualized-embeddings.spec.ts create mode 100644 e2e/live/embeddings.spec.ts rename e2e/{live.spec.ts => live/helpers.ts} (55%) create mode 100644 e2e/live/responses.spec.ts create mode 100644 e2e/live/search.spec.ts create mode 100644 e2e/live/streaming-chat.spec.ts create mode 100644 e2e/live/streaming-responses.spec.ts diff --git a/.github/workflows/functional-tests.yml b/.github/workflows/functional-tests.yml index 7cbf942..84e7f06 100644 --- a/.github/workflows/functional-tests.yml +++ b/.github/workflows/functional-tests.yml @@ -49,8 +49,12 @@ jobs: PPLX_API_TOKEN: ${{ secrets.PPLX_API_TOKEN }} run: | bazel test \ - //e2e:live_async \ - //e2e:live_fast \ - //e2e:live_responses \ - //e2e:live_streaming_responses \ + //e2e/live:async_chat_completion \ + //e2e/live:chat_completion \ + //e2e/live:contextualized_embeddings \ + //e2e/live:embeddings \ + //e2e/live:responses \ + //e2e/live:search \ + //e2e/live:streaming_chat \ + //e2e/live:streaming_responses \ --test_env=PPLX_API_TOKEN diff --git a/e2e/BUILD.bazel b/e2e/BUILD.bazel index 34f9178..57cdc89 100644 --- a/e2e/BUILD.bazel +++ b/e2e/BUILD.bazel @@ -39,54 +39,6 @@ js_test( tags = ["manual"], ) -js_test( - name = "live_fast", - data = [ - ":typecheck", - "//:node_modules/@perplexity-ai/perplexity_ai", - "//:node_modules/published-sdk", - ], - entry_point = "live.spec.ts", - env = {"LIVE_TEST_SHARD": "fast"}, - tags = ["manual"], -) - -js_test( - name = "live_responses", - data = [ - ":typecheck", - "//:node_modules/@perplexity-ai/perplexity_ai", - "//:node_modules/published-sdk", - ], - entry_point = "live.spec.ts", - env = {"LIVE_TEST_SHARD": "responses"}, - tags = ["manual"], -) - -js_test( - name = "live_streaming_responses", - data = [ - ":typecheck", - "//:node_modules/@perplexity-ai/perplexity_ai", - "//:node_modules/published-sdk", - ], - entry_point = "live.spec.ts", - env = {"LIVE_TEST_SHARD": "streaming-responses"}, - tags = ["manual"], -) - -js_test( - name = "live_async", - data = [ - ":typecheck", - "//:node_modules/@perplexity-ai/perplexity_ai", - "//:node_modules/published-sdk", - ], - entry_point = "live.spec.ts", - env = {"LIVE_TEST_SHARD": "async"}, - tags = ["manual"], -) - ts_project( name = "typecheck", testonly = True, @@ -111,10 +63,7 @@ ts_project( ts_test( name = "e2e_test", - srcs = [ - "live.spec.ts", - "sdk-parity.spec.ts", - ], + srcs = ["sdk-parity.spec.ts"], tags = ["manual"], tsconfig_types = ["node"], deps = [ diff --git a/e2e/live/BUILD.bazel b/e2e/live/BUILD.bazel new file mode 100644 index 0000000..9918fc0 --- /dev/null +++ b/e2e/live/BUILD.bazel @@ -0,0 +1,147 @@ +load("@aspect_rules_js//js:defs.bzl", "js_test") +load("@aspect_rules_ts//ts:defs.bzl", "ts_project") +load("//:ts.bzl", "ts_library", "ts_test") +load("//:tsconfig.bzl", "BASE_COMPILER_OPTIONS") + +package(default_visibility = ["//visibility:private"]) + +js_test( + name = "async_chat_completion", + data = [ + ":typecheck", + "//:node_modules/@perplexity-ai/perplexity_ai", + "//:node_modules/published-sdk", + ], + entry_point = "async-chat-completion.spec.ts", + tags = ["manual"], +) + +js_test( + name = "chat_completion", + data = [ + ":typecheck", + "//:node_modules/@perplexity-ai/perplexity_ai", + "//:node_modules/published-sdk", + ], + entry_point = "chat-completion.spec.ts", + tags = ["manual"], +) + +js_test( + name = "contextualized_embeddings", + data = [ + ":typecheck", + "//:node_modules/@perplexity-ai/perplexity_ai", + "//:node_modules/published-sdk", + ], + entry_point = "contextualized-embeddings.spec.ts", + tags = ["manual"], +) + +js_test( + name = "embeddings", + data = [ + ":typecheck", + "//:node_modules/@perplexity-ai/perplexity_ai", + "//:node_modules/published-sdk", + ], + entry_point = "embeddings.spec.ts", + tags = ["manual"], +) + +js_test( + name = "responses", + data = [ + ":typecheck", + "//:node_modules/@perplexity-ai/perplexity_ai", + "//:node_modules/published-sdk", + ], + entry_point = "responses.spec.ts", + tags = ["manual"], +) + +js_test( + name = "search", + data = [ + ":typecheck", + "//:node_modules/@perplexity-ai/perplexity_ai", + "//:node_modules/published-sdk", + ], + entry_point = "search.spec.ts", + tags = ["manual"], +) + +js_test( + name = "streaming_chat", + data = [ + ":typecheck", + "//:node_modules/@perplexity-ai/perplexity_ai", + "//:node_modules/published-sdk", + ], + entry_point = "streaming-chat.spec.ts", + tags = ["manual"], +) + +js_test( + name = "streaming_responses", + data = [ + ":typecheck", + "//:node_modules/@perplexity-ai/perplexity_ai", + "//:node_modules/published-sdk", + ], + entry_point = "streaming-responses.spec.ts", + tags = ["manual"], +) + +ts_project( + name = "typecheck", + testonly = True, + srcs = [":live_test_sources"], + no_emit = True, + tags = ["manual"], + tsc = "//:tsc", + tsconfig = { + "compilerOptions": dict( + BASE_COMPILER_OPTIONS, + module = "ESNext", + moduleResolution = "Bundler", + types = ["node"], + ), + }, + deps = [ + "//:node_modules/@perplexity-ai/perplexity_ai", + "//:node_modules/@types/node", + "//:node_modules/published-sdk", + ], +) + +ts_test( + name = "live_test", + srcs = [ + "async-chat-completion.spec.ts", + "chat-completion.spec.ts", + "contextualized-embeddings.spec.ts", + "embeddings.spec.ts", + "responses.spec.ts", + "search.spec.ts", + "streaming-chat.spec.ts", + "streaming-responses.spec.ts", + ], + tags = ["manual"], + tsconfig_types = ["node"], + deps = [ + ":live", + "//:node_modules/@types/node", + ], +) + +ts_library( + name = "live", + srcs = ["helpers.ts"], + tsconfig_types = ["node"], + visibility = ["//visibility:public"], + deps = [ + "//:node_modules/@types/node", + "//:node_modules/published-sdk", + ], +) diff --git a/e2e/live/async-chat-completion.spec.ts b/e2e/live/async-chat-completion.spec.ts new file mode 100644 index 0000000..e333a38 --- /dev/null +++ b/e2e/live/async-chat-completion.spec.ts @@ -0,0 +1,31 @@ +import assert from 'node:assert/strict'; +import { it } from 'node:test'; +import { asyncCompletionContract, createClients, pollAsyncCompletion } from './helpers.ts'; + +it('matches published async chat completion lifecycle', { timeout: 180_000 }, async () => { + const { candidate, published } = createClients(); + const request = { + request: { + max_tokens: 16, + messages: [{ content: 'Reply with only the word pong.', role: 'user' as const }], + model: 'sonar-deep-research', + }, + }; + const [candidateInitial, publishedInitial] = await Promise.all([ + candidate.async.chat.completions.create(request), + published.async.chat.completions.create(request), + ]); + assert.deepStrictEqual( + asyncCompletionContract(candidateInitial), + asyncCompletionContract(publishedInitial), + ); + + const [candidateCompleted, publishedCompleted] = await Promise.all([ + pollAsyncCompletion(candidateInitial, (id) => candidate.async.chat.completions.get(id)), + pollAsyncCompletion(publishedInitial, (id) => published.async.chat.completions.get(id)), + ]); + assert.deepStrictEqual( + asyncCompletionContract(candidateCompleted), + asyncCompletionContract(publishedCompleted), + ); +}); diff --git a/e2e/live/chat-completion.spec.ts b/e2e/live/chat-completion.spec.ts new file mode 100644 index 0000000..f8f8bd6 --- /dev/null +++ b/e2e/live/chat-completion.spec.ts @@ -0,0 +1,18 @@ +import { it } from 'node:test'; +import { completionContract, createClients, expectMatchingContracts } from './helpers.ts'; + +it('matches published chat completion response contract', async () => { + const { candidate, published } = createClients(); + const request = { + max_tokens: 16, + messages: [{ content: 'Reply with only the word pong.', role: 'user' as const }], + model: 'sonar', + temperature: 0, + }; + + await expectMatchingContracts( + () => candidate.chat.completions.create(request), + () => published.chat.completions.create(request), + completionContract, + ); +}); diff --git a/e2e/live/contextualized-embeddings.spec.ts b/e2e/live/contextualized-embeddings.spec.ts new file mode 100644 index 0000000..b08dc60 --- /dev/null +++ b/e2e/live/contextualized-embeddings.spec.ts @@ -0,0 +1,17 @@ +import { it } from 'node:test'; +import { contextualizedEmbeddingContract, createClients, expectMatchingContracts } from './helpers.ts'; + +it('matches published contextualized embeddings response contract', async () => { + const { candidate, published } = createClients(); + const request = { + dimensions: 128, + input: [['Perplexity answers questions.', 'Its answers include citations.']], + model: 'pplx-embed-context-v1-0.6b' as const, + }; + + await expectMatchingContracts( + () => candidate.contextualizedEmbeddings.create(request), + () => published.contextualizedEmbeddings.create(request), + contextualizedEmbeddingContract, + ); +}); diff --git a/e2e/live/embeddings.spec.ts b/e2e/live/embeddings.spec.ts new file mode 100644 index 0000000..2b7171d --- /dev/null +++ b/e2e/live/embeddings.spec.ts @@ -0,0 +1,17 @@ +import { it } from 'node:test'; +import { createClients, embeddingContract, expectMatchingContracts } from './helpers.ts'; + +it('matches published embeddings response contract', async () => { + const { candidate, published } = createClients(); + const request = { + dimensions: 128, + input: 'Perplexity answers questions.', + model: 'pplx-embed-v1-0.6b' as const, + }; + + await expectMatchingContracts( + () => candidate.embeddings.create(request), + () => published.embeddings.create(request), + embeddingContract, + ); +}); diff --git a/e2e/live.spec.ts b/e2e/live/helpers.ts similarity index 55% rename from e2e/live.spec.ts rename to e2e/live/helpers.ts index 251c871..227be5f 100644 --- a/e2e/live.spec.ts +++ b/e2e/live/helpers.ts @@ -1,5 +1,4 @@ import assert from 'node:assert/strict'; -import { describe, it } from 'node:test'; import { setTimeout as delay } from 'node:timers/promises'; // oxlint-disable-next-line no-restricted-imports -- Verify packaged public entrypoint. import { Perplexity as Candidate } from '@perplexity-ai/perplexity_ai'; @@ -29,7 +28,7 @@ interface SearchResponse { [key: string]: unknown; } -function completionContract(response: unknown) { +export function completionContract(response: unknown) { assert.ok(typeof response === 'object' && response !== null); const completion = response as CompletionResponse; @@ -51,7 +50,7 @@ function completionContract(response: unknown) { }; } -function searchContract(response: unknown) { +export function searchContract(response: unknown) { assert.ok(typeof response === 'object' && response !== null); const search = response as SearchResponse; assert.equal(typeof search.id, 'string'); @@ -74,7 +73,7 @@ function searchContract(response: unknown) { }; } -function embeddingContract(response: unknown) { +export function embeddingContract(response: unknown) { assert.ok(typeof response === 'object' && response !== null); const embeddings = response as { data?: Array<{ embedding?: string; index?: number }>; @@ -96,7 +95,7 @@ function embeddingContract(response: unknown) { }; } -function contextualizedEmbeddingContract(response: unknown) { +export function contextualizedEmbeddingContract(response: unknown) { assert.ok(typeof response === 'object' && response !== null); const embeddings = response as { data?: Array<{ data?: Array<{ embedding?: string; index?: number }>; index?: number }>; @@ -122,7 +121,7 @@ function contextualizedEmbeddingContract(response: unknown) { }; } -function responseContract(response: unknown) { +export function responseContract(response: unknown) { assert.ok(typeof response === 'object' && response !== null); const result = response as { id?: string; @@ -143,7 +142,7 @@ function responseContract(response: unknown) { }; } -function streamingCompletionContract(chunks: unknown[]) { +export function streamingCompletionContract(chunks: unknown[]) { assert.ok(chunks.length > 0); const chunk = chunks.find((item) => { if (typeof item !== 'object' || item === null) { @@ -166,7 +165,7 @@ function streamingCompletionContract(chunks: unknown[]) { }; } -function streamingResponseContract(events: unknown[]) { +export function streamingResponseContract(events: unknown[]) { assert.ok(events.length > 0); const types = events.map((event) => { assert.ok(typeof event === 'object' && event !== null); @@ -186,7 +185,7 @@ function streamingResponseContract(events: unknown[]) { }; } -function asyncCompletionContract(response: unknown) { +export function asyncCompletionContract(response: unknown) { assert.ok(typeof response === 'object' && response !== null); const completion = response as { created_at?: unknown; @@ -221,7 +220,7 @@ function asyncCompletionStatus(response: unknown) { return status; } -async function pollAsyncCompletion(initial: unknown, retrieve: (id: string) => PromiseLike) { +export async function pollAsyncCompletion(initial: unknown, retrieve: (id: string) => PromiseLike) { const id = asyncCompletionID(initial); let completion = initial; @@ -238,7 +237,7 @@ async function pollAsyncCompletion(initial: unknown, retrieve: (id: string) => P assert.fail(`Async completion ${id} did not finish`); } -async function collect(stream: AsyncIterable) { +export async function collect(stream: AsyncIterable) { const chunks: unknown[] = []; for await (const chunk of stream) { chunks.push(chunk); @@ -246,7 +245,7 @@ async function collect(stream: AsyncIterable) { return chunks; } -async function expectMatchingContracts( +export async function expectMatchingContracts( candidateCall: () => PromiseLike, publishedCall: () => PromiseLike, contract: (response: unknown) => unknown, @@ -255,7 +254,7 @@ async function expectMatchingContracts( assert.deepStrictEqual(contract(candidateResponse), contract(publishedResponse)); } -function createClients() { +export function createClients() { const apiKey = process.env['PPLX_API_TOKEN']; assert.ok(apiKey, 'PPLX_API_TOKEN must be set'); return { @@ -263,164 +262,3 @@ function createClients() { published: new Published({ apiKey, maxRetries: 0 }), }; } - -const liveTestShard = process.env['LIVE_TEST_SHARD']; -const runsShard = (shard: string) => !liveTestShard || liveTestShard === shard; - -if (runsShard('fast')) { - describe('live API parity: fast APIs', () => { - it('matches published chat completion response contract', async () => { - const { candidate, published } = createClients(); - const request = { - max_tokens: 16, - messages: [{ content: 'Reply with only the word pong.', role: 'user' as const }], - model: 'sonar', - temperature: 0, - }; - - await expectMatchingContracts( - () => candidate.chat.completions.create(request), - () => published.chat.completions.create(request), - completionContract, - ); - }); - - it('matches published streaming chat contract', async () => { - const { candidate, published } = createClients(); - const request = { - max_tokens: 16, - messages: [{ content: 'Reply with only the word pong.', role: 'user' as const }], - model: 'sonar', - stream: true as const, - temperature: 0, - }; - - await expectMatchingContracts( - async () => collect(await candidate.chat.completions.create(request)), - async () => collect(await published.chat.completions.create(request)), - (chunks) => { - assert.ok(Array.isArray(chunks)); - return streamingCompletionContract(chunks); - }, - ); - }); - - it('matches published search response contract', async () => { - const { candidate, published } = createClients(); - const request = { - max_results: 1, - query: 'Perplexity AI', - }; - - await expectMatchingContracts( - () => candidate.search.create(request), - () => published.search.create(request), - searchContract, - ); - }); - - it('matches published embeddings response contract', async () => { - const { candidate, published } = createClients(); - const request = { - dimensions: 128, - input: 'Perplexity answers questions.', - model: 'pplx-embed-v1-0.6b' as const, - }; - - await expectMatchingContracts( - () => candidate.embeddings.create(request), - () => published.embeddings.create(request), - embeddingContract, - ); - }); - - it('matches published contextualized embeddings response contract', async () => { - const { candidate, published } = createClients(); - const request = { - dimensions: 128, - input: [['Perplexity answers questions.', 'Its answers include citations.']], - model: 'pplx-embed-context-v1-0.6b' as const, - }; - - await expectMatchingContracts( - () => candidate.contextualizedEmbeddings.create(request), - () => published.contextualizedEmbeddings.create(request), - contextualizedEmbeddingContract, - ); - }); - }); -} - -if (runsShard('responses')) { - describe('live API parity: Responses API', () => { - it('matches published responses API contract', async () => { - const { candidate, published } = createClients(); - const request = { - input: 'Reply with only the word pong.', - max_output_tokens: 16, - preset: 'pro-search', - }; - - await expectMatchingContracts( - () => candidate.responses.create(request), - () => published.responses.create(request), - responseContract, - ); - }); - }); -} - -if (runsShard('streaming-responses')) { - describe('live API parity: streaming Responses API', () => { - it('matches published streaming responses API contract', async () => { - const { candidate, published } = createClients(); - const request = { - input: 'Reply with only the word pong.', - max_output_tokens: 16, - preset: 'pro-search', - stream: true as const, - }; - - await expectMatchingContracts( - async () => collect(await candidate.responses.create(request)), - async () => collect(await published.responses.create(request)), - (events) => { - assert.ok(Array.isArray(events)); - return streamingResponseContract(events); - }, - ); - }); - }); -} - -if (runsShard('async')) { - describe('live API parity: async API', () => { - it('matches published async chat completion lifecycle', { timeout: 180_000 }, async () => { - const { candidate, published } = createClients(); - const request = { - request: { - max_tokens: 16, - messages: [{ content: 'Reply with only the word pong.', role: 'user' as const }], - model: 'sonar-deep-research', - }, - }; - const [candidateInitial, publishedInitial] = await Promise.all([ - candidate.async.chat.completions.create(request), - published.async.chat.completions.create(request), - ]); - assert.deepStrictEqual( - asyncCompletionContract(candidateInitial), - asyncCompletionContract(publishedInitial), - ); - - const [candidateCompleted, publishedCompleted] = await Promise.all([ - pollAsyncCompletion(candidateInitial, (id) => candidate.async.chat.completions.get(id)), - pollAsyncCompletion(publishedInitial, (id) => published.async.chat.completions.get(id)), - ]); - assert.deepStrictEqual( - asyncCompletionContract(candidateCompleted), - asyncCompletionContract(publishedCompleted), - ); - }); - }); -} diff --git a/e2e/live/responses.spec.ts b/e2e/live/responses.spec.ts new file mode 100644 index 0000000..c5bec16 --- /dev/null +++ b/e2e/live/responses.spec.ts @@ -0,0 +1,17 @@ +import { it } from 'node:test'; +import { createClients, expectMatchingContracts, responseContract } from './helpers.ts'; + +it('matches published responses API contract', async () => { + const { candidate, published } = createClients(); + const request = { + input: 'Reply with only the word pong.', + max_output_tokens: 16, + preset: 'pro-search', + }; + + await expectMatchingContracts( + () => candidate.responses.create(request), + () => published.responses.create(request), + responseContract, + ); +}); diff --git a/e2e/live/search.spec.ts b/e2e/live/search.spec.ts new file mode 100644 index 0000000..15ba41f --- /dev/null +++ b/e2e/live/search.spec.ts @@ -0,0 +1,16 @@ +import { it } from 'node:test'; +import { createClients, expectMatchingContracts, searchContract } from './helpers.ts'; + +it('matches published search response contract', async () => { + const { candidate, published } = createClients(); + const request = { + max_results: 1, + query: 'Perplexity AI', + }; + + await expectMatchingContracts( + () => candidate.search.create(request), + () => published.search.create(request), + searchContract, + ); +}); diff --git a/e2e/live/streaming-chat.spec.ts b/e2e/live/streaming-chat.spec.ts new file mode 100644 index 0000000..5285f14 --- /dev/null +++ b/e2e/live/streaming-chat.spec.ts @@ -0,0 +1,23 @@ +import assert from 'node:assert/strict'; +import { it } from 'node:test'; +import { collect, createClients, expectMatchingContracts, streamingCompletionContract } from './helpers.ts'; + +it('matches published streaming chat contract', async () => { + const { candidate, published } = createClients(); + const request = { + max_tokens: 16, + messages: [{ content: 'Reply with only the word pong.', role: 'user' as const }], + model: 'sonar', + stream: true as const, + temperature: 0, + }; + + await expectMatchingContracts( + async () => collect(await candidate.chat.completions.create(request)), + async () => collect(await published.chat.completions.create(request)), + (chunks) => { + assert.ok(Array.isArray(chunks)); + return streamingCompletionContract(chunks); + }, + ); +}); diff --git a/e2e/live/streaming-responses.spec.ts b/e2e/live/streaming-responses.spec.ts new file mode 100644 index 0000000..52cba9f --- /dev/null +++ b/e2e/live/streaming-responses.spec.ts @@ -0,0 +1,22 @@ +import assert from 'node:assert/strict'; +import { it } from 'node:test'; +import { collect, createClients, expectMatchingContracts, streamingResponseContract } from './helpers.ts'; + +it('matches published streaming responses API contract', async () => { + const { candidate, published } = createClients(); + const request = { + input: 'Reply with only the word pong.', + max_output_tokens: 16, + preset: 'pro-search', + stream: true as const, + }; + + await expectMatchingContracts( + async () => collect(await candidate.responses.create(request)), + async () => collect(await published.responses.create(request)), + (events) => { + assert.ok(Array.isArray(events)); + return streamingResponseContract(events); + }, + ); +}); From dad7f462d9a96bd4888ab4897b537e7062744716 Mon Sep 17 00:00:00 2001 From: Long Ho Date: Tue, 28 Jul 2026 10:03:08 -0400 Subject: [PATCH 3/6] ci: allow manual functional tests on branches --- .github/workflows/functional-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/functional-tests.yml b/.github/workflows/functional-tests.yml index 84e7f06..6557c4e 100644 --- a/.github/workflows/functional-tests.yml +++ b/.github/workflows/functional-tests.yml @@ -17,7 +17,7 @@ concurrency: jobs: functional-tests: - if: github.ref == 'refs/heads/main' + if: github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' timeout-minutes: 15 runs-on: ubuntu-24.04 steps: From 5b958bda28863ff703f47f39586afc060177bac9 Mon Sep 17 00:00:00 2001 From: Long Ho Date: Tue, 28 Jul 2026 10:07:14 -0400 Subject: [PATCH 4/6] refactor: generate live test targets in loop --- e2e/live/BUILD.bazel | 90 +---------------------------------------- e2e/live/live_tests.bzl | 24 +++++++++++ 2 files changed, 26 insertions(+), 88 deletions(-) create mode 100644 e2e/live/live_tests.bzl diff --git a/e2e/live/BUILD.bazel b/e2e/live/BUILD.bazel index 9918fc0..118df6f 100644 --- a/e2e/live/BUILD.bazel +++ b/e2e/live/BUILD.bazel @@ -1,97 +1,11 @@ -load("@aspect_rules_js//js:defs.bzl", "js_test") load("@aspect_rules_ts//ts:defs.bzl", "ts_project") load("//:ts.bzl", "ts_library", "ts_test") load("//:tsconfig.bzl", "BASE_COMPILER_OPTIONS") +load(":live_tests.bzl", "live_tests") package(default_visibility = ["//visibility:private"]) -js_test( - name = "async_chat_completion", - data = [ - ":typecheck", - "//:node_modules/@perplexity-ai/perplexity_ai", - "//:node_modules/published-sdk", - ], - entry_point = "async-chat-completion.spec.ts", - tags = ["manual"], -) - -js_test( - name = "chat_completion", - data = [ - ":typecheck", - "//:node_modules/@perplexity-ai/perplexity_ai", - "//:node_modules/published-sdk", - ], - entry_point = "chat-completion.spec.ts", - tags = ["manual"], -) - -js_test( - name = "contextualized_embeddings", - data = [ - ":typecheck", - "//:node_modules/@perplexity-ai/perplexity_ai", - "//:node_modules/published-sdk", - ], - entry_point = "contextualized-embeddings.spec.ts", - tags = ["manual"], -) - -js_test( - name = "embeddings", - data = [ - ":typecheck", - "//:node_modules/@perplexity-ai/perplexity_ai", - "//:node_modules/published-sdk", - ], - entry_point = "embeddings.spec.ts", - tags = ["manual"], -) - -js_test( - name = "responses", - data = [ - ":typecheck", - "//:node_modules/@perplexity-ai/perplexity_ai", - "//:node_modules/published-sdk", - ], - entry_point = "responses.spec.ts", - tags = ["manual"], -) - -js_test( - name = "search", - data = [ - ":typecheck", - "//:node_modules/@perplexity-ai/perplexity_ai", - "//:node_modules/published-sdk", - ], - entry_point = "search.spec.ts", - tags = ["manual"], -) - -js_test( - name = "streaming_chat", - data = [ - ":typecheck", - "//:node_modules/@perplexity-ai/perplexity_ai", - "//:node_modules/published-sdk", - ], - entry_point = "streaming-chat.spec.ts", - tags = ["manual"], -) - -js_test( - name = "streaming_responses", - data = [ - ":typecheck", - "//:node_modules/@perplexity-ai/perplexity_ai", - "//:node_modules/published-sdk", - ], - entry_point = "streaming-responses.spec.ts", - tags = ["manual"], -) +live_tests() ts_project( name = "typecheck", diff --git a/e2e/live/live_tests.bzl b/e2e/live/live_tests.bzl new file mode 100644 index 0000000..beee5b0 --- /dev/null +++ b/e2e/live/live_tests.bzl @@ -0,0 +1,24 @@ +load("@aspect_rules_js//js:defs.bzl", "js_test") + +def live_tests(): + for name in [ + "async_chat_completion", + "chat_completion", + "contextualized_embeddings", + "embeddings", + "responses", + "search", + "streaming_chat", + "streaming_responses", + ]: + js_test( + name = name, + data = [ + "helpers.ts", + ":typecheck", + "//:node_modules/@perplexity-ai/perplexity_ai", + "//:node_modules/published-sdk", + ], + entry_point = name.replace("_", "-") + ".spec.ts", + tags = ["manual"], + ) From b6a15441117afc5663a01df2b0e3340c077e37f2 Mon Sep 17 00:00:00 2001 From: Long Ho Date: Tue, 28 Jul 2026 10:10:20 -0400 Subject: [PATCH 5/6] ci: discover live tests by tag --- .bazelrc | 1 + .github/workflows/functional-tests.yml | 12 +----------- e2e/live/live_tests.bzl | 2 +- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/.bazelrc b/.bazelrc index 944de7c..cb96a94 100644 --- a/.bazelrc +++ b/.bazelrc @@ -9,3 +9,4 @@ common:windows --host_platform=@gazelle_ts//platforms:local_windows_msvc build:linux --linkopt=-no-pie test --test_output=errors +test --test_tag_filters=-live diff --git a/.github/workflows/functional-tests.yml b/.github/workflows/functional-tests.yml index 6557c4e..5033fec 100644 --- a/.github/workflows/functional-tests.yml +++ b/.github/workflows/functional-tests.yml @@ -47,14 +47,4 @@ jobs: - name: Run functional tests env: PPLX_API_TOKEN: ${{ secrets.PPLX_API_TOKEN }} - run: | - bazel test \ - //e2e/live:async_chat_completion \ - //e2e/live:chat_completion \ - //e2e/live:contextualized_embeddings \ - //e2e/live:embeddings \ - //e2e/live:responses \ - //e2e/live:search \ - //e2e/live:streaming_chat \ - //e2e/live:streaming_responses \ - --test_env=PPLX_API_TOKEN + run: bazel test //e2e/... --test_env=PPLX_API_TOKEN --test_tag_filters=live diff --git a/e2e/live/live_tests.bzl b/e2e/live/live_tests.bzl index beee5b0..c2cef31 100644 --- a/e2e/live/live_tests.bzl +++ b/e2e/live/live_tests.bzl @@ -20,5 +20,5 @@ def live_tests(): "//:node_modules/published-sdk", ], entry_point = name.replace("_", "-") + ".spec.ts", - tags = ["manual"], + tags = ["live"], ) From 0c0dcbc7d3da60856b133c1258fc9849595279e5 Mon Sep 17 00:00:00 2001 From: Long Ho Date: Tue, 28 Jul 2026 10:16:21 -0400 Subject: [PATCH 6/6] ci: query manual functional tests --- .bazelrc | 1 - .github/workflows/functional-tests.yml | 10 +++++++++- e2e/live/live_tests.bzl | 5 ++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.bazelrc b/.bazelrc index cb96a94..944de7c 100644 --- a/.bazelrc +++ b/.bazelrc @@ -9,4 +9,3 @@ common:windows --host_platform=@gazelle_ts//platforms:local_windows_msvc build:linux --linkopt=-no-pie test --test_output=errors -test --test_tag_filters=-live diff --git a/.github/workflows/functional-tests.yml b/.github/workflows/functional-tests.yml index 5033fec..309769e 100644 --- a/.github/workflows/functional-tests.yml +++ b/.github/workflows/functional-tests.yml @@ -47,4 +47,12 @@ jobs: - name: Run functional tests env: PPLX_API_TOKEN: ${{ secrets.PPLX_API_TOKEN }} - run: bazel test //e2e/... --test_env=PPLX_API_TOKEN --test_tag_filters=live + run: | + mapfile -t functional_tests < <( + bazel query 'attr(tags, functional_test, tests(//e2e/...))' --output=label + ) + if (( ${#functional_tests[@]} == 0 )); then + echo 'No functional tests found' >&2 + exit 1 + fi + bazel test "${functional_tests[@]}" --test_env=PPLX_API_TOKEN diff --git a/e2e/live/live_tests.bzl b/e2e/live/live_tests.bzl index c2cef31..707b11b 100644 --- a/e2e/live/live_tests.bzl +++ b/e2e/live/live_tests.bzl @@ -20,5 +20,8 @@ def live_tests(): "//:node_modules/published-sdk", ], entry_point = name.replace("_", "-") + ".spec.ts", - tags = ["live"], + tags = [ + "functional_test", + "manual", + ], )