diff --git a/.github/workflows/functional-tests.yml b/.github/workflows/functional-tests.yml index 69aea5d..309769e 100644 --- a/.github/workflows/functional-tests.yml +++ b/.github/workflows/functional-tests.yml @@ -17,7 +17,7 @@ concurrency: jobs: functional-tests: - if: github.ref == 'refs/heads/main' + if: github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' timeout-minutes: 15 runs-on: ubuntu-24.04 steps: @@ -47,4 +47,12 @@ jobs: - name: Run functional tests env: PPLX_API_TOKEN: ${{ secrets.PPLX_API_TOKEN }} - run: bazel test //e2e:live --test_env=PPLX_API_TOKEN + run: | + mapfile -t functional_tests < <( + bazel query 'attr(tags, functional_test, tests(//e2e/...))' --output=label + ) + if (( ${#functional_tests[@]} == 0 )); then + echo 'No functional tests found' >&2 + exit 1 + fi + bazel test "${functional_tests[@]}" --test_env=PPLX_API_TOKEN diff --git a/e2e/BUILD.bazel b/e2e/BUILD.bazel index 65ebfac..57cdc89 100644 --- a/e2e/BUILD.bazel +++ b/e2e/BUILD.bazel @@ -39,17 +39,6 @@ js_test( tags = ["manual"], ) -js_test( - name = "live", - data = [ - ":typecheck", - "//:node_modules/@perplexity-ai/perplexity_ai", - "//:node_modules/published-sdk", - ], - entry_point = "live.spec.ts", - tags = ["manual"], -) - ts_project( name = "typecheck", testonly = True, @@ -74,10 +63,7 @@ ts_project( ts_test( name = "e2e_test", - srcs = [ - "live.spec.ts", - "sdk-parity.spec.ts", - ], + srcs = ["sdk-parity.spec.ts"], tags = ["manual"], tsconfig_types = ["node"], deps = [ diff --git a/e2e/live/BUILD.bazel b/e2e/live/BUILD.bazel new file mode 100644 index 0000000..118df6f --- /dev/null +++ b/e2e/live/BUILD.bazel @@ -0,0 +1,61 @@ +load("@aspect_rules_ts//ts:defs.bzl", "ts_project") +load("//:ts.bzl", "ts_library", "ts_test") +load("//:tsconfig.bzl", "BASE_COMPILER_OPTIONS") +load(":live_tests.bzl", "live_tests") + +package(default_visibility = ["//visibility:private"]) + +live_tests() + +ts_project( + name = "typecheck", + testonly = True, + srcs = [":live_test_sources"], + no_emit = True, + tags = ["manual"], + tsc = "//:tsc", + tsconfig = { + "compilerOptions": dict( + BASE_COMPILER_OPTIONS, + module = "ESNext", + moduleResolution = "Bundler", + types = ["node"], + ), + }, + deps = [ + "//:node_modules/@perplexity-ai/perplexity_ai", + "//:node_modules/@types/node", + "//:node_modules/published-sdk", + ], +) + +ts_test( + name = "live_test", + srcs = [ + "async-chat-completion.spec.ts", + "chat-completion.spec.ts", + "contextualized-embeddings.spec.ts", + "embeddings.spec.ts", + "responses.spec.ts", + "search.spec.ts", + "streaming-chat.spec.ts", + "streaming-responses.spec.ts", + ], + tags = ["manual"], + tsconfig_types = ["node"], + deps = [ + ":live", + "//:node_modules/@types/node", + ], +) + +ts_library( + name = "live", + srcs = ["helpers.ts"], + tsconfig_types = ["node"], + visibility = ["//visibility:public"], + deps = [ + "//:node_modules/@types/node", + "//:node_modules/published-sdk", + ], +) diff --git a/e2e/live/async-chat-completion.spec.ts b/e2e/live/async-chat-completion.spec.ts new file mode 100644 index 0000000..e333a38 --- /dev/null +++ b/e2e/live/async-chat-completion.spec.ts @@ -0,0 +1,31 @@ +import assert from 'node:assert/strict'; +import { it } from 'node:test'; +import { asyncCompletionContract, createClients, pollAsyncCompletion } from './helpers.ts'; + +it('matches published async chat completion lifecycle', { timeout: 180_000 }, async () => { + const { candidate, published } = createClients(); + const request = { + request: { + max_tokens: 16, + messages: [{ content: 'Reply with only the word pong.', role: 'user' as const }], + model: 'sonar-deep-research', + }, + }; + const [candidateInitial, publishedInitial] = await Promise.all([ + candidate.async.chat.completions.create(request), + published.async.chat.completions.create(request), + ]); + assert.deepStrictEqual( + asyncCompletionContract(candidateInitial), + asyncCompletionContract(publishedInitial), + ); + + const [candidateCompleted, publishedCompleted] = await Promise.all([ + pollAsyncCompletion(candidateInitial, (id) => candidate.async.chat.completions.get(id)), + pollAsyncCompletion(publishedInitial, (id) => published.async.chat.completions.get(id)), + ]); + assert.deepStrictEqual( + asyncCompletionContract(candidateCompleted), + asyncCompletionContract(publishedCompleted), + ); +}); diff --git a/e2e/live/chat-completion.spec.ts b/e2e/live/chat-completion.spec.ts new file mode 100644 index 0000000..f8f8bd6 --- /dev/null +++ b/e2e/live/chat-completion.spec.ts @@ -0,0 +1,18 @@ +import { it } from 'node:test'; +import { completionContract, createClients, expectMatchingContracts } from './helpers.ts'; + +it('matches published chat completion response contract', async () => { + const { candidate, published } = createClients(); + const request = { + max_tokens: 16, + messages: [{ content: 'Reply with only the word pong.', role: 'user' as const }], + model: 'sonar', + temperature: 0, + }; + + await expectMatchingContracts( + () => candidate.chat.completions.create(request), + () => published.chat.completions.create(request), + completionContract, + ); +}); diff --git a/e2e/live/contextualized-embeddings.spec.ts b/e2e/live/contextualized-embeddings.spec.ts new file mode 100644 index 0000000..b08dc60 --- /dev/null +++ b/e2e/live/contextualized-embeddings.spec.ts @@ -0,0 +1,17 @@ +import { it } from 'node:test'; +import { contextualizedEmbeddingContract, createClients, expectMatchingContracts } from './helpers.ts'; + +it('matches published contextualized embeddings response contract', async () => { + const { candidate, published } = createClients(); + const request = { + dimensions: 128, + input: [['Perplexity answers questions.', 'Its answers include citations.']], + model: 'pplx-embed-context-v1-0.6b' as const, + }; + + await expectMatchingContracts( + () => candidate.contextualizedEmbeddings.create(request), + () => published.contextualizedEmbeddings.create(request), + contextualizedEmbeddingContract, + ); +}); diff --git a/e2e/live/embeddings.spec.ts b/e2e/live/embeddings.spec.ts new file mode 100644 index 0000000..2b7171d --- /dev/null +++ b/e2e/live/embeddings.spec.ts @@ -0,0 +1,17 @@ +import { it } from 'node:test'; +import { createClients, embeddingContract, expectMatchingContracts } from './helpers.ts'; + +it('matches published embeddings response contract', async () => { + const { candidate, published } = createClients(); + const request = { + dimensions: 128, + input: 'Perplexity answers questions.', + model: 'pplx-embed-v1-0.6b' as const, + }; + + await expectMatchingContracts( + () => candidate.embeddings.create(request), + () => published.embeddings.create(request), + embeddingContract, + ); +}); diff --git a/e2e/live.spec.ts b/e2e/live/helpers.ts similarity index 58% rename from e2e/live.spec.ts rename to e2e/live/helpers.ts index 025051e..227be5f 100644 --- a/e2e/live.spec.ts +++ b/e2e/live/helpers.ts @@ -1,5 +1,4 @@ import assert from 'node:assert/strict'; -import { describe, it } from 'node:test'; import { setTimeout as delay } from 'node:timers/promises'; // oxlint-disable-next-line no-restricted-imports -- Verify packaged public entrypoint. import { Perplexity as Candidate } from '@perplexity-ai/perplexity_ai'; @@ -29,7 +28,7 @@ interface SearchResponse { [key: string]: unknown; } -function completionContract(response: unknown) { +export function completionContract(response: unknown) { assert.ok(typeof response === 'object' && response !== null); const completion = response as CompletionResponse; @@ -51,7 +50,7 @@ function completionContract(response: unknown) { }; } -function searchContract(response: unknown) { +export function searchContract(response: unknown) { assert.ok(typeof response === 'object' && response !== null); const search = response as SearchResponse; assert.equal(typeof search.id, 'string'); @@ -74,7 +73,7 @@ function searchContract(response: unknown) { }; } -function embeddingContract(response: unknown) { +export function embeddingContract(response: unknown) { assert.ok(typeof response === 'object' && response !== null); const embeddings = response as { data?: Array<{ embedding?: string; index?: number }>; @@ -96,7 +95,7 @@ function embeddingContract(response: unknown) { }; } -function contextualizedEmbeddingContract(response: unknown) { +export function contextualizedEmbeddingContract(response: unknown) { assert.ok(typeof response === 'object' && response !== null); const embeddings = response as { data?: Array<{ data?: Array<{ embedding?: string; index?: number }>; index?: number }>; @@ -122,7 +121,7 @@ function contextualizedEmbeddingContract(response: unknown) { }; } -function responseContract(response: unknown) { +export function responseContract(response: unknown) { assert.ok(typeof response === 'object' && response !== null); const result = response as { id?: string; @@ -143,7 +142,7 @@ function responseContract(response: unknown) { }; } -function streamingCompletionContract(chunks: unknown[]) { +export function streamingCompletionContract(chunks: unknown[]) { assert.ok(chunks.length > 0); const chunk = chunks.find((item) => { if (typeof item !== 'object' || item === null) { @@ -166,7 +165,7 @@ function streamingCompletionContract(chunks: unknown[]) { }; } -function streamingResponseContract(events: unknown[]) { +export function streamingResponseContract(events: unknown[]) { assert.ok(events.length > 0); const types = events.map((event) => { assert.ok(typeof event === 'object' && event !== null); @@ -186,7 +185,7 @@ function streamingResponseContract(events: unknown[]) { }; } -function asyncCompletionContract(response: unknown) { +export function asyncCompletionContract(response: unknown) { assert.ok(typeof response === 'object' && response !== null); const completion = response as { created_at?: unknown; @@ -221,7 +220,7 @@ function asyncCompletionStatus(response: unknown) { return status; } -async function pollAsyncCompletion(initial: unknown, retrieve: (id: string) => PromiseLike) { +export async function pollAsyncCompletion(initial: unknown, retrieve: (id: string) => PromiseLike) { const id = asyncCompletionID(initial); let completion = initial; @@ -238,7 +237,7 @@ async function pollAsyncCompletion(initial: unknown, retrieve: (id: string) => P assert.fail(`Async completion ${id} did not finish`); } -async function collect(stream: AsyncIterable) { +export async function collect(stream: AsyncIterable) { const chunks: unknown[] = []; for await (const chunk of stream) { chunks.push(chunk); @@ -246,7 +245,7 @@ async function collect(stream: AsyncIterable) { return chunks; } -async function expectMatchingContracts( +export async function expectMatchingContracts( candidateCall: () => PromiseLike, publishedCall: () => PromiseLike, contract: (response: unknown) => unknown, @@ -255,7 +254,7 @@ async function expectMatchingContracts( assert.deepStrictEqual(contract(candidateResponse), contract(publishedResponse)); } -function createClients() { +export function createClients() { const apiKey = process.env['PPLX_API_TOKEN']; assert.ok(apiKey, 'PPLX_API_TOKEN must be set'); return { @@ -263,147 +262,3 @@ function createClients() { published: new Published({ apiKey, maxRetries: 0 }), }; } - -describe('live API parity', () => { - it('matches published chat completion response contract', async () => { - const { candidate, published } = createClients(); - const request = { - max_tokens: 16, - messages: [{ content: 'Reply with only the word pong.', role: 'user' as const }], - model: 'sonar', - temperature: 0, - }; - - await expectMatchingContracts( - () => candidate.chat.completions.create(request), - () => published.chat.completions.create(request), - completionContract, - ); - }); - - it('matches published streaming chat contract', async () => { - const { candidate, published } = createClients(); - const request = { - max_tokens: 16, - messages: [{ content: 'Reply with only the word pong.', role: 'user' as const }], - model: 'sonar', - stream: true as const, - temperature: 0, - }; - - await expectMatchingContracts( - async () => collect(await candidate.chat.completions.create(request)), - async () => collect(await published.chat.completions.create(request)), - (chunks) => { - assert.ok(Array.isArray(chunks)); - return streamingCompletionContract(chunks); - }, - ); - }); - - it('matches published search response contract', async () => { - const { candidate, published } = createClients(); - const request = { - max_results: 1, - query: 'Perplexity AI', - }; - - await expectMatchingContracts( - () => candidate.search.create(request), - () => published.search.create(request), - searchContract, - ); - }); - - it('matches published embeddings response contract', async () => { - const { candidate, published } = createClients(); - const request = { - dimensions: 128, - input: 'Perplexity answers questions.', - model: 'pplx-embed-v1-0.6b' as const, - }; - - await expectMatchingContracts( - () => candidate.embeddings.create(request), - () => published.embeddings.create(request), - embeddingContract, - ); - }); - - it('matches published contextualized embeddings response contract', async () => { - const { candidate, published } = createClients(); - const request = { - dimensions: 128, - input: [['Perplexity answers questions.', 'Its answers include citations.']], - model: 'pplx-embed-context-v1-0.6b' as const, - }; - - await expectMatchingContracts( - () => candidate.contextualizedEmbeddings.create(request), - () => published.contextualizedEmbeddings.create(request), - contextualizedEmbeddingContract, - ); - }); - - it('matches published responses API contract', async () => { - const { candidate, published } = createClients(); - const request = { - input: 'Reply with only the word pong.', - max_output_tokens: 16, - preset: 'pro-search', - }; - - await expectMatchingContracts( - () => candidate.responses.create(request), - () => published.responses.create(request), - responseContract, - ); - }); - - it('matches published streaming responses API contract', async () => { - const { candidate, published } = createClients(); - const request = { - input: 'Reply with only the word pong.', - max_output_tokens: 16, - preset: 'pro-search', - stream: true as const, - }; - - await expectMatchingContracts( - async () => collect(await candidate.responses.create(request)), - async () => collect(await published.responses.create(request)), - (events) => { - assert.ok(Array.isArray(events)); - return streamingResponseContract(events); - }, - ); - }); - - it('matches published async chat completion lifecycle', { timeout: 180_000 }, async () => { - const { candidate, published } = createClients(); - const request = { - request: { - max_tokens: 16, - messages: [{ content: 'Reply with only the word pong.', role: 'user' as const }], - model: 'sonar-deep-research', - }, - }; - const [candidateInitial, publishedInitial] = await Promise.all([ - candidate.async.chat.completions.create(request), - published.async.chat.completions.create(request), - ]); - assert.deepStrictEqual( - asyncCompletionContract(candidateInitial), - asyncCompletionContract(publishedInitial), - ); - - const [candidateCompleted, publishedCompleted] = await Promise.all([ - pollAsyncCompletion(candidateInitial, (id) => candidate.async.chat.completions.get(id)), - pollAsyncCompletion(publishedInitial, (id) => published.async.chat.completions.get(id)), - ]); - assert.deepStrictEqual( - asyncCompletionContract(candidateCompleted), - asyncCompletionContract(publishedCompleted), - ); - }); -}); diff --git a/e2e/live/live_tests.bzl b/e2e/live/live_tests.bzl new file mode 100644 index 0000000..707b11b --- /dev/null +++ b/e2e/live/live_tests.bzl @@ -0,0 +1,27 @@ +load("@aspect_rules_js//js:defs.bzl", "js_test") + +def live_tests(): + for name in [ + "async_chat_completion", + "chat_completion", + "contextualized_embeddings", + "embeddings", + "responses", + "search", + "streaming_chat", + "streaming_responses", + ]: + js_test( + name = name, + data = [ + "helpers.ts", + ":typecheck", + "//:node_modules/@perplexity-ai/perplexity_ai", + "//:node_modules/published-sdk", + ], + entry_point = name.replace("_", "-") + ".spec.ts", + tags = [ + "functional_test", + "manual", + ], + ) diff --git a/e2e/live/responses.spec.ts b/e2e/live/responses.spec.ts new file mode 100644 index 0000000..c5bec16 --- /dev/null +++ b/e2e/live/responses.spec.ts @@ -0,0 +1,17 @@ +import { it } from 'node:test'; +import { createClients, expectMatchingContracts, responseContract } from './helpers.ts'; + +it('matches published responses API contract', async () => { + const { candidate, published } = createClients(); + const request = { + input: 'Reply with only the word pong.', + max_output_tokens: 16, + preset: 'pro-search', + }; + + await expectMatchingContracts( + () => candidate.responses.create(request), + () => published.responses.create(request), + responseContract, + ); +}); diff --git a/e2e/live/search.spec.ts b/e2e/live/search.spec.ts new file mode 100644 index 0000000..15ba41f --- /dev/null +++ b/e2e/live/search.spec.ts @@ -0,0 +1,16 @@ +import { it } from 'node:test'; +import { createClients, expectMatchingContracts, searchContract } from './helpers.ts'; + +it('matches published search response contract', async () => { + const { candidate, published } = createClients(); + const request = { + max_results: 1, + query: 'Perplexity AI', + }; + + await expectMatchingContracts( + () => candidate.search.create(request), + () => published.search.create(request), + searchContract, + ); +}); diff --git a/e2e/live/streaming-chat.spec.ts b/e2e/live/streaming-chat.spec.ts new file mode 100644 index 0000000..5285f14 --- /dev/null +++ b/e2e/live/streaming-chat.spec.ts @@ -0,0 +1,23 @@ +import assert from 'node:assert/strict'; +import { it } from 'node:test'; +import { collect, createClients, expectMatchingContracts, streamingCompletionContract } from './helpers.ts'; + +it('matches published streaming chat contract', async () => { + const { candidate, published } = createClients(); + const request = { + max_tokens: 16, + messages: [{ content: 'Reply with only the word pong.', role: 'user' as const }], + model: 'sonar', + stream: true as const, + temperature: 0, + }; + + await expectMatchingContracts( + async () => collect(await candidate.chat.completions.create(request)), + async () => collect(await published.chat.completions.create(request)), + (chunks) => { + assert.ok(Array.isArray(chunks)); + return streamingCompletionContract(chunks); + }, + ); +}); diff --git a/e2e/live/streaming-responses.spec.ts b/e2e/live/streaming-responses.spec.ts new file mode 100644 index 0000000..52cba9f --- /dev/null +++ b/e2e/live/streaming-responses.spec.ts @@ -0,0 +1,22 @@ +import assert from 'node:assert/strict'; +import { it } from 'node:test'; +import { collect, createClients, expectMatchingContracts, streamingResponseContract } from './helpers.ts'; + +it('matches published streaming responses API contract', async () => { + const { candidate, published } = createClients(); + const request = { + input: 'Reply with only the word pong.', + max_output_tokens: 16, + preset: 'pro-search', + stream: true as const, + }; + + await expectMatchingContracts( + async () => collect(await candidate.responses.create(request)), + async () => collect(await published.responses.create(request)), + (events) => { + assert.ok(Array.isArray(events)); + return streamingResponseContract(events); + }, + ); +});