diff --git a/cli-manifest.json b/cli-manifest.json index cc96cc2c..915ce685 100644 --- a/cli-manifest.json +++ b/cli-manifest.json @@ -1,4 +1,45 @@ [ + { + "site": "web", + "name": "fetch", + "description": "Fetch a URL locally without launching a browser", + "access": "read", + "strategy": "public", + "browser": false, + "args": [ + { + "name": "url", + "type": "string", + "required": true, + "help": "http(s) URL to fetch" + }, + { + "name": "timeout", + "type": "int", + "default": 30, + "required": false, + "help": "Fetch budget in seconds" + }, + { + "name": "max-chars", + "type": "int", + "default": 50000, + "required": false, + "help": "Maximum characters of extracted content" + }, + { + "name": "allow-private", + "type": "boolean", + "default": false, + "required": false, + "help": "Allow private/loopback addresses" + } + ], + "defaultFormat": "md", + "type": "js", + "modulePath": "web/fetch.js", + "sourceFile": "web/fetch.js" + }, { "site": "web", "name": "fetch-browser", diff --git a/clis/web/fetch.js b/clis/web/fetch.js index 113b6169..1670878c 100644 --- a/clis/web/fetch.js +++ b/clis/web/fetch.js @@ -1 +1,11 @@ -import '@agentrhq/webcmd/fetch/command'; +/** + * Discovery entry for client-owned `web fetch`. + * + * Execution stays on the main.ts fast path so hosted mode never cloud-routes + * this command. This module exists so build-manifest, `webcmd list`, + * completions, and Commander help can see the same registration as the + * always-available fast path. + */ +import { makeWebFetchCommand } from '@agentrhq/webcmd/fetch/command'; + +export const command = makeWebFetchCommand(); diff --git a/src/discovery.ts b/src/discovery.ts index c65f59d9..b50ae420 100644 --- a/src/discovery.ts +++ b/src/discovery.ts @@ -38,8 +38,8 @@ export const USER_WEBCMD_DIR = getUserWebcmdDir(); export const USER_CLIS_DIR = getUserClisDir(); /** Plugins directory: ~/.webcmd/plugins/ */ export const PLUGINS_DIR = getPluginsDir(); -/** Matches files that register commands via cli() or lifecycle hooks */ -const PLUGIN_MODULE_PATTERN = /\b(?:cli|registerSiteAuthCommands|onStartup|onBeforeExecute|onAfterExecute)\s*\(/; +/** Matches files that register commands via cli() / factories or lifecycle hooks */ +const PLUGIN_MODULE_PATTERN = /\b(?:cli|registerSiteAuthCommands|onStartup|onBeforeExecute|onAfterExecute)\s*\(|\bmake[A-Z]\w*Command\s*\(/; function parseStrategy(rawStrategy: string | undefined, fallback: Strategy = Strategy.COOKIE): Strategy { if (!rawStrategy) return fallback; diff --git a/src/fetch/command.test.ts b/src/fetch/command.test.ts index 5b9823d2..7d0de079 100644 --- a/src/fetch/command.test.ts +++ b/src/fetch/command.test.ts @@ -1,13 +1,119 @@ import { describe, expect, it, vi } from 'vitest'; -import { formatWebFetchMarkdown, runClientOwnedWebFetch } from './command.js'; +import { + formatWebFetchHelp, + formatWebFetchMarkdown, + runClientOwnedWebFetch, + WEB_FETCH_ARGS, + webFetchCommand, +} from './command.js'; describe('web fetch command', () => { it('renders fetch metadata before content', () => { - expect(formatWebFetchMarkdown({ status: 200, requestedUrl: 'https://a', finalUrl: 'https://b', contentType: 'text/plain', tier: 'plain', title: 'T', extractionSource: 'raw', truncated: false, content: 'body' })).toContain('Source: https://a'); + expect(formatWebFetchMarkdown({ + status: 200, + requestedUrl: 'https://a', + finalUrl: 'https://b', + contentType: 'text/plain', + tier: 'plain', + title: 'T', + extractionSource: 'raw', + truncated: false, + content: 'body', + })).toContain('Source: https://a'); }); + it('runs the client-owned command without Cloud routing', async () => { - const webFetch = vi.fn().mockResolvedValue({ status: 200, requestedUrl: 'https://a', finalUrl: 'https://a', contentType: 'text/plain', tier: 'plain', title: '', extractionSource: 'raw', truncated: false, content: 'ok' }); - await runClientOwnedWebFetch(['web', 'fetch', '--url', 'https://a'], { webFetch, stdout: { write: vi.fn() } as never }); + const webFetch = vi.fn().mockResolvedValue({ + status: 200, + requestedUrl: 'https://a', + finalUrl: 'https://a', + contentType: 'text/plain', + tier: 'plain', + title: '', + extractionSource: 'raw', + truncated: false, + content: 'ok', + }); + await runClientOwnedWebFetch(['web', 'fetch', '--url', 'https://a'], { + webFetch, + stdout: { write: vi.fn() } as never, + }); expect(webFetch).toHaveBeenCalledOnce(); }); + + it('keeps discovery args aligned with the registered command', () => { + expect(webFetchCommand.site).toBe('web'); + expect(webFetchCommand.name).toBe('fetch'); + expect(webFetchCommand.browser).toBe(false); + expect(webFetchCommand.args).toEqual(WEB_FETCH_ARGS); + }); + + it('prints real help for -h and --help without requiring --url', async () => { + for (const flag of ['-h', '--help'] as const) { + const write = vi.fn(); + const webFetch = vi.fn(); + await runClientOwnedWebFetch(['web', 'fetch', flag], { + webFetch, + stdout: { write } as never, + }); + expect(webFetch).not.toHaveBeenCalled(); + expect(write).toHaveBeenCalledOnce(); + const help = String(write.mock.calls[0]![0]); + expect(help).toContain('Usage:'); + expect(help).toContain('web fetch'); + expect(help).toContain('--url'); + expect(help).toContain('--timeout'); + expect(help).toBe(formatWebFetchHelp()); + } + }); + + it('honours -f for output instead of always printing markdown', async () => { + const result = { + status: 200, + requestedUrl: 'https://a', + finalUrl: 'https://a', + contentType: 'text/plain', + tier: 'plain', + title: '', + extractionSource: 'raw', + truncated: false, + content: 'ok', + }; + const write = vi.fn(); + await runClientOwnedWebFetch(['web', 'fetch', '--url', 'https://a', '-f', 'json'], { + webFetch: vi.fn().mockResolvedValue(result), + stdout: { write } as never, + }); + expect(JSON.parse(String(write.mock.calls[0]![0]))).toMatchObject({ content: 'ok' }); + }); + + it('serves structured help for --help -f yaml', async () => { + const write = vi.fn(); + await runClientOwnedWebFetch(['web', 'fetch', '--help', '-f', 'yaml'], { + webFetch: vi.fn(), + stdout: { write } as never, + }); + expect(String(write.mock.calls[0]![0])).toContain('name: fetch'); + }); + + it('rejects an unsupported --format instead of silently printing a table', async () => { + await expect(runClientOwnedWebFetch(['web', 'fetch', '--url', 'https://a', '-f', 'xml'], { + webFetch: vi.fn(), + stdout: { write: vi.fn() } as never, + })).rejects.toThrow('--format must be one of'); + }); + + it('rejects a flag-shaped value for --timeout instead of coercing it', async () => { + await expect(runClientOwnedWebFetch(['web', 'fetch', '--url', 'https://a', '--timeout', '-5'], { + webFetch: vi.fn(), + stdout: { write: vi.fn() } as never, + })).rejects.toThrow('--timeout requires a value'); + }); + + it('rejects missing --url with ArgumentError when help is not requested', async () => { + await expect(runClientOwnedWebFetch(['web', 'fetch'], { + webFetch: vi.fn(), + stdout: { write: vi.fn() } as never, + })).rejects.toThrow('--url must be an http or https URL'); + }); }); diff --git a/src/fetch/command.ts b/src/fetch/command.ts index 103e8487..3d5536a0 100644 --- a/src/fetch/command.ts +++ b/src/fetch/command.ts @@ -1,37 +1,146 @@ -import { cli, Strategy } from '../registry.js'; +import { cli, Strategy, type Arg, type CliCommand } from '../registry.js'; import { ArgumentError } from '../errors.js'; +import { commandHelpData, formatCommandHelp, toPresentableCommand } from '../command-presentation.js'; +import { renderStructuredHelp } from '../help.js'; +import { formatOutput } from '../output.js'; import { webFetch, type WebFetchOptions, type WebFetchResult } from './client.js'; -export const webFetchCommand = cli({ - site: 'web', name: 'fetch', access: 'read', strategy: Strategy.PUBLIC, browser: false, - description: 'Fetch a URL locally without launching a browser', defaultFormat: 'md', - args: [ - { name: 'url', type: 'string', required: true }, - { name: 'timeout', type: 'int', default: 30 }, - { name: 'max-chars', type: 'int', default: 50000 }, - { name: 'allow-private', type: 'boolean', default: false }, - ], - func: async kwargs => webFetch({ url: String(kwargs.url), timeoutSeconds: Number(kwargs.timeout ?? 30), maxChars: Number(kwargs['max-chars'] ?? 50000), allowPrivate: kwargs['allow-private'] === true }), -}); +/** Single source of truth for discovery, Commander help, and the client-owned fast path. */ +export const WEB_FETCH_ARGS: Arg[] = [ + { name: 'url', type: 'string', required: true, help: 'http(s) URL to fetch' }, + { name: 'timeout', type: 'int', default: 30, help: 'Fetch budget in seconds' }, + { name: 'max-chars', type: 'int', default: 50000, help: 'Maximum characters of extracted content' }, + { name: 'allow-private', type: 'boolean', default: false, help: 'Allow private/loopback addresses' }, +]; + +const DEFAULT_TIMEOUT = 30; +const DEFAULT_MAX_CHARS = 50000; + +let registered: CliCommand | undefined; + +/** + * Register (or return) the builtin `web fetch` command. + * Called from clis/web/fetch.js so build-manifest / filesystem discovery see it. + * Safe to call more than once — returns the same command object. + */ +export function makeWebFetchCommand(): CliCommand { + if (registered) return registered; + registered = cli({ + site: 'web', + name: 'fetch', + access: 'read', + strategy: Strategy.PUBLIC, + browser: false, + description: 'Fetch a URL locally without launching a browser', + defaultFormat: 'md', + args: WEB_FETCH_ARGS, + func: async kwargs => webFetch(kwargsToOptions(kwargs)), + }); + return registered; +} + +/** Eager registration for consumers that import the command object directly. */ +export const webFetchCommand = makeWebFetchCommand(); export function formatWebFetchMarkdown(result: WebFetchResult): string { return [`# ${result.title || 'Fetched content'}`, '', `Source: ${result.requestedUrl}`, `Final URL: ${result.finalUrl}`, `Content type: ${result.contentType || 'unknown'}`, `Extraction: ${result.extractionSource}`, '', result.content].join('\n'); } +export function formatWebFetchHelp(): string { + return formatCommandHelp(toPresentableCommand(webFetchCommand)); +} + +function defaultInt(name: string): number { + if (name === 'timeout') return DEFAULT_TIMEOUT; + if (name === 'max-chars') return DEFAULT_MAX_CHARS; + return 0; +} + +function kwargsToOptions(kwargs: Record): WebFetchOptions { + return { + url: String(kwargs.url), + timeoutSeconds: Number(kwargs.timeout ?? DEFAULT_TIMEOUT), + maxChars: Number(kwargs['max-chars'] ?? DEFAULT_MAX_CHARS), + allowPrivate: kwargs['allow-private'] === true, + }; +} + +function wantsHelp(argv: readonly string[]): boolean { + return argv.slice(2).some(arg => arg === '-h' || arg === '--help'); +} + +/** Formats advertised by the common-options block the help text prints. */ +const OUTPUT_FORMATS = ['table', 'plain', 'json', 'yaml', 'md', 'csv'] as const; + +/** Reads -f/--format in the same shapes Commander accepts, so help cannot over-promise. */ +function requestedFormat(argv: readonly string[]): string | undefined { + for (let index = 2; index < argv.length; index++) { + const arg = argv[index]!; + let value: string | undefined; + if (arg === '-f' || arg === '--format') value = argv[index + 1]; + else if (arg.startsWith('--format=')) value = arg.slice('--format='.length); + else if (arg.startsWith('-f') && arg.length > 2) value = arg.slice(2); + else continue; + if (value === undefined || !OUTPUT_FORMATS.includes(value as typeof OUTPUT_FORMATS[number])) { + throw new ArgumentError(`--format must be one of: ${OUTPUT_FORMATS.join(', ')}`); + } + return value; + } + return undefined; +} + function clientOptions(argv: readonly string[]): WebFetchOptions { const values: Record = {}; for (let index = 2; index < argv.length; index++) { const arg = argv[index]!; if (!arg.startsWith('--')) continue; - const name = arg.slice(2); const value = argv[index + 1]; - if (value && !value.startsWith('--')) { values[name] = value; index++; } else values[name] = true; + const name = arg.slice(2); + const value = argv[index + 1]; + if (value && !value.startsWith('-')) { + values[name] = value; + index++; + } else { + values[name] = true; + } } - if (typeof values.url !== 'string' || !/^https?:\/\//i.test(values.url)) throw new ArgumentError('--url must be an http or https URL'); - const int = (name: string, fallback: number) => { const value = values[name]; const number = value === undefined ? fallback : Number(value); if (!Number.isInteger(number) || number < 0) throw new ArgumentError(`--${name} must be a non-negative integer`); return number; }; - return { url: values.url, timeoutSeconds: int('timeout', 30), maxChars: int('max-chars', 50000), allowPrivate: values['allow-private'] === true || values['allow-private'] === 'true' }; + if (typeof values.url !== 'string' || !/^https?:\/\//i.test(values.url)) { + throw new ArgumentError('--url must be an http or https URL'); + } + const int = (name: string) => { + const fallback = defaultInt(name); + const value = values[name]; + if (value === true) throw new ArgumentError(`--${name} requires a value`); + const number = value === undefined ? fallback : Number(value); + if (!Number.isInteger(number) || number < 0) { + throw new ArgumentError(`--${name} must be a non-negative integer`); + } + return number; + }; + return { + url: values.url, + timeoutSeconds: int('timeout'), + maxChars: int('max-chars'), + allowPrivate: values['allow-private'] === true || values['allow-private'] === 'true', + }; } -export async function runClientOwnedWebFetch(argv: readonly string[], dependencies: { webFetch?: typeof webFetch; stdout?: NodeJS.WritableStream } = {}): Promise { +export async function runClientOwnedWebFetch( + argv: readonly string[], + dependencies: { + webFetch?: typeof webFetch; + stdout?: NodeJS.WritableStream; + } = {}, +): Promise { + const stdout = dependencies.stdout ?? process.stdout; + const format = requestedFormat(argv); + if (wantsHelp(argv)) { + stdout.write(format === 'yaml' || format === 'json' + ? renderStructuredHelp(commandHelpData(toPresentableCommand(webFetchCommand)), format) + : formatWebFetchHelp()); + return; + } const result = await (dependencies.webFetch ?? webFetch)(clientOptions(argv)); - (dependencies.stdout ?? process.stdout).write(`${formatWebFetchMarkdown(result)}\n`); + stdout.write(format === undefined || format === 'md' + ? `${formatWebFetchMarkdown(result)}\n` + : formatOutput(result, { fmt: format, fmtExplicit: true })); }