fix: make client-owned web fetch discoverable (#252)

cli-manifest.json

@@ -1,4 +1,45 @@
 [
+  {
+    "site": "web",
+    "name": "fetch",
+    "description": "Fetch a URL locally without launching a browser",
+    "access": "read",
+    "strategy": "public",
+    "browser": false,
+    "args": [
+      {
+        "name": "url",
+        "type": "string",
+        "required": true,
+        "help": "http(s) URL to fetch"
+      },
+      {
+        "name": "timeout",
+        "type": "int",
+        "default": 30,
+        "required": false,
+        "help": "Fetch budget in seconds"
+      },
+      {
+        "name": "max-chars",
+        "type": "int",
+        "default": 50000,
+        "required": false,
+        "help": "Maximum characters of extracted content"
+      },
+      {
+        "name": "allow-private",
+        "type": "boolean",
+        "default": false,
+        "required": false,
+        "help": "Allow private/loopback addresses"
+      }
+    ],
+    "defaultFormat": "md",
+    "type": "js",
+    "modulePath": "web/fetch.js",
+    "sourceFile": "web/fetch.js"
+  },
   {
     "site": "web",
     "name": "fetch-browser",

clis/web/fetch.js

@@ -1 +1,11 @@
-import '@agentrhq/webcmd/fetch/command';
+/**
+ * Discovery entry for client-owned `web fetch`.
+ *
+ * Execution stays on the main.ts fast path so hosted mode never cloud-routes
+ * this command. This module exists so build-manifest, `webcmd list`,
+ * completions, and Commander help can see the same registration as the
+ * always-available fast path.
+ */
+import { makeWebFetchCommand } from '@agentrhq/webcmd/fetch/command';
+
+export const command = makeWebFetchCommand();

src/discovery.ts

@@ -38,8 +38,8 @@ export const USER_WEBCMD_DIR = getUserWebcmdDir();
 export const USER_CLIS_DIR = getUserClisDir();
 /** Plugins directory: ~/.webcmd/plugins/ */
 export const PLUGINS_DIR = getPluginsDir();
-/** Matches files that register commands via cli() or lifecycle hooks */
-const PLUGIN_MODULE_PATTERN = /\b(?:cli|registerSiteAuthCommands|onStartup|onBeforeExecute|onAfterExecute)\s*\(/;
+/** Matches files that register commands via cli() / factories or lifecycle hooks */
+const PLUGIN_MODULE_PATTERN = /\b(?:cli|registerSiteAuthCommands|onStartup|onBeforeExecute|onAfterExecute)\s*\(|\bmake[A-Z]\w*Command\s*\(/;
 
 function parseStrategy(rawStrategy: string | undefined, fallback: Strategy = Strategy.COOKIE): Strategy {
   if (!rawStrategy) return fallback;

src/fetch/command.test.ts

@@ -1,13 +1,119 @@
 import { describe, expect, it, vi } from 'vitest';
-import { formatWebFetchMarkdown, runClientOwnedWebFetch } from './command.js';
+import {
+  formatWebFetchHelp,
+  formatWebFetchMarkdown,
+  runClientOwnedWebFetch,
+  WEB_FETCH_ARGS,
+  webFetchCommand,
+} from './command.js';
 
 describe('web fetch command', () => {
   it('renders fetch metadata before content', () => {
-    expect(formatWebFetchMarkdown({ status: 200, requestedUrl: 'https://a', finalUrl: 'https://b', contentType: 'text/plain', tier: 'plain', title: 'T', extractionSource: 'raw', truncated: false, content: 'body' })).toContain('Source: https://a');
+    expect(formatWebFetchMarkdown({
+      status: 200,
+      requestedUrl: 'https://a',
+      finalUrl: 'https://b',
+      contentType: 'text/plain',
+      tier: 'plain',
+      title: 'T',
+      extractionSource: 'raw',
+      truncated: false,
+      content: 'body',
+    })).toContain('Source: https://a');
   });
+
   it('runs the client-owned command without Cloud routing', async () => {
-    const webFetch = vi.fn().mockResolvedValue({ status: 200, requestedUrl: 'https://a', finalUrl: 'https://a', contentType: 'text/plain', tier: 'plain', title: '', extractionSource: 'raw', truncated: false, content: 'ok' });
-    await runClientOwnedWebFetch(['web', 'fetch', '--url', 'https://a'], { webFetch, stdout: { write: vi.fn() } as never });
+    const webFetch = vi.fn().mockResolvedValue({
+      status: 200,
+      requestedUrl: 'https://a',
+      finalUrl: 'https://a',
+      contentType: 'text/plain',
+      tier: 'plain',
+      title: '',
+      extractionSource: 'raw',
+      truncated: false,
+      content: 'ok',
+    });
+    await runClientOwnedWebFetch(['web', 'fetch', '--url', 'https://a'], {
+      webFetch,
+      stdout: { write: vi.fn() } as never,
+    });
     expect(webFetch).toHaveBeenCalledOnce();
   });
+
+  it('keeps discovery args aligned with the registered command', () => {
+    expect(webFetchCommand.site).toBe('web');
+    expect(webFetchCommand.name).toBe('fetch');
+    expect(webFetchCommand.browser).toBe(false);
+    expect(webFetchCommand.args).toEqual(WEB_FETCH_ARGS);
+  });
+
+  it('prints real help for -h and --help without requiring --url', async () => {
+    for (const flag of ['-h', '--help'] as const) {
+      const write = vi.fn();
+      const webFetch = vi.fn();
+      await runClientOwnedWebFetch(['web', 'fetch', flag], {
+        webFetch,
+        stdout: { write } as never,
+      });
+      expect(webFetch).not.toHaveBeenCalled();
+      expect(write).toHaveBeenCalledOnce();
+      const help = String(write.mock.calls[0]![0]);
+      expect(help).toContain('Usage:');
+      expect(help).toContain('web fetch');
+      expect(help).toContain('--url');
+      expect(help).toContain('--timeout');
+      expect(help).toBe(formatWebFetchHelp());
+    }
+  });
+
+  it('honours -f for output instead of always printing markdown', async () => {
+    const result = {
+      status: 200,
+      requestedUrl: 'https://a',
+      finalUrl: 'https://a',
+      contentType: 'text/plain',
+      tier: 'plain',
+      title: '',
+      extractionSource: 'raw',
+      truncated: false,
+      content: 'ok',
+    };
+    const write = vi.fn();
+    await runClientOwnedWebFetch(['web', 'fetch', '--url', 'https://a', '-f', 'json'], {
+      webFetch: vi.fn().mockResolvedValue(result),
+      stdout: { write } as never,
+    });
+    expect(JSON.parse(String(write.mock.calls[0]![0]))).toMatchObject({ content: 'ok' });
+  });
+
+  it('serves structured help for --help -f yaml', async () => {
+    const write = vi.fn();
+    await runClientOwnedWebFetch(['web', 'fetch', '--help', '-f', 'yaml'], {
+      webFetch: vi.fn(),
+      stdout: { write } as never,
+    });
+    expect(String(write.mock.calls[0]![0])).toContain('name: fetch');
+  });
+
+  it('rejects an unsupported --format instead of silently printing a table', async () => {
+    await expect(runClientOwnedWebFetch(['web', 'fetch', '--url', 'https://a', '-f', 'xml'], {
+      webFetch: vi.fn(),
+      stdout: { write: vi.fn() } as never,
+    })).rejects.toThrow('--format must be one of');
+  });
+
+  it('rejects a flag-shaped value for --timeout instead of coercing it', async () => {
+    await expect(runClientOwnedWebFetch(['web', 'fetch', '--url', 'https://a', '--timeout', '-5'], {
+      webFetch: vi.fn(),
+      stdout: { write: vi.fn() } as never,
+    })).rejects.toThrow('--timeout requires a value');
+  });
+
+  it('rejects missing --url with ArgumentError when help is not requested', async () => {
+    await expect(runClientOwnedWebFetch(['web', 'fetch'], {
+      webFetch: vi.fn(),
+      stdout: { write: vi.fn() } as never,
+    })).rejects.toThrow('--url must be an http or https URL');
+  });
 });

src/fetch/command.ts

@@ -1,37 +1,146 @@
-import { cli, Strategy } from '../registry.js';
+import { cli, Strategy, type Arg, type CliCommand } from '../registry.js';
 import { ArgumentError } from '../errors.js';
+import { commandHelpData, formatCommandHelp, toPresentableCommand } from '../command-presentation.js';
+import { renderStructuredHelp } from '../help.js';
+import { formatOutput } from '../output.js';
 import { webFetch, type WebFetchOptions, type WebFetchResult } from './client.js';
 
-export const webFetchCommand = cli({
-  site: 'web', name: 'fetch', access: 'read', strategy: Strategy.PUBLIC, browser: false,
-  description: 'Fetch a URL locally without launching a browser', defaultFormat: 'md',
-  args: [
-    { name: 'url', type: 'string', required: true },
-    { name: 'timeout', type: 'int', default: 30 },
-    { name: 'max-chars', type: 'int', default: 50000 },
-    { name: 'allow-private', type: 'boolean', default: false },
-  ],
-  func: async kwargs => webFetch({ url: String(kwargs.url), timeoutSeconds: Number(kwargs.timeout ?? 30), maxChars: Number(kwargs['max-chars'] ?? 50000), allowPrivate: kwargs['allow-private'] === true }),
-});
+/** Single source of truth for discovery, Commander help, and the client-owned fast path. */
+export const WEB_FETCH_ARGS: Arg[] = [
+  { name: 'url', type: 'string', required: true, help: 'http(s) URL to fetch' },
+  { name: 'timeout', type: 'int', default: 30, help: 'Fetch budget in seconds' },
+  { name: 'max-chars', type: 'int', default: 50000, help: 'Maximum characters of extracted content' },
+  { name: 'allow-private', type: 'boolean', default: false, help: 'Allow private/loopback addresses' },
+];
+
+const DEFAULT_TIMEOUT = 30;
+const DEFAULT_MAX_CHARS = 50000;
+
+let registered: CliCommand | undefined;
+
+/**
+ * Register (or return) the builtin `web fetch` command.
+ * Called from clis/web/fetch.js so build-manifest / filesystem discovery see it.
+ * Safe to call more than once — returns the same command object.
+ */
+export function makeWebFetchCommand(): CliCommand {
+  if (registered) return registered;
+  registered = cli({
+    site: 'web',
+    name: 'fetch',
+    access: 'read',
+    strategy: Strategy.PUBLIC,
+    browser: false,
+    description: 'Fetch a URL locally without launching a browser',
+    defaultFormat: 'md',
+    args: WEB_FETCH_ARGS,
+    func: async kwargs => webFetch(kwargsToOptions(kwargs)),
+  });
+  return registered;
+}
+
+/** Eager registration for consumers that import the command object directly. */
+export const webFetchCommand = makeWebFetchCommand();
 
 export function formatWebFetchMarkdown(result: WebFetchResult): string {
   return [`# ${result.title || 'Fetched content'}`, '', `Source: ${result.requestedUrl}`, `Final URL: ${result.finalUrl}`, `Content type: ${result.contentType || 'unknown'}`, `Extraction: ${result.extractionSource}`, '', result.content].join('\n');
 }
 
+export function formatWebFetchHelp(): string {
+  return formatCommandHelp(toPresentableCommand(webFetchCommand));
+}
+
+function defaultInt(name: string): number {
+  if (name === 'timeout') return DEFAULT_TIMEOUT;
+  if (name === 'max-chars') return DEFAULT_MAX_CHARS;
+  return 0;
+}
+
+function kwargsToOptions(kwargs: Record<string, unknown>): WebFetchOptions {
+  return {
+    url: String(kwargs.url),
+    timeoutSeconds: Number(kwargs.timeout ?? DEFAULT_TIMEOUT),
+    maxChars: Number(kwargs['max-chars'] ?? DEFAULT_MAX_CHARS),
+    allowPrivate: kwargs['allow-private'] === true,
+  };
+}
+
+function wantsHelp(argv: readonly string[]): boolean {
+  return argv.slice(2).some(arg => arg === '-h' || arg === '--help');
+}
+
+/** Formats advertised by the common-options block the help text prints. */
+const OUTPUT_FORMATS = ['table', 'plain', 'json', 'yaml', 'md', 'csv'] as const;
+
+/** Reads -f/--format in the same shapes Commander accepts, so help cannot over-promise. */
+function requestedFormat(argv: readonly string[]): string | undefined {
+  for (let index = 2; index < argv.length; index++) {
+    const arg = argv[index]!;
+    let value: string | undefined;
+    if (arg === '-f' || arg === '--format') value = argv[index + 1];
+    else if (arg.startsWith('--format=')) value = arg.slice('--format='.length);
+    else if (arg.startsWith('-f') && arg.length > 2) value = arg.slice(2);
+    else continue;
+    if (value === undefined || !OUTPUT_FORMATS.includes(value as typeof OUTPUT_FORMATS[number])) {
+      throw new ArgumentError(`--format must be one of: ${OUTPUT_FORMATS.join(', ')}`);
+    }
+    return value;
+  }
+  return undefined;
+}
+
 function clientOptions(argv: readonly string[]): WebFetchOptions {
   const values: Record<string, string | boolean> = {};
   for (let index = 2; index < argv.length; index++) {
     const arg = argv[index]!;
     if (!arg.startsWith('--')) continue;
-    const name = arg.slice(2); const value = argv[index + 1];
-    if (value && !value.startsWith('--')) { values[name] = value; index++; } else values[name] = true;
+    const name = arg.slice(2);
+    const value = argv[index + 1];
+    if (value && !value.startsWith('-')) {
+      values[name] = value;
+      index++;
+    } else {
+      values[name] = true;
+    }
   }
-  if (typeof values.url !== 'string' || !/^https?:\/\//i.test(values.url)) throw new ArgumentError('--url must be an http or https URL');
-  const int = (name: string, fallback: number) => { const value = values[name]; const number = value === undefined ? fallback : Number(value); if (!Number.isInteger(number) || number < 0) throw new ArgumentError(`--${name} must be a non-negative integer`); return number; };
-  return { url: values.url, timeoutSeconds: int('timeout', 30), maxChars: int('max-chars', 50000), allowPrivate: values['allow-private'] === true || values['allow-private'] === 'true' };
+  if (typeof values.url !== 'string' || !/^https?:\/\//i.test(values.url)) {
+    throw new ArgumentError('--url must be an http or https URL');
+  }
+  const int = (name: string) => {
+    const fallback = defaultInt(name);
+    const value = values[name];
+    if (value === true) throw new ArgumentError(`--${name} requires a value`);
+    const number = value === undefined ? fallback : Number(value);
+    if (!Number.isInteger(number) || number < 0) {
+      throw new ArgumentError(`--${name} must be a non-negative integer`);
+    }
+    return number;
+  };
+  return {
+    url: values.url,
+    timeoutSeconds: int('timeout'),
+    maxChars: int('max-chars'),
+    allowPrivate: values['allow-private'] === true || values['allow-private'] === 'true',
+  };
 }
 
-export async function runClientOwnedWebFetch(argv: readonly string[], dependencies: { webFetch?: typeof webFetch; stdout?: NodeJS.WritableStream } = {}): Promise<void> {
+export async function runClientOwnedWebFetch(
+  argv: readonly string[],
+  dependencies: {
+    webFetch?: typeof webFetch;
+    stdout?: NodeJS.WritableStream;
+  } = {},
+): Promise<void> {
+  const stdout = dependencies.stdout ?? process.stdout;
+  const format = requestedFormat(argv);
+  if (wantsHelp(argv)) {
+    stdout.write(format === 'yaml' || format === 'json'
+      ? renderStructuredHelp(commandHelpData(toPresentableCommand(webFetchCommand)), format)
+      : formatWebFetchHelp());
+    return;
+  }
   const result = await (dependencies.webFetch ?? webFetch)(clientOptions(argv));
-  (dependencies.stdout ?? process.stdout).write(`${formatWebFetchMarkdown(result)}\n`);
+  stdout.write(format === undefined || format === 'md'
+    ? `${formatWebFetchMarkdown(result)}\n`
+    : formatOutput(result, { fmt: format, fmtExplicit: true }));
 }