From e48f78ea7ea185c417932f67da06115a742942fe Mon Sep 17 00:00:00 2001 From: Eduardo Jose Costa <59846713+EduCosta85@users.noreply.github.com> Date: Sun, 20 Sep 2026 18:15:41 -0300 Subject: [PATCH 1/2] feat: add push-to-talk voice dictation with Parakeet Support push-to-talk voice dictation directly in the composer. Speech clips recorded via MediaRecorder are validated, decoded, and forwarded over JSON-RPC to a local speech-to-text server exposing an OpenAI-compatible transcription endpoint (such as NVIDIA Parakeet running on mlx-audio). - protocol: define dictation/status and dictation/transcribe in JSON-RPC schema - config: add user-owned DictationConfig with endpoint, model, timeout, and cap settings - core: implement pure audio container sniffing and OpenAI-compatible client - application: implement DictationService with model egress policy checks - desktop: add useDictation hook and composer push-to-talk microphone button - docs & tests: add complete unit test suites and setup documentation --- app_server/dispatcher.py | 20 + app_server/protocol/methods.py | 2 + app_server/protocol/retry.py | 3 + core/application/application.py | 2 + core/application/dictation_service.py | 227 +++++++++ core/application/errors.py | 46 ++ core/config.py | 131 +++++- core/dictation/__init__.py | 41 ++ core/dictation/audio.py | 186 ++++++++ core/dictation/client.py | 144 ++++++ desktop/src/app/i18n.ts | 5 + .../features/execution/Composer.module.css | 39 ++ desktop/src/features/execution/Composer.tsx | 84 ++++ .../features/execution/useDictation.test.ts | 377 +++++++++++++++ .../src/features/execution/useDictation.ts | 313 +++++++++++++ desktop/src/generated/app-server.ts | 18 + docs/guide/README.md | 1 + docs/guide/dictation.md | 65 +++ protocol/app-server.schema.json | 79 +++- tests/test_config_layering.py | 60 +++ tests/test_dictation.py | 408 +++++++++++++++++ tests/test_dictation_service.py | 429 ++++++++++++++++++ 22 files changed, 2667 insertions(+), 13 deletions(-) create mode 100644 core/application/dictation_service.py create mode 100644 core/dictation/__init__.py create mode 100644 core/dictation/audio.py create mode 100644 core/dictation/client.py create mode 100644 desktop/src/features/execution/useDictation.test.ts create mode 100644 desktop/src/features/execution/useDictation.ts create mode 100644 docs/guide/dictation.md create mode 100644 tests/test_dictation.py create mode 100644 tests/test_dictation_service.py diff --git a/app_server/dispatcher.py b/app_server/dispatcher.py index 93d49253a..48b674975 100644 --- a/app_server/dispatcher.py +++ b/app_server/dispatcher.py @@ -321,6 +321,8 @@ def __init__( rpc_methods.TERMINAL_CLOSE: self._terminal_close, rpc_methods.TEST_DISCOVER: self._test_discover, rpc_methods.TEST_RUN: self._test_run, + rpc_methods.DICTATION_STATUS: self._dictation_status, + rpc_methods.DICTATION_TRANSCRIBE: self._dictation_transcribe, } @property @@ -1058,6 +1060,24 @@ def _turn_list(self, params: Params) -> dict[str, Any]: "hasMore": len(turns) > limit, } + def _dictation_status(self, params: Params) -> dict[str, Any]: + params.only("projectId") + return self.application.dictation.status( + project_id=params.string("projectId", required=False) + ) + + def _dictation_transcribe(self, params: Params) -> dict[str, Any]: + params.only("audio", "mimeType", "language", "projectId") + language = ( + params.nullable_string("language") if "language" in params.values else None + ) + return self.application.dictation.transcribe( + audio=str(params.string("audio")), + mime_type=str(params.string("mimeType")), + language=language, + project_id=params.string("projectId", required=False), + ) + def _model_reasoning(self, params: Params) -> dict[str, Any]: params.only("projectId", "connectionId", "model") capabilities = self.application.llm.model_reasoning( diff --git a/app_server/protocol/methods.py b/app_server/protocol/methods.py index 0f073f961..504621e42 100644 --- a/app_server/protocol/methods.py +++ b/app_server/protocol/methods.py @@ -105,3 +105,5 @@ TERMINAL_CLOSE = "terminal/close" TEST_DISCOVER = "test/discover" TEST_RUN = "test/run" +DICTATION_STATUS = "dictation/status" +DICTATION_TRANSCRIBE = "dictation/transcribe" diff --git a/app_server/protocol/retry.py b/app_server/protocol/retry.py index bbb1a7893..58f671354 100644 --- a/app_server/protocol/retry.py +++ b/app_server/protocol/retry.py @@ -17,6 +17,9 @@ "mcp/list", "mcp/presets", "diagnostics/read", + # Status is a pure config read; transcription is deliberately absent — + # it is a long, expensive call that must be re-issued by the user. + "dictation/status", "automation/list", "automation/runs", "thread/list", diff --git a/core/application/application.py b/core/application/application.py index 770a5edcc..ffdbcd45b 100644 --- a/core/application/application.py +++ b/core/application/application.py @@ -17,6 +17,7 @@ from core.application.automation_scheduler import AutomationScheduler from core.application.automation_service import AutomationService from core.application.diagnostics_service import DiagnosticsService +from core.application.dictation_service import DictationService from core.application.errors import UpgradeRequiresExclusiveAccessError from core.application.event_service import ( DEFAULT_RELAY_BATCH_SIZE, @@ -100,6 +101,7 @@ def __init__( self.projects, credential_store=self.credentials, ) + self.dictation = DictationService(self.projects) self.skill_hosts = SkillWorkspaceRegistry() self.plugins = PluginService(LocalPluginHost(self.skill_hosts)) effective_session_factory = session_factory or DefaultAgentSessionFactory() diff --git a/core/application/dictation_service.py b/core/application/dictation_service.py new file mode 100644 index 000000000..f575668ec --- /dev/null +++ b/core/application/dictation_service.py @@ -0,0 +1,227 @@ +"""Application service for voice dictation (prompt-box microphone input). + +Why this is a service and not a tool call: a transcript is not agent input by +itself. The user dictates into the composer, sees the text, edits it, and sends +it — so the audio is transcribed *before* a Turn exists and the result never +reaches the model on its own. The service therefore lives on the input path, +not in the agent loop. + +Responsibilities, in order: + +1. **Is dictation configured at all?** An absent ``dictation`` block means the + feature is off and the app server reports it as unavailable, so the prompt + box never offers a microphone that cannot work. +2. **Is the endpoint allowed?** A recording is a verbatim copy of what the user + said, so the endpoint receiving it is a trust boundary, evaluated against the + same ``providers.egress`` policy as model traffic. +3. **What are the bytes?** Format and size checks stay in + :mod:`core.dictation.audio`, which is pure and testable on its own. +4. **Translate failures.** Every outcome becomes either a transcript or a stable + application error; no bare transport exception escapes this module. +""" + +from __future__ import annotations + +import os +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from loguru import logger + +from core.application.config_store import ConfigStore +from core.application.errors import ( + DictationNotConfiguredError, + DictationUnavailableError, + InvalidArgumentError, +) +from core.application.project_service import ProjectService +from core.config import ( + DeepCodeConfig, + DictationConfig, + load_config, + load_config_for_workspace, +) +from core.dictation.audio import ( + UnsupportedAudioError, + canonical_mime_type, + decode_audio, +) +from core.dictation.client import SpeechToTextClient, TranscriptionFailed +from core.providers.egress import ( + WARN, + evaluate_provider_egress, + resolve_egress_policy, +) + +#: Builds the client for one request from the resolved endpoint config, the API +#: key (or ``None``), and the language hint to send. Injectable so tests can +#: substitute a client without patching process-global state. +ClientFactory = Callable[[DictationConfig, str | None, str | None], SpeechToTextClient] + + +def _default_client_factory( + config: DictationConfig, + api_key: str | None, + language: str | None, +) -> SpeechToTextClient: + return SpeechToTextClient( + config.endpoint, + config.model, + language=language, + api_key=api_key, + timeout_seconds=config.timeout_seconds, + ) + + +class DictationService: + """Resolve dictation config and transcribe one clip per request.""" + + def __init__( + self, + projects: ProjectService | None = None, + *, + config_store: ConfigStore | None = None, + client_factory: ClientFactory | None = None, + ) -> None: + self.projects = projects + self.config_store = config_store or ConfigStore() + self._client_factory = client_factory or _default_client_factory + + def status(self, project_id: str | None = None) -> dict[str, Any]: + """Report the capability without revealing whether a secret resolves. + + ``model`` and ``maxAudioSeconds`` describe the *configured* endpoint, so + they are ``None`` exactly when ``available`` is ``False``. The status + deliberately says nothing about credentials: whether a bearer token + happens to be set in the environment is not a fact the renderer needs, + and the failure would surface on the first clip anyway. + """ + + config = self._config(project_id).dictation + if config is None: + return {"available": False, "model": None, "maxAudioSeconds": None} + return { + "available": True, + "model": config.model, + "maxAudioSeconds": config.max_audio_seconds, + } + + def transcribe( + self, + *, + audio: str, + mime_type: str, + language: str | None = None, + project_id: str | None = None, + ) -> dict[str, Any]: + """Transcribe one base64 clip into ``{"text", "model"}``. + + ``language`` is a per-clip hint used only when ``dictation.language`` is + unset: a value in the config file is an explicit user choice and wins + over whatever a client offers for a single request. + """ + + loaded = self._config(project_id) + config = loaded.dictation + if config is None: + raise DictationNotConfiguredError() + + denial = self._egress_denial(config, loaded) + if denial is not None: + raise DictationUnavailableError(denial, retryable=False) + + try: + data, filename = decode_audio(audio, mime_type) + except UnsupportedAudioError as exc: + raise InvalidArgumentError(str(exc)) from exc + + client = self._client_factory( + config, self._api_key(config), config.language or (language or None) + ) + try: + text = client.transcribe( + data, + filename=filename, + mime_type=canonical_mime_type(mime_type), + ) + except TranscriptionFailed as exc: + raise DictationUnavailableError(str(exc), retryable=exc.retryable) from exc + logger.trace( + "Dictation transcribed: model={} bytes={} chars={}", + config.model, + len(data), + len(text), + ) + return {"text": text, "model": config.model} + + # -- config and secrets ------------------------------------------------- + + def _config(self, project_id: str | None) -> DeepCodeConfig: + """Load the config that applies to this request. + + Same precedence as ``LLMConfigurationService._config``: an unscoped + request reads the user config, a project-scoped one reads the layered + user + project config. The project layer cannot carry a ``dictation`` + block (``_project_runtime_layer`` drops it), so a project-scoped read + never changes *whether* dictation is on, only honours the workspace. + """ + + if project_id is None: + return load_config(config_path=self.config_store.path) + if self.projects is None: + raise InvalidArgumentError( + "project-scoped dictation settings are unavailable" + ) + project = self.projects.read(project_id) + workspace = Path(project.canonical_path).resolve(strict=False) + return load_config_for_workspace(workspace) + + def _api_key(self, config: DictationConfig) -> str | None: + """Read the bearer token from the environment variable the user named.""" + + if not config.api_key_env: + return None + key = os.environ.get(config.api_key_env) + if not key: + raise DictationNotConfiguredError( + f"dictation.apiKeyEnv names {config.api_key_env!r}, which is not " + "set in the environment" + ) + return key + + # -- egress ------------------------------------------------------------- + + def _egress_denial( + self, config: DictationConfig, loaded: DeepCodeConfig + ) -> str | None: + """Return a denial message, or ``None`` when the endpoint may be used. + + Mirrors ``make_llm_provider``: the decision is a hostname comparison, so + it costs nothing per request, and ``warn`` mode records the same message + without blocking. The message is written here rather than taken from + ``EgressDecision.reason`` because that text talks about + ``providers..apiBase``, and the operator needs to be pointed at + the key this endpoint actually lives under. + """ + + policy = resolve_egress_policy(loaded) + decision = evaluate_provider_egress( + config.endpoint, + endpoint_class="dictation", + allowed_domains=policy.allowed_domains, + blocked_domains=policy.blocked_domains, + ) + if decision.allowed: + logger.trace( + "Dictation egress ok: host={} model={}", decision.host, config.model + ) + return None + message = ( + f"dictation endpoint host {decision.host or config.endpoint!r} is not " + "allowed by the model egress policy (providers.egress)" + ) + if policy.mode == WARN: + logger.warning(message) + return None + return message diff --git a/core/application/errors.py b/core/application/errors.py index 74b11afed..4ea77993a 100644 --- a/core/application/errors.py +++ b/core/application/errors.py @@ -269,3 +269,49 @@ class TerminalNotFoundError(ApplicationError): class NotSupportedApplicationError(ApplicationError): code = "NOT_SUPPORTED" + + +class DictationNotConfiguredError(ApplicationError): + """Voice input was requested but no ``dictation`` block exists. + + Permanent as long as the config is: the endpoint is user-owned and cannot + be supplied per request, so this is not worth retrying from the UI. + """ + + code = "DICTATION_NOT_CONFIGURED" + + def __init__(self, message: str = "dictation is not configured") -> None: + super().__init__( + message, + user_message=( + "Voice input is not configured. Add a dictation block with the " + "Parakeet endpoint to your DeepCode config." + ), + ) + + +class DictationUnavailableError(ApplicationError): + """The configured endpoint could not produce a transcript. + + Covers every runtime failure on the remote side: unreachable, timed out, + refused, or answered with an error. ``retryable`` mirrors the client's + judgement (5xx and timeouts yes, 4xx no) rather than being hardcoded, so + the UI can offer "try again" only when it would be honest. + """ + + code = "DICTATION_UNAVAILABLE" + + def __init__( + self, + message: str, + *, + retryable: bool = True, + ) -> None: + super().__init__( + message, + user_message=( + "Voice input could not be transcribed. Check that the dictation " + "endpoint is running, then try again." + ), + ) + self.retryable = retryable diff --git a/core/config.py b/core/config.py index 3ecd9d3cd..ab3e92c26 100644 --- a/core/config.py +++ b/core/config.py @@ -20,7 +20,7 @@ - :class:`AgentDefaults`, :class:`AgentPhase`, :class:`ProviderConfig`, :class:`ToolsConfig`, :class:`WorkspaceConfig`, :class:`DocumentSegmentationConfig`, :class:`LoggerConfig`, - :class:`LLMLoggerConfig` – sub-models + :class:`DictationConfig`, :class:`LLMLoggerConfig` – sub-models - :func:`load_config` – read JSON and resolve ``${ENV_VAR}`` references - :func:`make_llm_provider` – build the right :class:`core.providers.base.LLMProvider` for a workflow phase @@ -34,6 +34,7 @@ from dataclasses import dataclass from pathlib import Path from typing import Any, Literal +from urllib.parse import urlsplit from loguru import logger from pydantic import AliasChoices, BaseModel, ConfigDict, Field, model_validator @@ -121,10 +122,22 @@ class AgentPhase(_Base): reasoning_effort: str | None = None +# Named per-phase blocks of ``agents``. A phase listed here may override any +# :class:`AgentPhase` field; anything else resolves straight to ``defaults``. +# Settings writes and the runtime both read this one list, so a new phase can +# never be configurable in one place and rejected in the other. +AGENT_PHASES: frozenset[str] = frozenset({"planning", "implementation", "subagent"}) + + class AgentsConfig(_Base): defaults: AgentDefaults = Field(default_factory=AgentDefaults) planning: AgentPhase = Field(default_factory=AgentPhase) implementation: AgentPhase = Field(default_factory=AgentPhase) + # Delegation tier. Sub-agents are separate conversations, so routing them to + # a cheaper model neither rides nor invalidates the parent's provider-side + # prefix cache. Unset (the default) means a spawned child inherits the + # parent Session's resolved profile exactly as before. + subagent: AgentPhase = Field(default_factory=AgentPhase) @dataclass(frozen=True, slots=True) @@ -405,6 +418,73 @@ class WorkspaceConfig(_Base): max_input_mb: int = 100 +class DictationConfig(_Base): + """Voice input for the prompt box, transcribed by a Parakeet-style server. + + Dictation is opt-in by presence: with no ``dictation`` block the prompt box + offers no microphone and the app server reports the capability as + unavailable, so nothing changes for an install that does not want it. + + ``endpoint`` is the *trust boundary* of this feature — a voice recording is + a verbatim copy of what the user said, so the host that terminates the + connection reads all of it (and is subject to ``providers.egress`` like any + other model endpoint). It speaks the OpenAI transcription contract, which + local Parakeet servers (``mlx_audio.server``) implement, so the audio can + stay on the machine: + + - ``endpoint``: base URL without a trailing slash, e.g. + ``http://127.0.0.1:8000/v1``. The client posts to + ``/audio/transcriptions``. + - ``model``: the model name the server should transcribe with, e.g. + ``mlx-community/parakeet-tdt-0.6b-v3``. Required: the server decides what + an unknown or empty name means, so guessing one here would silently pick + a model the user did not choose. + - ``language``: optional pass-through for endpoints that take a language + hint. Parakeet v3 detects the language itself, so ``None`` (the default) + sends no hint at all. + - ``apiKeyEnv``: name of the environment variable holding the bearer token. + The secret is deliberately never read from the config file — a project + config is a file the user's repository controls. + - ``timeoutSeconds``: whole-request budget. The first request usually loads + the model on the server, so the default is generous. + - ``maxAudioSeconds``: cap the clients enforce by stopping the recording. + + A project-level config cannot set any of this (see + :func:`_project_runtime_layer`): a repository must not be able to redirect + the microphone. + """ + + endpoint: str = Field(min_length=1) + model: str = Field(min_length=1) + language: str | None = None + api_key_env: str | None = None + timeout_seconds: float = Field(default=60.0, gt=0, le=600) + max_audio_seconds: int = Field(default=120, ge=1, le=600) + + @model_validator(mode="after") + def validate_endpoint(self): + endpoint = self.endpoint.strip() + if not endpoint: + raise ValueError("dictation.endpoint must not be empty") + if not endpoint.startswith(("http://", "https://")): + raise ValueError( + "dictation.endpoint must be an absolute http:// or https:// URL" + ) + parsed = urlsplit(endpoint) + if not parsed.netloc: + raise ValueError("dictation.endpoint must name a host") + if parsed.username or parsed.password: + raise ValueError( + "dictation.endpoint must not embed credentials; use apiKeyEnv" + ) + if parsed.query or parsed.fragment: + raise ValueError("dictation.endpoint must not carry a query or fragment") + # Normalized once here so the request path is a plain concatenation and + # a trailing slash cannot produce `//audio/transcriptions`. + self.endpoint = endpoint.rstrip("/") + return self + + class DocumentSegmentationConfig(_Base): enabled: bool = True size_threshold_chars: int = 50000 @@ -497,6 +577,11 @@ class DeepCodeConfig(BaseSettings): skills: SkillsConfig = Field(default_factory=SkillsConfig) security: SecurityConfig = Field(default_factory=SecurityConfig) workspace: WorkspaceConfig = Field(default_factory=WorkspaceConfig) + # Absent means "voice input is off"; present means the user picked an + # endpoint for it. A default *instance* could not tell those apart, and a + # disabled microphone that still ships an endpoint would be a config that + # reads as configured while doing nothing. + dictation: DictationConfig | None = None document_segmentation: DocumentSegmentationConfig = Field( default_factory=DocumentSegmentationConfig, validation_alias=AliasChoices("documentSegmentation", "document_segmentation"), @@ -547,16 +632,32 @@ def mcp_servers(self) -> dict[str, MCPServerConfig]: # ---- phase resolution ---- + def phase_override(self, phase: str = "default") -> AgentPhase | None: + """Return ``phase``'s configured block, or ``None`` for ``defaults``.""" + name = (phase or "default").strip().lower() + if name not in AGENT_PHASES: + return None + return getattr(self.agents, name) + + def phase_is_overridden(self, phase: str = "default") -> bool: + """True when ``phase`` changes anything relative to ``agents.defaults``. + + Optional phases (``subagent``) must leave every existing config on its + previous path: a caller that would otherwise resolve a profile for such + a phase checks this first and inherits the parent's profile verbatim + instead of resolving an equivalent-but-distinct one. + """ + override = self.phase_override(phase) + if override is None: + return False + return any( + getattr(override, name) is not None for name in AgentPhase.model_fields + ) + def resolve_phase(self, phase: str = "default") -> ResolvedAgentSettings: """Merge ``agents.defaults`` with the phase override (if any).""" defaults = self.agents.defaults - override: AgentPhase | None - if phase == "planning": - override = self.agents.planning - elif phase == "implementation": - override = self.agents.implementation - else: - override = None + override = self.phase_override(phase) def _pick(name: str) -> Any: if override is not None: @@ -770,12 +871,18 @@ def _project_runtime_layer(raw: dict[str, Any]) -> dict[str, Any]: environment. For legacy compatibility a project-level literal ``apiKey`` remains readable against the registry provider's trusted default endpoint; all routing fields are discarded. + + ``dictation`` is discarded for the same reason: its endpoint receives raw + microphone audio, so a cloned repository must not be able to point it at a + host of its own choosing. """ + sanitized = dict(raw) + if sanitized.pop("dictation", None) is not None: + logger.trace("Ignoring project dictation endpoint; dictation is user-owned") providers = raw.get("providers") if not isinstance(providers, dict): - return raw - sanitized = dict(raw) + return sanitized sanitized_providers: dict[str, Any] = {} for name, value in providers.items(): if name == "profiles" or not isinstance(value, dict): @@ -1115,14 +1222,16 @@ def make_llm_provider( __all__ = [ + "AGENT_PHASES", "DEEPCODE_HOME_ENV", "AgentDefaults", "AgentPhase", "AgentsConfig", "ConfigError", "ConnectionProfileConfig", - "ManualModelConfig", "DeepCodeConfig", + "DictationConfig", + "ManualModelConfig", "DocumentSegmentationConfig", "EgressPolicyConfig", "LLMLoggerConfig", diff --git a/core/dictation/__init__.py b/core/dictation/__init__.py new file mode 100644 index 000000000..0907d9892 --- /dev/null +++ b/core/dictation/__init__.py @@ -0,0 +1,41 @@ +"""Voice dictation: turn a recorded clip into text through a Parakeet server. + +The feature is split so each half can be tested on its own: + +- :mod:`core.dictation.audio` — pure byte handling. It decodes the base64 + payload the client sends, decides what container the bytes really are, and + names the upload accordingly. No I/O, no config. +- :mod:`core.dictation.client` — the one HTTP call. It speaks the OpenAI + transcription contract, which is what local Parakeet servers expose, so the + audio never has to leave the machine. +- :class:`core.application.dictation_service.DictationService` — the policy + layer: whether dictation is configured at all, whether the endpoint is + allowed by the egress policy, and how the failures map onto application + errors. +""" + +from core.dictation.audio import ( + MAX_AUDIO_BYTES, + MAX_ENCODED_AUDIO_BYTES, + SUPPORTED_MIME_TYPES, + UnsupportedAudioError, + decode_audio, + extension_for, + sniff_container, +) +from core.dictation.client import ( + SpeechToTextClient, + TranscriptionFailed, +) + +__all__ = [ + "MAX_AUDIO_BYTES", + "MAX_ENCODED_AUDIO_BYTES", + "SUPPORTED_MIME_TYPES", + "SpeechToTextClient", + "TranscriptionFailed", + "UnsupportedAudioError", + "decode_audio", + "extension_for", + "sniff_container", +] diff --git a/core/dictation/audio.py b/core/dictation/audio.py new file mode 100644 index 000000000..5cf9b04f1 --- /dev/null +++ b/core/dictation/audio.py @@ -0,0 +1,186 @@ +"""Byte handling for dictation clips: decode, identify, and name the upload. + +A client records with ``MediaRecorder`` and sends what it got as base64 inside a +JSON-RPC message, so three things must be established before the bytes are +worth forwarding: + +1. the payload is really base64 and really small enough to be a dictation clip; +2. the container inside it is one the endpoint can decode — the declared MIME + type is a *claim* from the browser, and browsers disagree (Safari records + ``audio/mp4``, Chromium ``audio/webm``); +3. the uploaded filename matches the bytes, because OpenAI-compatible servers + sniff the extension and some of them refuse a mismatch. + +Everything here is pure: same input, same decision, no I/O and no environment. +""" + +from __future__ import annotations + +import base64 +import binascii + +#: Decoded ceiling for one clip. 512 KiB is ~2 minutes of Opus at the bitrate +#: ``MediaRecorder`` uses by default and ~40 seconds of 16-bit 22 kHz PCM, which +#: covers the ``maxAudioSeconds`` default with room to spare. +MAX_AUDIO_BYTES = 512 * 1024 + +#: Encoded ceiling. Base64 inflates by 4/3, and the transport caps a whole +#: JSON-RPC message at 1 MiB (``app_server/protocol/codec.py``), so the encoded +#: form may not grow past 768 KiB or the request would die in the codec before +#: any of this ran. +MAX_ENCODED_AUDIO_BYTES = 768 * 1024 + +#: Containers each declared MIME type is allowed to actually contain. The MIME +#: type is trusted for nothing more than selecting this set; the bytes decide +#: the extension. +MIME_CONTAINERS: dict[str, frozenset[str]] = { + "audio/webm": frozenset({"webm"}), + "audio/ogg": frozenset({"ogg"}), + "audio/mp4": frozenset({"mp4"}), + "audio/m4a": frozenset({"mp4"}), + "audio/x-m4a": frozenset({"mp4"}), + "audio/mpeg": frozenset({"mp3"}), + "audio/mp3": frozenset({"mp3"}), + "audio/wav": frozenset({"wav"}), + "audio/wave": frozenset({"wav"}), + "audio/x-wav": frozenset({"wav"}), + "audio/flac": frozenset({"flac"}), + "audio/x-flac": frozenset({"flac"}), + "audio/aac": frozenset({"aac"}), +} + +#: MIME types the endpoint may be asked to decode, derived so the two cannot +#: drift apart. +SUPPORTED_MIME_TYPES = frozenset(MIME_CONTAINERS) + +#: Filename extension per *sniffed* container. ``mp4`` maps to ``m4a`` because +#: that is what a server-side extension check expects for MP4 audio. +CONTAINER_EXTENSIONS: dict[str, str] = { + "webm": "webm", + "ogg": "ogg", + "mp4": "m4a", + "mp3": "mp3", + "wav": "wav", + "flac": "flac", + "aac": "aac", +} + +#: Synchsafe magic numbers. Every one of them is checked at a fixed offset, so +#: the result does not depend on how much of the file was sent. +_MAGIC: tuple[tuple[bytes, int, str], ...] = ( + (b"RIFF", 0, "wav"), + (b"OggS", 0, "ogg"), + (b"fLaC", 0, "flac"), + (b"ID3", 0, "mp3"), + (b"\x1aE\xdf\xa3", 0, "webm"), + (b"ftyp", 4, "mp4"), +) + +#: WAV needs a second magic at offset 8 (``RIFF....WAVE``). +_WAVE_TAG = (b"WAVE", 8) + + +class UnsupportedAudioError(ValueError): + """The clip cannot be forwarded as it arrived. + + Raised for every client-side problem — bad base64, too large, an unknown + MIME type, or bytes that are not the container they claim to be. The + service turns it into an ``INVALID_REQUEST`` application error, and the + message is written to be shown to the user as-is. + """ + + +def sniff_container(data: bytes) -> str | None: + """Return the container the bytes actually look like, or ``None``. + + Deterministic: only fixed offsets are inspected, so a truncated clip is + identified the same way as a complete one. ``None`` means the bytes match + no known container — including the near-miss of a ``RIFF`` header that is + not ``WAVE`` — and the caller rejects it. + """ + + for magic, offset, container in _MAGIC: + if data[offset : offset + len(magic)] == magic: + if ( + container != "wav" + or data[_WAVE_TAG[1] : _WAVE_TAG[1] + 4] == _WAVE_TAG[0] + ): + return container + return None + if len(data) >= 2 and data[0] == 0xFF: + second = data[1] + # ADTS (AAC) and MPEG (MP3) share the 11-bit frame sync, so the ADTS + # bit pattern is tested first: `0xFF 0xF1` is AAC, not MP3. + if (second & 0xF6) == 0xF0: + return "aac" + if (second & 0xE0) == 0xE0: + return "mp3" + return None + + +def extension_for(container: str) -> str: + """Filename extension for a sniffed container (``mp4`` -> ``m4a``).""" + + return CONTAINER_EXTENSIONS[container] + + +def canonical_mime_type(mime_type: str) -> str: + """Lowercase a MIME type and drop its parameters (``;codecs=opus``).""" + + return mime_type.split(";", 1)[0].strip().lower() + + +def decode_audio(audio: str, mime_type: str) -> tuple[bytes, str]: + """Validate a clip and return ``(data, filename)``. + + ``audio`` is the base64 body as it arrived; ``mime_type`` is what the + client claims to have recorded. Raises :class:`UnsupportedAudioError` with + a user-facing message for anything the endpoint could not decode. + """ + + declared = canonical_mime_type(mime_type) + if not declared: + raise UnsupportedAudioError("mimeType must not be empty") + if declared not in SUPPORTED_MIME_TYPES: + supported = ", ".join(sorted(SUPPORTED_MIME_TYPES)) + raise UnsupportedAudioError( + f"unsupported audio mimeType {declared!r}; supported: {supported}" + ) + + if not audio: + raise UnsupportedAudioError("audio must not be empty") + encoded = audio.strip() + if len(encoded) > MAX_ENCODED_AUDIO_BYTES: + raise UnsupportedAudioError( + f"recorded audio is larger than {MAX_ENCODED_AUDIO_BYTES // 1024} KiB " + "once encoded; record a shorter clip" + ) + # Clients that build the payload by hand may drop the padding base64 wants + # back; nothing else is repaired, so a payload that is not base64 at all + # still fails instead of being silently repaired into noise. + padded = encoded + "=" * (-len(encoded) % 4) + try: + data = base64.b64decode(padded, validate=True) + except (binascii.Error, ValueError) as exc: + raise UnsupportedAudioError(f"audio is not valid base64 ({exc})") from exc + + if not data: + raise UnsupportedAudioError("audio must not be empty") + if len(data) > MAX_AUDIO_BYTES: + raise UnsupportedAudioError( + f"recorded audio is larger than {MAX_AUDIO_BYTES // 1024} KiB; " + "record a shorter clip" + ) + + container = sniff_container(data) + if container is None: + raise UnsupportedAudioError( + "audio is not a recognisable recording (expected webm, ogg, mp4, " + "mp3, wav, flac, or aac)" + ) + if container not in MIME_CONTAINERS[declared]: + raise UnsupportedAudioError( + f"audio mimeType {declared!r} does not match the recorded data " + f"({container})" + ) + return data, f"dictation.{extension_for(container)}" diff --git a/core/dictation/client.py b/core/dictation/client.py new file mode 100644 index 000000000..e1cdd9c0f --- /dev/null +++ b/core/dictation/client.py @@ -0,0 +1,144 @@ +"""The transcription request: one POST, OpenAI-compatible, no surprises. + +Parakeet is served by ``mlx_audio.server``, which exposes the same route the +OpenAI API does (``POST /audio/transcriptions`` as multipart form data). +Speaking that contract instead of a bespoke one means the audio can stay on the +machine while the client stays worth nothing to a vendor-specific implementation. + +Two properties matter more than the happy path: + +- **The response body is never echoed.** An ASR endpoint that answers with an + HTML error page or a stack trace must not have that text travel back to the + UI, into logs, or into a model prompt. Only the status code is reported. +- **Redirects are not followed.** The request carries the user's audio and, on a + hosted endpoint, a bearer token; letting a 3xx choose a new destination for + both would undo the egress check the caller just performed. +""" + +from __future__ import annotations + +import httpx + +#: Default whole-request budget. The first request usually pays for loading the +#: model on the server, and a local Parakeet model is a few hundred megabytes. +DEFAULT_TIMEOUT_SECONDS = 60.0 + +#: Statuses worth retrying: the endpoint was reachable and the clip is fine. +_RETRYABLE_STATUSES = frozenset({408, 425, 429, 500, 502, 503, 504}) + + +class TranscriptionFailed(RuntimeError): + """The endpoint did not return a transcript. + + ``retryable`` tells the caller whether trying again could help, and + ``status_code`` is the HTTP status when there was one. The message never + contains the response body: it is written by this client, from the status + code and the URL alone. + """ + + def __init__( + self, + message: str, + *, + status_code: int | None = None, + retryable: bool = False, + ) -> None: + super().__init__(message) + self.status_code = status_code + self.retryable = retryable + + +class SpeechToTextClient: + """Synchronous client for one speech-to-text endpoint. + + Synchronous on purpose: the app server runs request handlers in a thread + pool, so a blocking call here costs one worker thread and no event loop + time, while an async client would have to be threaded through the pool + anyway. + """ + + def __init__( + self, + endpoint: str, + model: str, + *, + language: str | None = None, + api_key: str | None = None, + timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, + transport: httpx.BaseTransport | None = None, + ) -> None: + self._url = f"{endpoint.rstrip('/')}/audio/transcriptions" + self._model = model + self._language = language + self._api_key = api_key + self._timeout_seconds = timeout_seconds + # Injected only by tests, so the request this client builds can be + # asserted without a socket. + self._transport = transport + + @property + def url(self) -> str: + """The resolved request URL, for logs and error messages.""" + + return self._url + + def transcribe(self, audio: bytes, *, filename: str, mime_type: str) -> str: + """Post one clip and return its transcript (possibly empty). + + ``filename`` comes from the sniffed container, not from the client, so + a server that dispatches on the extension dispatches on the real bytes. + """ + + form: dict[str, str] = {"model": self._model} + if self._language: + form["language"] = self._language + headers: dict[str, str] = {} + if self._api_key: + headers["Authorization"] = f"Bearer {self._api_key}" + + try: + with httpx.Client( + timeout=httpx.Timeout(self._timeout_seconds), + follow_redirects=False, + transport=self._transport, + ) as client: + response = client.post( + self._url, + data=form, + files={"file": (filename, audio, mime_type)}, + headers=headers, + ) + except httpx.TimeoutException as exc: + raise TranscriptionFailed( + f"dictation endpoint {self._url} did not answer within " + f"{self._timeout_seconds:g}s", + retryable=True, + ) from exc + except httpx.HTTPError as exc: + # Connection refused, DNS failure, TLS failure. `exc` may quote the + # URL but never the body, and the body is what could hold a secret. + raise TranscriptionFailed( + f"dictation endpoint {self._url} is unreachable ({type(exc).__name__})", + retryable=True, + ) from exc + + if response.status_code >= 300: + raise TranscriptionFailed( + f"dictation endpoint {self._url} answered HTTP {response.status_code}", + status_code=response.status_code, + retryable=response.status_code in _RETRYABLE_STATUSES, + ) + + try: + payload = response.json() + except ValueError as exc: + raise TranscriptionFailed( + f"dictation endpoint {self._url} answered with a non-JSON body", + status_code=response.status_code, + ) from exc + if not isinstance(payload, dict) or not isinstance(payload.get("text"), str): + raise TranscriptionFailed( + f"dictation endpoint {self._url} answered without a 'text' field", + status_code=response.status_code, + ) + return payload["text"] diff --git a/desktop/src/app/i18n.ts b/desktop/src/app/i18n.ts index c99665c13..e2314f7e5 100644 --- a/desktop/src/app/i18n.ts +++ b/desktop/src/app/i18n.ts @@ -182,6 +182,11 @@ const ZH_CN: Record = { "composer.hint.queueSteer": "↵ 排队 · ⌘↵ 引导", "composer.hint.newline": "⇧↵ 换行", "composer.queueNext": "排队下一个", + "composer.dictation.start": "开始语音输入", + "composer.dictation.stop": "停止录音并转写", + "composer.dictation.cancel": "丢弃录音", + "composer.dictation.recording": "正在录音 {{seconds}} 秒 · 停止后转写", + "composer.dictation.transcribing": "正在转写…", // Thread header "thread.startThread": "开始本地编码会话", "thread.folderUnavailable": "文件夹不可用", diff --git a/desktop/src/features/execution/Composer.module.css b/desktop/src/features/execution/Composer.module.css index ae9ad639c..7a3aeee7e 100644 --- a/desktop/src/features/execution/Composer.module.css +++ b/desktop/src/features/execution/Composer.module.css @@ -437,6 +437,45 @@ position: relative; } +.dictationButton, +.dictationCancel { + display: grid; + width: 31px; + height: 31px; + flex: 0 0 auto; + place-items: center; + border-radius: var(--radius-sm); + color: var(--text-tertiary); +} + +.dictationButton:hover:not(:disabled), +.dictationCancel:hover:not(:disabled) { + background: var(--surface-hover); + color: var(--text-primary); +} + +/* Recording is a state, not a hover: it has to stay visible while the user + speaks, including when the cursor moved away. */ +.dictationButton[data-recording="true"], +.dictationButton[data-recording="true"]:hover:not(:disabled) { + background: var(--danger-soft); + color: var(--danger); + animation: dictation-pulse 1.6s ease-in-out infinite; +} + +@keyframes dictation-pulse { + 50% { + box-shadow: 0 0 0 4px var(--danger-soft); + } +} + +@media (prefers-reduced-motion: reduce) { + .dictationButton[data-recording="true"], + .dictationButton[data-recording="true"]:hover:not(:disabled) { + animation: none; + } +} + .skillButton b { position: absolute; top: 1px; diff --git a/desktop/src/features/execution/Composer.tsx b/desktop/src/features/execution/Composer.tsx index 6f2242496..5edfdec58 100644 --- a/desktop/src/features/execution/Composer.tsx +++ b/desktop/src/features/execution/Composer.tsx @@ -1,6 +1,7 @@ import { ArrowUp, Check, + Mic, Paperclip, ShieldCheck, Sparkles, @@ -41,6 +42,7 @@ import { type ComposerCommand, } from "./commands"; import styles from "./Composer.module.css"; +import { useDictation } from "./useDictation"; import { usePromptDraft } from "./usePromptDraft"; import { ModelPicker } from "./ModelPicker"; import { TranscriptModePicker } from "./TranscriptModePicker"; @@ -147,6 +149,7 @@ export function Composer({ initialLaunch?.prompt, ); const textareaRef = useRef(null); + const caretRef = useRef(null); const [contextError, setContextError] = useState(null); const [commandError, setCommandError] = useState(null); const [skillPickerOpen, setSkillPickerOpen] = useState(false); @@ -155,6 +158,10 @@ export function Composer({ initialLaunch?.skillIds ?? [], ); const [deliveryNotice, setDeliveryNotice] = useState(null); + const dictation = useDictation({ + runtime, + onTranscript: insertDictation, + }); const skillCatalog = useSkillCatalog(runtime, project?.id ?? null); const presetCatalog = usePresetCatalog( runtime, @@ -205,6 +212,13 @@ export function Composer({ if (!textarea) return; textarea.style.height = "0px"; textarea.style.height = `${Math.min(textarea.scrollHeight, 190)}px`; + // A dictated transcript is inserted mid-text, so the caret has to be put + // back where the text ended instead of jumping to the end of the draft. + const caret = caretRef.current; + if (caret !== null) { + caretRef.current = null; + textarea.setSelectionRange(caret, caret); + } }, [prompt]); useEffect(() => { @@ -213,6 +227,27 @@ export function Composer({ onLaunchIntentConsumed(); }, [initialLaunch, onLaunchIntentConsumed]); + /** + * Insert a transcript at the caret. + * + * Function declaration (not a const) because `useDictation` is constructed + * above it and only calls back after the user speaks. + */ + function insertDictation(text: string): void { + const textarea = textareaRef.current; + const start = textarea ? textarea.selectionStart : prompt.length; + const end = textarea ? textarea.selectionEnd : prompt.length; + const before = prompt.slice(0, start); + const after = prompt.slice(end); + const separator = before && !/\s$/.test(before) ? " " : ""; + const insertion = `${separator}${text}`; + setPrompt(`${before}${insertion}${after}`); + caretRef.current = start + insertion.length; + setCommandError(null); + setDeliveryNotice(null); + textarea?.focus(); + } + const submit = async () => { const value = prompt.trim(); if (!value || !canExecute || busy) return; @@ -278,6 +313,13 @@ export function Composer({ setSkillPickerOpen(false); }; const commandSuggestions = matchingCommands(prompt); + const dictationStatus = dictation.recording + ? t("composer.dictation.recording", "Recording {{seconds}}s · stop to transcribe", { + seconds: dictation.elapsedSeconds, + }) + : dictation.transcribing + ? t("composer.dictation.transcribing", "Transcribing…") + : null; const pickContextFiles = async () => { setContextError(null); @@ -297,6 +339,11 @@ export function Composer({ }; const onKeyDown = (event: KeyboardEvent) => { + if (event.key === "Escape" && dictation.recording) { + event.preventDefault(); + dictation.cancel(); + return; + } if ( event.key === "Enter" && !event.shiftKey && @@ -570,6 +617,41 @@ export function Composer({ {selectedSkills.length ? {selectedSkills.length} : null} + {dictation.available ? ( + + ) : null} + {dictation.recording ? ( + + ) : null} {thread?.mode === "paper" ? "Paper2Code" : "Local"} @@ -670,7 +752,9 @@ export function Composer({

{commandError ?? contextError ?? + dictation.error ?? deliveryNotice ?? + dictationStatus ?? disabledReason ?? "DeepCode may ask before sensitive tools run."} diff --git a/desktop/src/features/execution/useDictation.test.ts b/desktop/src/features/execution/useDictation.test.ts new file mode 100644 index 000000000..101ee9ef5 --- /dev/null +++ b/desktop/src/features/execution/useDictation.test.ts @@ -0,0 +1,377 @@ +import { act, renderHook } from "@testing-library/react"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; + +import type { + DictationStatusResult, + MethodParams, + MethodResults, +} from "../../generated/app-server"; +import type { ClientRuntime, RpcMethod } from "../../rpc/contracts"; +import { useDictation } from "./useDictation"; + +const MODEL = "mlx-community/parakeet-tdt-0.6b-v3"; +const PAYLOAD = new Uint8Array([0x1a, 0x45, 0xdf, 0xa3, 0x00, 0x01, 0x02, 0x03]); + +class DictationRuntime { + status: DictationStatusResult = { + available: true, + model: MODEL, + maxAudioSeconds: 120, + }; + statusError: Error | null = null; + transcript = { text: "olá mundo", model: MODEL }; + transcribeError: Error | null = null; + holdTranscribe = false; + readonly requests: Array<{ method: RpcMethod; params: unknown }> = []; + private pending: Array<() => void> = []; + + async request( + method: M, + params: MethodParams[M], + ): Promise { + this.requests.push({ method, params }); + if (method === "dictation/status") { + if (this.statusError) throw this.statusError; + return this.status as MethodResults[M]; + } + if (method !== "dictation/transcribe") { + throw new Error(`Unexpected method: ${method}`); + } + if (this.holdTranscribe) { + await new Promise((resolve) => this.pending.push(resolve)); + } + if (this.transcribeError) throw this.transcribeError; + return this.transcript as MethodResults[M]; + } + + releaseTranscribe() { + const pending = this.pending; + this.pending = []; + for (const resolve of pending) resolve(); + } + + get methods(): string[] { + return this.requests.map((request) => request.method); + } + + transcribeParams(): MethodParams["dictation/transcribe"] { + const request = this.requests.find( + (entry) => entry.method === "dictation/transcribe", + ); + expect(request).toBeDefined(); + return request?.params as MethodParams["dictation/transcribe"]; + } +} + +class FakeTrack { + stopped = false; + + stop() { + this.stopped = true; + } +} + +class FakeStream { + readonly track = new FakeTrack(); + + getTracks(): FakeTrack[] { + return [this.track]; + } +} + +class FakeRecorder { + static supported: string[] = ["audio/webm", "audio/ogg", "audio/mp4"]; + static created: FakeRecorder[] = []; + static payload: Uint8Array | null = PAYLOAD; + /** Container the recorder reports once running, as a real one does. */ + static recordedType: string | null = null; + + static isTypeSupported(type: string): boolean { + return FakeRecorder.supported.includes(type); + } + + state: "inactive" | "recording" | "paused" = "inactive"; + mimeType: string; + ondataavailable: ((event: BlobEvent) => void) | null = null; + onstop: (() => void) | null = null; + + constructor( + readonly stream: FakeStream, + readonly options?: { mimeType?: string }, + ) { + this.mimeType = FakeRecorder.recordedType ?? options?.mimeType ?? ""; + FakeRecorder.created.push(this); + } + + start() { + this.state = "recording"; + } + + stop() { + if (this.state === "inactive") return; + this.state = "inactive"; + const payload = FakeRecorder.payload; + if (payload) { + this.ondataavailable?.({ + data: new Blob([payload.buffer as ArrayBuffer], { type: this.mimeType }), + } as BlobEvent); + } + this.onstop?.(); + } +} + +let stream: FakeStream; +let getUserMedia: ReturnType; + +function installRecorderEnvironment() { + stream = new FakeStream(); + getUserMedia = vi.fn(async () => stream as unknown as MediaStream); + FakeRecorder.created = []; + FakeRecorder.supported = ["audio/webm", "audio/ogg", "audio/mp4"]; + FakeRecorder.recordedType = null; + FakeRecorder.payload = PAYLOAD; + vi.stubGlobal("MediaRecorder", FakeRecorder); + Object.defineProperty(navigator, "mediaDevices", { + value: { getUserMedia }, + configurable: true, + }); +} + +function removeRecorderEnvironment() { + vi.stubGlobal("MediaRecorder", undefined); + Object.defineProperty(navigator, "mediaDevices", { + value: undefined, + configurable: true, + }); +} + +/** + * Let pending promises and React work settle. + * + * Fake timers are installed, so `waitFor` (which polls on a real timer) cannot + * be used here; this drains the microtask queue and any zero-delay timer + * instead, which is what every awaited step in the hook produces. + */ +async function flush(rounds = 8) { + await act(async () => { + for (let round = 0; round < rounds; round += 1) { + vi.advanceTimersByTime(0); + await Promise.resolve(); + } + }); +} + +function renderDictation(backend: DictationRuntime) { + const onTranscript = vi.fn(); + const view = renderHook(() => + useDictation({ + runtime: backend as unknown as ClientRuntime, + onTranscript, + }), + ); + return { ...view, onTranscript }; +} + +async function ready(backend: DictationRuntime) { + const view = renderDictation(backend); + await flush(); + expect(view.result.current.available).toBe(true); + return view; +} + +beforeEach(() => { + vi.useFakeTimers(); + installRecorderEnvironment(); +}); + +afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); + +test("offers no microphone when the server reports dictation unavailable", async () => { + const backend = new DictationRuntime(); + backend.status = { available: false, model: null, maxAudioSeconds: null }; + const { result } = renderDictation(backend); + + await flush(); + expect(result.current.available).toBe(false); + + await act(async () => result.current.toggle()); + + expect(getUserMedia).not.toHaveBeenCalled(); + expect(backend.methods).toEqual(["dictation/status"]); +}); + +test("treats a failing status request as unavailable", async () => { + const backend = new DictationRuntime(); + backend.statusError = new Error("method not found: dictation/status"); + const { result } = renderDictation(backend); + + await flush(); + expect(backend.methods).toContain("dictation/status"); + expect(result.current.available).toBe(false); +}); + +test("records a clip, uploads it, and hands the transcript to the caller", async () => { + const backend = new DictationRuntime(); + const { result, onTranscript } = await ready(backend); + + await act(async () => result.current.toggle()); + await flush(); + expect(result.current.recording).toBe(true); + expect(FakeRecorder.created[0].options?.mimeType).toBe("audio/webm"); + + await act(async () => { + vi.advanceTimersByTime(2000); + }); + expect(result.current.elapsedSeconds).toBe(2); + + await act(async () => result.current.toggle()); + await flush(); + + expect(onTranscript).toHaveBeenCalledWith("olá mundo"); + const params = backend.transcribeParams(); + expect(params.mimeType).toBe("audio/webm"); + expect(atob(params.audio)).toBe( + String.fromCharCode(...PAYLOAD), + ); + expect(result.current.recording).toBe(false); + expect(result.current.transcribing).toBe(false); + expect(result.current.elapsedSeconds).toBe(0); +}); + +test("reports the browser's container when the preferred one is unsupported", async () => { + // WKWebView records MP4 only; the upload must follow what was recorded. + FakeRecorder.supported = []; + FakeRecorder.recordedType = "audio/mp4;codecs=mp4a.40.2"; + const backend = new DictationRuntime(); + const { result, onTranscript } = await ready(backend); + + await act(async () => result.current.toggle()); + expect(FakeRecorder.created[0].options).toBeUndefined(); + + await act(async () => result.current.toggle()); + await flush(); + + expect(onTranscript).toHaveBeenCalled(); + expect(backend.transcribeParams().mimeType).toBe("audio/mp4"); +}); + +test("cancelling discards the clip without uploading anything", async () => { + const backend = new DictationRuntime(); + const { result } = await ready(backend); + + await act(async () => result.current.toggle()); + await act(async () => { + vi.advanceTimersByTime(3000); + }); + await act(async () => result.current.cancel()); + + expect(backend.methods).toEqual(["dictation/status"]); + expect(result.current.recording).toBe(false); + expect(result.current.elapsedSeconds).toBe(0); + expect(stream.track.stopped).toBe(true); +}); + +test("drops a transcript that arrives after the user cancelled", async () => { + const backend = new DictationRuntime(); + backend.holdTranscribe = true; + const { result, onTranscript } = await ready(backend); + + await act(async () => result.current.toggle()); + await act(async () => result.current.toggle()); + await flush(); + expect(result.current.transcribing).toBe(true); + + await act(async () => result.current.cancel()); + await act(async () => { + backend.releaseTranscribe(); + }); + await flush(); + + expect(onTranscript).not.toHaveBeenCalled(); + expect(result.current.transcribing).toBe(false); +}); + +test("stops recording at the configured cap", async () => { + const backend = new DictationRuntime(); + backend.status = { available: true, model: MODEL, maxAudioSeconds: 2 }; + const { result, onTranscript } = await ready(backend); + + await act(async () => result.current.toggle()); + await act(async () => { + vi.advanceTimersByTime(3000); + }); + await flush(); + + expect(onTranscript).toHaveBeenCalled(); + expect(result.current.recording).toBe(false); + expect(backend.methods).toContain("dictation/transcribe"); +}); + +test("reports a denied microphone without recording", async () => { + getUserMedia.mockRejectedValueOnce( + new DOMException("permission denied", "NotAllowedError"), + ); + const backend = new DictationRuntime(); + const { result } = await ready(backend); + + await act(async () => result.current.toggle()); + + expect(result.current.recording).toBe(false); + expect(result.current.error).toMatch(/Microphone access was denied/); + expect(backend.methods).toEqual(["dictation/status"]); +}); + +test("reports an environment without a recorder", async () => { + removeRecorderEnvironment(); + const backend = new DictationRuntime(); + const { result } = await ready(backend); + + await act(async () => result.current.toggle()); + + expect(result.current.error).toMatch(/not available in this environment/); + expect(result.current.recording).toBe(false); +}); + +test("surfaces an endpoint failure as an error instead of a transcript", async () => { + const backend = new DictationRuntime(); + backend.transcribeError = new Error( + "Voice input could not be transcribed. Check that the dictation endpoint is running, then try again.", + ); + const { result, onTranscript } = await ready(backend); + + await act(async () => result.current.toggle()); + await act(async () => result.current.toggle()); + await flush(); + + expect(result.current.error).toMatch(/could not be transcribed/); + expect(onTranscript).not.toHaveBeenCalled(); + expect(result.current.transcribing).toBe(false); +}); + +test("ignores a silent clip without inserting anything", async () => { + const backend = new DictationRuntime(); + backend.transcript = { text: " ", model: MODEL }; + const { result, onTranscript } = await ready(backend); + + await act(async () => result.current.toggle()); + await act(async () => result.current.toggle()); + await flush(); + + expect(result.current.error).toMatch(/No speech was detected/); + expect(onTranscript).not.toHaveBeenCalled(); +}); + +test("releases the microphone when the composer unmounts", async () => { + const backend = new DictationRuntime(); + const { result, unmount } = await ready(backend); + + await act(async () => result.current.toggle()); + unmount(); + + expect(stream.track.stopped).toBe(true); + expect(backend.methods).toEqual(["dictation/status"]); +}); diff --git a/desktop/src/features/execution/useDictation.ts b/desktop/src/features/execution/useDictation.ts new file mode 100644 index 000000000..3aea5ce35 --- /dev/null +++ b/desktop/src/features/execution/useDictation.ts @@ -0,0 +1,313 @@ +import { useCallback, useEffect, useRef, useState } from "react"; + +import type { DictationStatusResult } from "../../generated/app-server"; +import type { ClientRuntime } from "../../rpc/contracts"; + +/** + * Push-to-talk dictation for the composer. + * + * The audio never becomes prompt text by itself: the hook records, uploads the + * clip, and hands the transcript to `onTranscript`, so the caller decides where + * it lands (the composer inserts it at the caret). Three behaviours are worth + * knowing before changing anything here: + * + * - Nothing is uploaded when dictation is not configured. `dictation/status` + * answers `available: false` and the caller renders no microphone at all. + * - Recording is local until the user stops it. `cancel()` discards the clip + * without a request, which is what Escape does. + * - A transcript that arrives after `cancel()` is dropped, so a slow endpoint + * cannot paste text the user already abandoned. + */ + +/** Containers to try, best first. Chromium records webm, WKWebView only mp4. */ +const MIME_CANDIDATES = ["audio/webm", "audio/ogg", "audio/mp4"] as const; + +const RECORDING_UNAVAILABLE = + "Dictation recording is not available in this environment."; + +export interface UseDictationOptions { + runtime: ClientRuntime; + /** Receives the transcript. Called only for non-empty text. */ + onTranscript(text: string): void; +} + +export interface UseDictationResult { + /** True only when the app server reports a configured endpoint. */ + available: boolean; + recording: boolean; + transcribing: boolean; + elapsedSeconds: number; + error: string | null; + /** Start recording, or stop and transcribe when already recording. */ + toggle(): void; + /** Stop recording and throw the clip away without transcribing it. */ + cancel(): void; +} + +function messageOf(cause: unknown): string { + return cause instanceof Error ? cause.message : String(cause); +} + +function errorName(cause: unknown): string { + if (typeof cause !== "object" || cause === null || !("name" in cause)) return ""; + return String((cause as { name: unknown }).name); +} + +function microphoneMessage(cause: unknown): string { + const name = errorName(cause); + if (name === "NotAllowedError" || name === "SecurityError") { + return ( + "Microphone access was denied. Allow it for DeepCode in your system " + + "settings, then try again." + ); + } + if (name === "NotFoundError" || name === "DevicesNotFoundError") { + return "No microphone was found on this machine."; + } + return `The microphone could not be opened (${messageOf(cause)}).`; +} + +/** Strip codec parameters: the endpoint dispatches on the container alone. */ +function canonicalMimeType(mimeType: string): string { + return mimeType.split(";")[0].trim().toLowerCase(); +} + +function preferredMimeType(): string | null { + const Recorder = globalThis.MediaRecorder; + if (typeof Recorder?.isTypeSupported !== "function") return null; + for (const candidate of MIME_CANDIDATES) { + if (Recorder.isTypeSupported(candidate)) return candidate; + } + return null; +} + +/** + * Base64 in 32 KiB chunks: `String.fromCharCode(...bytes)` on a whole clip + * overflows the argument limit and would throw on a long recording. + */ +async function blobToBase64(blob: Blob): Promise { + const bytes = new Uint8Array(await blob.arrayBuffer()); + let binary = ""; + const chunk = 0x8000; + for (let offset = 0; offset < bytes.length; offset += chunk) { + binary += String.fromCharCode(...bytes.subarray(offset, offset + chunk)); + } + return btoa(binary); +} + +export function useDictation({ + runtime, + onTranscript, +}: UseDictationOptions): UseDictationResult { + const [status, setStatus] = useState(null); + const [recording, setRecording] = useState(false); + const [transcribing, setTranscribing] = useState(false); + const [elapsedSeconds, setElapsedSeconds] = useState(0); + const [error, setError] = useState(null); + + const recorderRef = useRef(null); + const streamRef = useRef(null); + const chunksRef = useRef([]); + const mimeTypeRef = useRef(""); + const discardRef = useRef(false); + const timerRef = useRef | null>(null); + const startedAtRef = useRef(0); + const maxSecondsRef = useRef(null); + const transcriptRef = useRef(onTranscript); + + useEffect(() => { + transcriptRef.current = onTranscript; + }, [onTranscript]); + + // The capability is a config read; asking once per mount is enough, and a + // failure (including a server that predates the method) means "no mic". + useEffect(() => { + let cancelled = false; + void runtime + .request("dictation/status", {}) + .then((result) => { + if (cancelled) return; + setStatus(result); + }) + .catch(() => { + if (!cancelled) setStatus(null); + }); + return () => { + cancelled = true; + }; + }, [runtime]); + + const stopTimer = useCallback(() => { + if (timerRef.current !== null) clearInterval(timerRef.current); + timerRef.current = null; + }, []); + + const releaseStream = useCallback(() => { + streamRef.current?.getTracks().forEach((track) => track.stop()); + streamRef.current = null; + }, []); + + const finish = useCallback(async () => { + stopTimer(); + releaseStream(); + setRecording(false); + setElapsedSeconds(0); + const chunks = chunksRef.current; + chunksRef.current = []; + const recorder = recorderRef.current; + recorderRef.current = null; + const mimeType = canonicalMimeType( + recorder?.mimeType || mimeTypeRef.current || chunks[0]?.type || "", + ); + if (discardRef.current) { + discardRef.current = false; + return; + } + if (!chunks.length || !mimeType) { + setError("The recording was empty; nothing was transcribed."); + return; + } + setTranscribing(true); + try { + const audio = await blobToBase64(new Blob(chunks, { type: mimeType })); + const result = await runtime.request("dictation/transcribe", { + audio, + mimeType, + }); + // A clip the user cancelled while it was uploading must not paste text. + if (discardRef.current) { + discardRef.current = false; + return; + } + const text = result.text.trim(); + if (text) transcriptRef.current(text); + else setError("No speech was detected in the recording."); + } catch (cause) { + if (!discardRef.current) setError(messageOf(cause)); + } finally { + discardRef.current = false; + setTranscribing(false); + } + }, [releaseStream, runtime, stopTimer]); + + const stop = useCallback(() => { + const recorder = recorderRef.current; + if (recorder && recorder.state !== "inactive") { + recorder.stop(); + return; + } + void finish(); + }, [finish]); + + const available = status?.available === true; + + const start = useCallback(async () => { + // The caller renders no microphone while dictation is unavailable, so this + // guard only matters to a host that calls the hook directly. + if (!available) return; + setError(null); + const media = globalThis.navigator?.mediaDevices; + if (!media?.getUserMedia || typeof globalThis.MediaRecorder !== "function") { + setError(RECORDING_UNAVAILABLE); + return; + } + let stream: MediaStream; + try { + stream = await media.getUserMedia({ audio: true }); + } catch (cause) { + setError(microphoneMessage(cause)); + return; + } + const mimeType = preferredMimeType(); + let recorder: MediaRecorder; + try { + recorder = new MediaRecorder( + stream, + mimeType ? { mimeType } : undefined, + ); + } catch (cause) { + stream.getTracks().forEach((track) => track.stop()); + setError(`Recording could not start (${messageOf(cause)}).`); + return; + } + streamRef.current = stream; + recorderRef.current = recorder; + mimeTypeRef.current = mimeType ?? ""; + chunksRef.current = []; + discardRef.current = false; + recorder.ondataavailable = (event: BlobEvent) => { + if (event.data?.size) chunksRef.current.push(event.data); + }; + recorder.onstop = () => { + void finish(); + }; + recorder.start(); + setRecording(true); + setTranscribing(false); + setElapsedSeconds(0); + startedAtRef.current = Date.now(); + stopTimer(); + timerRef.current = setInterval(() => { + const seconds = Math.floor((Date.now() - startedAtRef.current) / 1000); + setElapsedSeconds(seconds); + const max = maxSecondsRef.current; + const active = recorderRef.current; + // The configured cap is enforced here rather than trusted to the + // recorder: a long clip would exceed the upload ceiling otherwise. + if (max !== null && seconds >= max && active?.state === "recording") { + active.stop(); + } + }, 1000); + }, [available, finish, stopTimer]); + + const cancel = useCallback(() => { + discardRef.current = true; + stopTimer(); + const recorder = recorderRef.current; + if (recorder && recorder.state !== "inactive") { + recorder.stop(); + return; + } + recorderRef.current = null; + chunksRef.current = []; + releaseStream(); + setRecording(false); + setTranscribing(false); + setElapsedSeconds(0); + }, [releaseStream, stopTimer]); + + useEffect(() => { + maxSecondsRef.current = status?.available + ? status.maxAudioSeconds + : null; + }, [status]); + + useEffect( + () => () => { + discardRef.current = true; + if (timerRef.current !== null) clearInterval(timerRef.current); + timerRef.current = null; + const recorder = recorderRef.current; + if (recorder && recorder.state !== "inactive") recorder.stop(); + recorderRef.current = null; + streamRef.current?.getTracks().forEach((track) => track.stop()); + streamRef.current = null; + }, + [], + ); + + const toggle = useCallback(() => { + if (recording) stop(); + else void start(); + }, [recording, start, stop]); + + return { + available, + recording, + transcribing, + elapsedSeconds, + error, + toggle, + cancel, + }; +} diff --git a/desktop/src/generated/app-server.ts b/desktop/src/generated/app-server.ts index 4549f5a4c..2c2e1c7ab 100644 --- a/desktop/src/generated/app-server.ts +++ b/desktop/src/generated/app-server.ts @@ -196,6 +196,8 @@ export interface MethodParams { "provider/login/poll": ProviderLoginFlowParams; "provider/login/cancel": ProviderLoginFlowParams; "provider/logout": ProviderLogoutParams; + "dictation/status": EmptyParams; + "dictation/transcribe": DictationTranscribeParams; } export interface InitializeParams { protocolVersion: "1.0"; @@ -670,6 +672,11 @@ export interface ProviderLoginFlowParams { export interface ProviderLogoutParams { connectionId: string; } +export interface DictationTranscribeParams { + audio: string; + mimeType: string; + language?: string | null; +} export interface MethodResults { initialize: InitializeResult; shutdown: { @@ -912,6 +919,8 @@ export interface MethodResults { "provider/login/poll": ProviderLoginFlow; "provider/login/cancel": ProviderLoginFlow; "provider/logout": ProviderLogoutResult; + "dictation/status": DictationStatusResult; + "dictation/transcribe": DictationTranscribeResult; } export interface InitializeResult { protocolVersion: "1.0"; @@ -1811,6 +1820,15 @@ export interface ProviderLogoutResult { remoteRevoked: false; manageUrl: string; } +export interface DictationStatusResult { + available: boolean; + model: string | null; + maxAudioSeconds: number | null; +} +export interface DictationTranscribeResult { + text: string; + model: string; +} export interface Notifications { "thread.updated": Event; "turn.started": Event; diff --git a/docs/guide/README.md b/docs/guide/README.md index f49125451..9e40b343b 100644 --- a/docs/guide/README.md +++ b/docs/guide/README.md @@ -21,6 +21,7 @@ For installation options, see the [quick start](../../README.md#quick-start). |---|---| | Complete my first task | [Getting started](getting-started.md) | | Connect another provider or use a local model | [Models and providers](models.md) | +| Dictate prompts with voice (Parakeet) | [Voice dictation](dictation.md) | | Attach files, use shortcuts, or control a running task | [The terminal UI](the-tui.md) | | Resume a conversation or manage long chats | [Sessions](sessions.md) | | Add project conventions and reusable instructions | [Skills and memory](skills-and-memory.md) | diff --git a/docs/guide/dictation.md b/docs/guide/dictation.md new file mode 100644 index 000000000..71ac3284d --- /dev/null +++ b/docs/guide/dictation.md @@ -0,0 +1,65 @@ +# Voice Dictation (Parakeet) + +DeepCode supports push-to-talk voice input directly in the composer. Spoken text is transcribed through a speech-to-text model—such as NVIDIA Parakeet—and inserted right where your cursor is, without bypassing your review before sending. + +## Overview + +- **No audio leaves without configuration.** The microphone button only appears when a `dictation` block is declared in your configuration. +- **Local first.** With a local engine such as `mlx-audio`, recordings stay entirely on your machine. +- **Composer integration.** The transcribed text lands in the composer draft at the current cursor position so you can review, edit, or append to it before starting or steering a turn. +- **Escape to discard.** Pressing `Escape` (or clicking the discard button) while recording drops the audio immediately without making a request. + +--- + +## Setting up a Local Parakeet Server + +You can run NVIDIA Parakeet locally on Apple Silicon using `mlx-audio`: + +```bash +# 1. Install mlx-audio in an isolated environment +pip install mlx-audio + +# 2. Start the transcription server +mlx_audio.server --host 127.0.0.1 --port 8000 +``` + +The server exposes an OpenAI-compatible speech-to-text API at: +`http://127.0.0.1:8000/v1/audio/transcriptions` + +--- + +## Configuring DeepCode + +Add the `dictation` block to your user configuration file (`~/.deepcode/deepcode_config.json`): + +```json +{ + "dictation": { + "endpoint": "http://127.0.0.1:8000/v1", + "model": "mlx-community/parakeet-tdt-0.6b-v3", + "language": "pt", + "maxAudioSeconds": 120, + "timeoutSeconds": 60 + } +} +``` + +### Configuration Options + +| Field | Type | Default | Description | +|---|---|---|---| +| `endpoint` | string | *required* | Base URL of the OpenAI-compatible transcription server (e.g. `http://127.0.0.1:8000/v1`). | +| `model` | string | *required* | The speech-to-text model identifier recognized by the server. | +| `language` | string \| null | `null` | Optional ISO 639-1 language hint (e.g. `"en"`, `"pt"`, `"es"`). | +| `apiKeyEnv` | string \| null | `null` | Name of an environment variable containing the bearer token (for authenticated endpoints). | +| `maxAudioSeconds` | integer | `120` | Maximum recording length in seconds (1–600). Recording stops automatically when this cap is reached. | +| `timeoutSeconds` | float | `60.0` | Whole-request transcription timeout budget (1–600). | + +--- + +## Security and Privacy Guardrails + +1. **User-owned configuration.** Project configuration files cannot configure or redirect the dictation endpoint; repository-level dictation overrides are discarded automatically to prevent malicious repositories from redirecting microphone audio. +2. **Model egress policy.** If you enforce an egress policy (`providers.egress`), the dictation endpoint host is checked against allowed and blocked domains before any audio data leaves the machine. +3. **Clip size limits.** Decoded audio is capped at 512 KiB (~2 minutes of Opus), keeping requests safely within the JSON-RPC message envelope. +4. **Secure contexts.** Web browsers and WebViews permit microphone access only in secure contexts (`https://` or `http://127.0.0.1`). Connecting over a remote plain HTTP IP address disables browser recording capabilities. diff --git a/protocol/app-server.schema.json b/protocol/app-server.schema.json index 244448ecd..0cff18389 100644 --- a/protocol/app-server.schema.json +++ b/protocol/app-server.schema.json @@ -6274,6 +6274,12 @@ }, "provider/logout": { "$ref": "#/$defs/ProviderLogoutParams" + }, + "dictation/status": { + "$ref": "#/$defs/EmptyParams" + }, + "dictation/transcribe": { + "$ref": "#/$defs/DictationTranscribeParams" } }, "required": [ @@ -6381,7 +6387,9 @@ "provider/login/start", "provider/login/poll", "provider/login/cancel", - "provider/logout" + "provider/logout", + "dictation/status", + "dictation/transcribe" ] }, "MethodResults": { @@ -7367,6 +7375,12 @@ }, "provider/logout": { "$ref": "#/$defs/ProviderLogoutResult" + }, + "dictation/status": { + "$ref": "#/$defs/DictationStatusResult" + }, + "dictation/transcribe": { + "$ref": "#/$defs/DictationTranscribeResult" } }, "required": [ @@ -7474,7 +7488,9 @@ "provider/login/start", "provider/login/poll", "provider/login/cancel", - "provider/logout" + "provider/logout", + "dictation/status", + "dictation/transcribe" ] }, "Notifications": { @@ -8172,6 +8188,65 @@ "threadId", "expectedGoalId" ] + }, + "DictationStatusResult": { + "type": "object", + "additionalProperties": false, + "properties": { + "available": { + "type": "boolean" + }, + "model": { + "type": ["string", "null"] + }, + "maxAudioSeconds": { + "type": ["integer", "null"], + "minimum": 1, + "maximum": 600 + } + }, + "required": [ + "available", + "model", + "maxAudioSeconds" + ] + }, + "DictationTranscribeParams": { + "type": "object", + "additionalProperties": false, + "properties": { + "audio": { + "type": "string", + "minLength": 1 + }, + "mimeType": { + "type": "string", + "minLength": 1 + }, + "language": { + "type": ["string", "null"] + } + }, + "required": [ + "audio", + "mimeType" + ] + }, + "DictationTranscribeResult": { + "type": "object", + "additionalProperties": false, + "properties": { + "text": { + "type": "string" + }, + "model": { + "type": "string" + } + }, + "required": [ + "text", + "model" + ] } } } diff --git a/tests/test_config_layering.py b/tests/test_config_layering.py index dcd873e04..668e433cc 100644 --- a/tests/test_config_layering.py +++ b/tests/test_config_layering.py @@ -181,6 +181,66 @@ def test_named_connections_are_user_owned_and_project_cannot_redirect_them( assert cfg.agents.defaults.model == "moonshotai/kimi-k2.5" +def test_project_cannot_configure_or_redirect_dictation(layered): + home, project = layered + _write_config( + home, + { + "dictation": { + "endpoint": "http://127.0.0.1:8000/v1", + "model": "mlx-community/parakeet-tdt-0.6b-v3", + } + }, + ) + _write_config( + project, + { + "dictation": { + "endpoint": "https://untrusted.example/v1", + "model": "attacker-model", + } + }, + ) + + cfg = load_config_for_workspace(project) + + # The endpoint receives raw microphone audio, so a repository must not be + # able to choose it — or to turn the feature on for a user who did not. + assert cfg.dictation is not None + assert cfg.dictation.endpoint == "http://127.0.0.1:8000/v1" + assert cfg.dictation.model == "mlx-community/parakeet-tdt-0.6b-v3" + + +def test_project_dictation_is_dropped_even_without_a_user_block(layered): + _home, project = layered + _write_config( + project, + {"dictation": {"endpoint": "http://127.0.0.1:8000/v1", "model": "local"}}, + ) + + assert load_config_for_workspace(project).dictation is None + + +def test_project_dictation_is_dropped_alongside_other_overrides(layered): + # The sanitizer copies the layer before touching providers, so a project + # that also sets unrelated keys must still lose its dictation block. + _home, project = layered + _write_config( + project, + { + "workspace": {"maxInputMb": 5}, + "dictation": {"endpoint": "http://127.0.0.1:8000/v1", "model": "local"}, + "providers": {"openai": {"apiKey": "sk-project"}}, + }, + ) + + cfg = load_config_for_workspace(project) + + assert cfg.workspace.max_input_mb == 5 + assert cfg.providers.openai.api_key == "sk-project" + assert cfg.dictation is None + + def test_project_cannot_redirect_a_legacy_user_provider_credential(layered): home, project = layered _write_config( diff --git a/tests/test_dictation.py b/tests/test_dictation.py new file mode 100644 index 000000000..28b31a8a1 --- /dev/null +++ b/tests/test_dictation.py @@ -0,0 +1,408 @@ +"""Tests for the dictation clip pipeline: bytes in, transcript out. + +Two layers are covered here and nothing above them: + +- :mod:`core.dictation.audio` is pure, so every rejection path is a direct + assertion on a value rather than on a mock. +- :mod:`core.dictation.client` is exercised through an ``httpx.MockTransport``, + which is the only way to assert what actually went on the wire (multipart + shape, filename, headers) without a server. +""" + +from __future__ import annotations + +import base64 +import json +import sys +from pathlib import Path + +import httpx +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from app_server.protocol.codec import DEFAULT_MAX_MESSAGE_BYTES # noqa: E402 +from core.dictation.audio import ( # noqa: E402 + CONTAINER_EXTENSIONS, + MAX_AUDIO_BYTES, + MAX_ENCODED_AUDIO_BYTES, + MIME_CONTAINERS, + SUPPORTED_MIME_TYPES, + UnsupportedAudioError, + canonical_mime_type, + decode_audio, + extension_for, + sniff_container, +) +from core.dictation.client import SpeechToTextClient, TranscriptionFailed # noqa: E402 + +WEBM = b"\x1aE\xdf\xa3" + b"\x00" * 60 +OGG = b"OggS" + b"\x00" * 60 +MP4 = b"\x00\x00\x00\x18ftypM4A " + b"\x00" * 48 +WAV = b"RIFF" + b"\x24\x00\x00\x00" + b"WAVEfmt " + b"\x00" * 48 +MP3 = b"ID3\x03\x00" + b"\x00" * 58 +MP3_FRAME = b"\xff\xfb\x90\x00" + b"\x00" * 60 +AAC_ADTS = b"\xff\xf1\x50\x80" + b"\x00" * 60 +FLAC = b"fLaC" + b"\x00" * 60 + + +def _b64(data: bytes) -> str: + return base64.b64encode(data).decode("ascii") + + +def _client(handler, **kwargs) -> SpeechToTextClient: + return SpeechToTextClient( + "http://127.0.0.1:8000/v1", + "mlx-community/parakeet-tdt-0.6b-v3", + transport=httpx.MockTransport(handler), + **kwargs, + ) + + +# --------------------------------------------------------------------------- +# container sniffing +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("data", "expected"), + [ + (WEBM, "webm"), + (OGG, "ogg"), + (MP4, "mp4"), + (WAV, "wav"), + (MP3, "mp3"), + (MP3_FRAME, "mp3"), + (AAC_ADTS, "aac"), + (FLAC, "flac"), + ], +) +def test_sniff_container_identifies_supported_formats(data, expected): + assert sniff_container(data) == expected + + +def test_sniff_container_is_decided_by_fixed_offsets_not_length(): + # A clipped recording must be identified the same way as a complete one. + assert sniff_container(WEBM[:6]) == "webm" + assert sniff_container(WAV[:12]) == "wav" + + +def test_sniff_container_rejects_near_misses_and_noise(): + # RIFF alone is not audio: a WebP or an AVI starts the same way. + assert sniff_container(b"RIFF\x24\x00\x00\x00WEBP") is None + assert sniff_container(b"") is None + assert sniff_container(b"\x00\x01\x02\x03") is None + + +def test_adts_is_not_reported_as_mp3(): + # 0xFF 0xF1 is inside the MPEG frame-sync mask as well, so order matters. + assert sniff_container(AAC_ADTS) == "aac" + + +def test_mp4_extension_is_m4a_for_upload(): + assert extension_for("mp4") == "m4a" + + +# --------------------------------------------------------------------------- +# decode_audio +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("data", "declared", "filename"), + [ + (WEBM, "audio/webm", "dictation.webm"), + (OGG, "audio/ogg", "dictation.ogg"), + (MP4, "audio/mp4", "dictation.m4a"), + (WAV, "audio/wav", "dictation.wav"), + (MP3, "audio/mpeg", "dictation.mp3"), + (FLAC, "audio/flac", "dictation.flac"), + (AAC_ADTS, "audio/aac", "dictation.aac"), + ], +) +def test_decode_audio_names_the_upload_from_the_sniffed_container( + data, declared, filename +): + decoded, name = decode_audio(_b64(data), declared) + assert decoded == data + assert name == filename + + +def test_decode_audio_accepts_mime_parameters_and_case(): + decoded, name = decode_audio(_b64(WEBM), "Audio/WebM;codecs=opus") + + assert decoded == WEBM + assert name == "dictation.webm" + assert canonical_mime_type("Audio/WebM;codecs=opus") == "audio/webm" + + +def test_decode_audio_accepts_unpadded_base64(): + padded = _b64(WEBM) + assert padded.rstrip("=") != padded # the fixture actually has padding + + decoded, _ = decode_audio(padded.rstrip("="), "audio/webm") + + assert decoded == WEBM + + +def test_decode_audio_rejects_unknown_mime_type(): + with pytest.raises(UnsupportedAudioError) as excinfo: + decode_audio(_b64(WEBM), "video/mp4") + + message = str(excinfo.value) + assert "video/mp4" in message + assert "audio/webm" in message + + +def test_decode_audio_rejects_empty_mime_type(): + with pytest.raises(UnsupportedAudioError, match="must not be empty"): + decode_audio(_b64(WEBM), " ") + + +def test_decode_audio_rejects_bytes_that_contradict_the_declared_type(): + # WKWebView records MP4 but a client may still label it webm; the bytes win + # and the request fails instead of uploading an undecodable file. + with pytest.raises(UnsupportedAudioError, match="does not match"): + decode_audio(_b64(MP4), "audio/webm") + + +def test_decode_audio_rejects_invalid_base64(): + with pytest.raises(UnsupportedAudioError, match="valid base64"): + decode_audio("not base64!!!", "audio/webm") + + +def test_decode_audio_rejects_empty_payload(): + with pytest.raises(UnsupportedAudioError, match="must not be empty"): + decode_audio("", "audio/webm") + + +def test_decode_audio_rejects_unrecognisable_bytes(): + with pytest.raises(UnsupportedAudioError, match="not a recognisable recording"): + decode_audio(_b64(b"\x00" * 64), "audio/webm") + + +def test_decode_audio_rejects_payload_over_the_decoded_ceiling(): + oversized = b"\x1aE\xdf\xa3" + b"\x00" * MAX_AUDIO_BYTES + + with pytest.raises(UnsupportedAudioError, match="larger than"): + decode_audio(_b64(oversized), "audio/webm") + + +def test_decode_audio_rejects_payload_over_the_encoded_ceiling(): + # Checked before decoding, so the message can mention the encoded size + # without ever materialising the bytes. + with pytest.raises(UnsupportedAudioError, match="once encoded"): + decode_audio("A" * (MAX_ENCODED_AUDIO_BYTES + 4), "audio/webm") + + +def test_encoded_ceiling_leaves_room_for_the_rpc_envelope(): + # The clip travels inside one JSON-RPC message, so the encoded ceiling must + # stay clear of the codec's own limit or the request dies before any of + # this validation runs. + assert MAX_ENCODED_AUDIO_BYTES + 4096 < DEFAULT_MAX_MESSAGE_BYTES + + +def test_every_supported_mime_type_maps_to_a_container_with_an_extension(): + # The declared type is only useful if it can be checked against something + # and the result can be named on the wire. + for mime_type, containers in MIME_CONTAINERS.items(): + assert mime_type in SUPPORTED_MIME_TYPES + assert containers + for container in containers: + assert container in CONTAINER_EXTENSIONS + + +# --------------------------------------------------------------------------- +# SpeechToTextClient +# --------------------------------------------------------------------------- + + +def _capture(handler): + """Wrap a handler so the test can inspect the single request made.""" + + seen: list[httpx.Request] = [] + + def wrapped(request: httpx.Request) -> httpx.Response: + seen.append(request) + return handler(request) + + return wrapped, seen + + +def test_client_posts_multipart_with_sniffed_filename_and_model(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"text": "olá mundo"}) + + wrapped, seen = _capture(handler) + text = _client(wrapped, language="pt").transcribe( + MP4, filename="dictation.m4a", mime_type="audio/mp4" + ) + + assert text == "olá mundo" + request = seen[0] + assert request.method == "POST" + assert str(request.url) == "http://127.0.0.1:8000/v1/audio/transcriptions" + assert request.headers["content-type"].startswith("multipart/form-data; boundary=") + body = request.read() + assert b'name="file"; filename="dictation.m4a"' in body + assert b"Content-Type: audio/mp4" in body + assert b'name="model"' in body + assert b"mlx-community/parakeet-tdt-0.6b-v3" in body + assert b'name="language"' in body + assert b"\r\npt\r\n" in body + + +def test_client_omits_language_and_authorization_when_unset(): + wrapped, seen = _capture(lambda request: httpx.Response(200, json={"text": ""})) + _client(wrapped).transcribe(WEBM, filename="dictation.webm", mime_type="audio/webm") + + body = seen[0].read() + assert b'name="language"' not in body + assert "authorization" not in seen[0].headers + + +def test_client_sends_bearer_token_when_configured(): + wrapped, seen = _capture(lambda request: httpx.Response(200, json={"text": "x"})) + _client(wrapped, api_key="s3cret").transcribe( + WEBM, filename="dictation.webm", mime_type="audio/webm" + ) + + assert seen[0].headers["authorization"] == "Bearer s3cret" + + +def test_client_does_not_follow_a_redirect(): + calls: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + calls.append(request) + return httpx.Response(307, headers={"Location": "http://evil.example/v1"}) + + with pytest.raises(TranscriptionFailed) as excinfo: + _client(handler, api_key="s3cret").transcribe( + WEBM, filename="dictation.webm", mime_type="audio/webm" + ) + + # One request only: neither the audio nor the token may be re-sent to a + # destination chosen by the response. + assert len(calls) == 1 + assert excinfo.value.status_code == 307 + assert excinfo.value.retryable is False + + +def test_client_reports_status_without_echoing_the_response_body(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(500, text="SECRET-STACKTRACE api_key=abc") + + with pytest.raises(TranscriptionFailed) as excinfo: + _client(handler).transcribe( + WEBM, filename="dictation.webm", mime_type="audio/webm" + ) + + message = str(excinfo.value) + assert "HTTP 500" in message + assert "SECRET-STACKTRACE" not in message + assert "api_key" not in message + assert excinfo.value.retryable is True + + +@pytest.mark.parametrize( + ("status", "retryable"), + [(400, False), (404, False), (413, False), (429, True), (503, True)], +) +def test_client_classifies_retryability_by_status(status, retryable): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(status, text="nope") + + with pytest.raises(TranscriptionFailed) as excinfo: + _client(handler).transcribe( + WEBM, filename="dictation.webm", mime_type="audio/webm" + ) + + assert excinfo.value.status_code == status + assert excinfo.value.retryable is retryable + + +def test_client_reports_timeout_as_retryable(): + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.ReadTimeout("too slow", request=request) + + with pytest.raises(TranscriptionFailed) as excinfo: + _client(handler, timeout_seconds=1).transcribe( + WEBM, filename="dictation.webm", mime_type="audio/webm" + ) + + assert excinfo.value.retryable is True + assert "did not answer within 1s" in str(excinfo.value) + + +def test_client_reports_unreachable_endpoint_as_retryable(): + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("connection refused", request=request) + + with pytest.raises(TranscriptionFailed) as excinfo: + _client(handler).transcribe( + WEBM, filename="dictation.webm", mime_type="audio/webm" + ) + + assert excinfo.value.retryable is True + assert "unreachable" in str(excinfo.value) + + +@pytest.mark.parametrize( + ("response", "fragment"), + [ + (httpx.Response(200, text="not json"), "non-JSON body"), + (httpx.Response(200, json={"segments": []}), "without a 'text' field"), + (httpx.Response(200, json={"text": 42}), "without a 'text' field"), + ], +) +def test_client_rejects_a_200_that_is_not_a_transcript(response, fragment): + with pytest.raises(TranscriptionFailed) as excinfo: + _client(lambda request: response).transcribe( + WEBM, filename="dictation.webm", mime_type="audio/webm" + ) + + # A 200 that is not a transcript means the URL is not an ASR endpoint; + # retrying the same URL cannot help. + assert excinfo.value.retryable is False + assert fragment in str(excinfo.value) + + +def test_client_returns_empty_text_for_silence(): + # Silence is not a failure: the caller decides to ignore an empty + # transcript, and MacOS/Chromium happily produce one. + wrapped, _ = _capture(lambda request: httpx.Response(200, json={"text": ""})) + + assert ( + _client(wrapped).transcribe( + WEBM, filename="dictation.webm", mime_type="audio/webm" + ) + == "" + ) + + +def test_client_never_puts_the_api_key_in_the_url_or_error(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(401, json={"error": "invalid key"}) + + client = _client(handler, api_key="s3cret") + assert "s3cret" not in client.url + + with pytest.raises(TranscriptionFailed) as excinfo: + client.transcribe(WEBM, filename="dictation.webm", mime_type="audio/webm") + + assert "s3cret" not in str(excinfo.value) + + +def test_json_error_body_is_not_parsed_into_the_message(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(422, content=json.dumps({"detail": "LEAK"}).encode()) + + with pytest.raises(TranscriptionFailed) as excinfo: + _client(handler).transcribe( + WEBM, filename="dictation.webm", mime_type="audio/webm" + ) + + assert "LEAK" not in str(excinfo.value) diff --git a/tests/test_dictation_service.py b/tests/test_dictation_service.py new file mode 100644 index 000000000..0502851d2 --- /dev/null +++ b/tests/test_dictation_service.py @@ -0,0 +1,429 @@ +"""Tests for the dictation application service and its RPC surface. + +The service is the policy layer: whether dictation is configured, whether the +endpoint passes the egress policy, and how each failure is reported. Each of +those decisions is asserted here, including the ones that must *not* make a +request — a blocked endpoint or a malformed clip must fail before a byte +leaves the machine. +""" + +from __future__ import annotations + +import base64 +import json +import sys +from pathlib import Path + +import httpx +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from app_server.connection import ConnectionState # noqa: E402 +from app_server.dispatcher import Dispatcher, InvalidParams, Params # noqa: E402 +from core.application.application import DeepCodeApplication # noqa: E402 +from core.application.config_store import ConfigStore # noqa: E402 +from core.application.dictation_service import DictationService # noqa: E402 +from core.application.errors import ( # noqa: E402 + DictationNotConfiguredError, + DictationUnavailableError, + InvalidArgumentError, +) +from core.config import home_config_path # noqa: E402 +from core.dictation.client import SpeechToTextClient # noqa: E402 + +WEBM = b"\x1aE\xdf\xa3" + b"\x00" * 60 +ENDPOINT = "http://127.0.0.1:8000/v1" +MODEL = "mlx-community/parakeet-tdt-0.6b-v3" + +SCHEMA = ROOT / "protocol" / "app-server.schema.json" + + +def _clip() -> str: + return base64.b64encode(WEBM).decode("ascii") + + +def _config_file(tmp_path: Path, payload: dict) -> ConfigStore: + path = tmp_path / "config.json" + path.write_text(json.dumps(payload), encoding="utf-8") + return ConfigStore(path) + + +def _dictation_config(**overrides) -> dict: + block = {"endpoint": ENDPOINT, "model": MODEL} + block.update(overrides) + return {"dictation": block} + + +def _service(tmp_path: Path, payload: dict | None = None, **kwargs) -> DictationService: + store = _config_file(tmp_path, payload) if payload is not None else None + if store is None: + return DictationService( + config_store=ConfigStore(tmp_path / "absent.json"), **kwargs + ) + return DictationService(config_store=store, **kwargs) + + +def _transport(handler): + """A client factory that routes the real client through a stub transport.""" + + seen: list[httpx.Request] = [] + + def wrapped(request: httpx.Request) -> httpx.Response: + seen.append(request) + return handler(request) + + def factory(config, api_key, language): + return SpeechToTextClient( + config.endpoint, + config.model, + language=language, + api_key=api_key, + timeout_seconds=config.timeout_seconds, + transport=httpx.MockTransport(wrapped), + ) + + return factory, seen + + +# --------------------------------------------------------------------------- +# status +# --------------------------------------------------------------------------- + + +def test_status_is_unavailable_without_a_dictation_block(tmp_path): + assert _service(tmp_path).status() == { + "available": False, + "model": None, + "maxAudioSeconds": None, + } + + +def test_status_reports_the_configured_endpoint(tmp_path): + service = _service(tmp_path, _dictation_config(maxAudioSeconds=30)) + + assert service.status() == { + "available": True, + "model": MODEL, + "maxAudioSeconds": 30, + } + + +def test_status_defaults_the_audio_cap(tmp_path): + assert _service(tmp_path, _dictation_config()).status()["maxAudioSeconds"] == 120 + + +def test_status_does_not_reveal_whether_a_secret_resolves(tmp_path): + # A missing bearer token must not make the microphone look disabled: the + # user finds out on the first clip, with a message that names the variable. + payload = _dictation_config(apiKeyEnv="DEEPCODE_TEST_MISSING_KEY") + assert _service(tmp_path, payload).status()["available"] is True + + +# --------------------------------------------------------------------------- +# transcribe +# --------------------------------------------------------------------------- + + +def test_transcribe_without_a_dictation_block_is_permanently_unconfigured(tmp_path): + with pytest.raises(DictationNotConfiguredError) as excinfo: + _service(tmp_path).transcribe(audio=_clip(), mime_type="audio/webm") + + assert excinfo.value.code == "DICTATION_NOT_CONFIGURED" + assert excinfo.value.retryable is False + + +def test_transcribe_returns_text_and_the_configured_model(tmp_path): + factory, seen = _transport( + lambda request: httpx.Response(200, json={"text": "olá"}) + ) + service = _service(tmp_path, _dictation_config(), client_factory=factory) + + result = service.transcribe(audio=_clip(), mime_type="audio/webm") + + assert result == {"text": "olá", "model": MODEL} + assert str(seen[0].url) == f"{ENDPOINT}/audio/transcriptions" + assert b'filename="dictation.webm"' in seen[0].read() + + +def test_transcribe_rejects_a_malformed_clip_before_any_request(tmp_path): + factory, seen = _transport(lambda request: httpx.Response(200, json={"text": "x"})) + service = _service(tmp_path, _dictation_config(), client_factory=factory) + + with pytest.raises(InvalidArgumentError, match="valid base64"): + service.transcribe(audio="!!!", mime_type="audio/webm") + + assert seen == [] + + +def test_transcribe_rejects_an_unsupported_mime_type(tmp_path): + factory, seen = _transport(lambda request: httpx.Response(200, json={"text": "x"})) + service = _service(tmp_path, _dictation_config(), client_factory=factory) + + with pytest.raises(InvalidArgumentError, match="unsupported audio mimeType"): + service.transcribe(audio=_clip(), mime_type="video/mp4") + + assert seen == [] + + +def test_transcribe_surfaces_a_server_error_as_unavailable(tmp_path): + factory, _ = _transport(lambda request: httpx.Response(503, text="loading model")) + service = _service(tmp_path, _dictation_config(), client_factory=factory) + + with pytest.raises(DictationUnavailableError) as excinfo: + service.transcribe(audio=_clip(), mime_type="audio/webm") + + assert excinfo.value.code == "DICTATION_UNAVAILABLE" + assert excinfo.value.retryable is True + assert "loading model" not in str(excinfo.value) + + +def test_transcribe_marks_a_request_error_as_not_retryable(tmp_path): + # A 404 means the path is wrong, which retrying cannot fix; the UI must not + # offer "try again" for it. + factory, _ = _transport(lambda request: httpx.Response(404, text="not found")) + service = _service(tmp_path, _dictation_config(), client_factory=factory) + + with pytest.raises(DictationUnavailableError) as excinfo: + service.transcribe(audio=_clip(), mime_type="audio/webm") + + assert excinfo.value.retryable is False + + +def test_transcribe_passes_an_empty_transcript_through(tmp_path): + # A clip with no speech is not an error; the caller decides what to do. + factory, _ = _transport(lambda request: httpx.Response(200, json={"text": ""})) + service = _service(tmp_path, _dictation_config(), client_factory=factory) + + assert service.transcribe(audio=_clip(), mime_type="audio/webm")["text"] == "" + + +# --------------------------------------------------------------------------- +# secrets and language +# --------------------------------------------------------------------------- + + +def test_transcribe_reads_the_api_key_from_the_named_environment_variable( + tmp_path, monkeypatch +): + monkeypatch.setenv("DEEPCODE_TEST_DICTATION_KEY", "s3cret") + factory, seen = _transport(lambda request: httpx.Response(200, json={"text": "x"})) + service = _service( + tmp_path, + _dictation_config(apiKeyEnv="DEEPCODE_TEST_DICTATION_KEY"), + client_factory=factory, + ) + + service.transcribe(audio=_clip(), mime_type="audio/webm") + + assert seen[0].headers["authorization"] == "Bearer s3cret" + + +def test_transcribe_fails_when_the_named_environment_variable_is_unset( + tmp_path, monkeypatch +): + monkeypatch.delenv("DEEPCODE_TEST_DICTATION_KEY", raising=False) + factory, seen = _transport(lambda request: httpx.Response(200, json={"text": "x"})) + service = _service( + tmp_path, + _dictation_config(apiKeyEnv="DEEPCODE_TEST_DICTATION_KEY"), + client_factory=factory, + ) + + with pytest.raises( + DictationNotConfiguredError, match="DEEPCODE_TEST_DICTATION_KEY" + ): + service.transcribe(audio=_clip(), mime_type="audio/webm") + + assert seen == [] + + +def test_transcribe_sends_no_token_without_an_api_key_env(tmp_path): + factory, seen = _transport(lambda request: httpx.Response(200, json={"text": "x"})) + service = _service(tmp_path, _dictation_config(), client_factory=factory) + + service.transcribe(audio=_clip(), mime_type="audio/webm") + + assert "authorization" not in seen[0].headers + + +def test_configured_language_wins_over_the_request_hint(tmp_path): + factory, seen = _transport(lambda request: httpx.Response(200, json={"text": "x"})) + service = _service( + tmp_path, _dictation_config(language="pt"), client_factory=factory + ) + + service.transcribe(audio=_clip(), mime_type="audio/webm", language="en") + + body = seen[0].read() + assert b"\r\npt\r\n" in body + assert b"\r\nen\r\n" not in body + + +def test_request_language_is_used_when_none_is_configured(tmp_path): + factory, seen = _transport(lambda request: httpx.Response(200, json={"text": "x"})) + service = _service(tmp_path, _dictation_config(), client_factory=factory) + + service.transcribe(audio=_clip(), mime_type="audio/webm", language="de") + + assert b"\r\nde\r\n" in seen[0].read() + + +# --------------------------------------------------------------------------- +# egress policy +# --------------------------------------------------------------------------- + + +def _blocking_config(**dictation) -> dict: + payload = _dictation_config(**dictation) + payload["providers"] = {"egress": {"blockedDomains": ["127.0.0.1"]}} + return payload + + +def test_egress_policy_blocks_the_endpoint_before_the_audio_is_sent(tmp_path): + factory, seen = _transport(lambda request: httpx.Response(200, json={"text": "x"})) + service = _service(tmp_path, _blocking_config(), client_factory=factory) + + with pytest.raises(DictationUnavailableError) as excinfo: + service.transcribe(audio=_clip(), mime_type="audio/webm") + + assert "egress" in str(excinfo.value) + # Not retryable: the same call will be refused identically until the user + # changes the policy. + assert excinfo.value.retryable is False + assert seen == [] + + +def test_egress_warn_mode_records_the_denial_and_continues(tmp_path, monkeypatch): + monkeypatch.setenv("DEEPCODE_EGRESS_MODE", "warn") + factory, seen = _transport(lambda request: httpx.Response(200, json={"text": "ok"})) + service = _service(tmp_path, _blocking_config(), client_factory=factory) + + assert service.transcribe(audio=_clip(), mime_type="audio/webm")["text"] == "ok" + assert len(seen) == 1 + + +def test_egress_allows_an_endpoint_that_is_not_blocked(tmp_path): + payload = _dictation_config() + payload["providers"] = {"egress": {"allowedDomains": ["127.0.0.1"]}} + factory, seen = _transport(lambda request: httpx.Response(200, json={"text": "x"})) + service = _service(tmp_path, payload, client_factory=factory) + + service.transcribe(audio=_clip(), mime_type="audio/webm") + + assert len(seen) == 1 + + +# --------------------------------------------------------------------------- +# project scoping +# --------------------------------------------------------------------------- + + +def test_project_scoped_transcription_without_a_project_service_is_invalid(tmp_path): + service = _service(tmp_path, _dictation_config()) + + with pytest.raises(InvalidArgumentError, match="project-scoped"): + service.transcribe(audio=_clip(), mime_type="audio/webm", project_id="p1") + + +# --------------------------------------------------------------------------- +# RPC surface +# --------------------------------------------------------------------------- + + +def _dispatcher(tmp_path: Path, name: str, *, configured: bool) -> Dispatcher: + """A real application over an isolated home config. + + ``configured`` decides whether the home config carries a dictation block, + which is what the whole feature hangs off. + """ + + if configured: + home = home_config_path() + home.parent.mkdir(parents=True, exist_ok=True) + home.write_text(json.dumps(_dictation_config()), encoding="utf-8") + application = DeepCodeApplication.open(tmp_path / f"{name}.sqlite3") + return Dispatcher(application, ConnectionState(application.broker)) + + +@pytest.fixture +def unconfigured(tmp_path) -> Dispatcher: + return _dispatcher(tmp_path, "unconfigured", configured=False) + + +@pytest.fixture +def configured(tmp_path) -> Dispatcher: + return _dispatcher(tmp_path, "configured", configured=True) + + +def test_dictation_status_reports_the_capability_as_absent(unconfigured): + assert unconfigured._dictation_status(Params({})) == { + "available": False, + "model": None, + "maxAudioSeconds": None, + } + + +def test_dictation_status_reports_the_configured_endpoint(configured): + assert configured._dictation_status(Params({})) == { + "available": True, + "model": MODEL, + "maxAudioSeconds": 120, + } + + +def test_dictation_status_rejects_unknown_parameters(configured): + with pytest.raises(InvalidParams): + configured._dictation_status(Params({"audio": "x"})) + + +def test_dictation_transcribe_requires_audio_and_mime_type(configured): + with pytest.raises(InvalidParams): + configured._dictation_transcribe(Params({})) + with pytest.raises(InvalidParams): + configured._dictation_transcribe(Params({"audio": "x"})) + with pytest.raises(InvalidParams): + configured._dictation_transcribe(Params({"audio": "x", "mimeType": " "})) + + +def test_dictation_transcribe_rejects_unknown_parameters(configured): + with pytest.raises(InvalidParams): + configured._dictation_transcribe( + Params({"audio": "x", "mimeType": "audio/webm", "model": "other"}) + ) + + +def test_dictation_transcribe_reports_an_unconfigured_endpoint(unconfigured): + with pytest.raises(DictationNotConfiguredError): + unconfigured._dictation_transcribe( + Params({"audio": _clip(), "mimeType": "audio/webm"}) + ) + + +def test_dictation_transcribe_validates_the_clip_before_the_endpoint(configured): + # Same reason as the service test: nothing may be sent for a clip that + # cannot be read, so this fails without a listening endpoint. + with pytest.raises(InvalidArgumentError, match="valid base64"): + configured._dictation_transcribe( + Params({"audio": "!!!", "mimeType": "audio/webm"}) + ) + + +def test_schema_requires_the_dictation_audio_payload(): + schema = json.loads(SCHEMA.read_text(encoding="utf-8")) + definitions = schema["$defs"] + params = definitions["MethodParams"]["properties"] + + assert params["dictation/status"] == {"$ref": "#/$defs/EmptyParams"} + transcribe = definitions["DictationTranscribeParams"] + assert transcribe["required"] == ["audio", "mimeType"] + assert transcribe["properties"]["audio"]["minLength"] == 1 + assert transcribe["properties"]["language"]["type"] == ["string", "null"] + assert definitions["MethodResults"]["properties"]["dictation/transcribe"] == { + "$ref": "#/$defs/DictationTranscribeResult" + } + assert definitions["DictationStatusResult"]["properties"]["maxAudioSeconds"][ + "type" + ] == ["integer", "null"] From 9b9f07f9528d42c67d6594567182bf025e4f0c7c Mon Sep 17 00:00:00 2001 From: Eduardo Jose Costa <59846713+EduCosta85@users.noreply.github.com> Date: Sun, 20 Sep 2026 18:30:50 -0300 Subject: [PATCH 2/2] feat(dictation): support local zero-server runner via parakeet-mlx --- core/application/dictation_service.py | 13 ++- core/config.py | 5 +- core/dictation/__init__.py | 2 + core/dictation/local_runner.py | 115 ++++++++++++++++++++++++++ docs/guide/dictation.md | 28 ++++++- tests/test_dictation_service.py | 14 ++++ 6 files changed, 171 insertions(+), 6 deletions(-) create mode 100644 core/dictation/local_runner.py diff --git a/core/application/dictation_service.py b/core/application/dictation_service.py index f575668ec..9d48e33ad 100644 --- a/core/application/dictation_service.py +++ b/core/application/dictation_service.py @@ -48,6 +48,7 @@ decode_audio, ) from core.dictation.client import SpeechToTextClient, TranscriptionFailed +from core.dictation.local_runner import LocalParakeetClient from core.providers.egress import ( WARN, evaluate_provider_egress, @@ -57,14 +58,20 @@ #: Builds the client for one request from the resolved endpoint config, the API #: key (or ``None``), and the language hint to send. Injectable so tests can #: substitute a client without patching process-global state. -ClientFactory = Callable[[DictationConfig, str | None, str | None], SpeechToTextClient] +ClientFactory = Callable[[DictationConfig, str | None, str | None], Any] def _default_client_factory( config: DictationConfig, api_key: str | None, language: str | None, -) -> SpeechToTextClient: +) -> Any: + if config.endpoint == "local": + return LocalParakeetClient( + config.model, + language=language, + timeout_seconds=config.timeout_seconds, + ) return SpeechToTextClient( config.endpoint, config.model, @@ -204,6 +211,8 @@ def _egress_denial( ``providers..apiBase``, and the operator needs to be pointed at the key this endpoint actually lives under. """ + if config.endpoint == "local": + return None policy = resolve_egress_policy(loaded) decision = evaluate_provider_egress( diff --git a/core/config.py b/core/config.py index ab3e92c26..4166b5a35 100644 --- a/core/config.py +++ b/core/config.py @@ -466,9 +466,12 @@ def validate_endpoint(self): endpoint = self.endpoint.strip() if not endpoint: raise ValueError("dictation.endpoint must not be empty") + if endpoint == "local": + self.endpoint = "local" + return self if not endpoint.startswith(("http://", "https://")): raise ValueError( - "dictation.endpoint must be an absolute http:// or https:// URL" + "dictation.endpoint must be 'local' or an absolute http:// or https:// URL" ) parsed = urlsplit(endpoint) if not parsed.netloc: diff --git a/core/dictation/__init__.py b/core/dictation/__init__.py index 0907d9892..d6cc77c5e 100644 --- a/core/dictation/__init__.py +++ b/core/dictation/__init__.py @@ -27,8 +27,10 @@ SpeechToTextClient, TranscriptionFailed, ) +from core.dictation.local_runner import LocalParakeetClient __all__ = [ + "LocalParakeetClient", "MAX_AUDIO_BYTES", "MAX_ENCODED_AUDIO_BYTES", "SUPPORTED_MIME_TYPES", diff --git a/core/dictation/local_runner.py b/core/dictation/local_runner.py new file mode 100644 index 000000000..b140b766c --- /dev/null +++ b/core/dictation/local_runner.py @@ -0,0 +1,115 @@ +"""Local Speech-to-Text inference runner for Parakeet. + +Runs Parakeet models directly on Apple Silicon / local machine without requiring +a separate HTTP daemon. + +Tries loading the model via: +1. Direct Python in-process worker / `parakeet-mlx` if available. +2. CLI fallback via `parakeet-mlx` command binary. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import tempfile +from pathlib import Path + +from core.dictation.client import TranscriptionFailed + +_PARAKEET_CLI = shutil.which("parakeet-mlx") or os.path.expanduser( + "~/.local/bin/parakeet-mlx" +) + + +class LocalParakeetClient: + """Synchronous speech-to-text runner using local parakeet-mlx.""" + + def __init__( + self, + model: str, + *, + language: str | None = None, + timeout_seconds: float = 60.0, + ) -> None: + self._model = model + self._language = language + self._timeout_seconds = timeout_seconds + + @property + def url(self) -> str: + return "local:parakeet-mlx" + + def transcribe(self, audio: bytes, *, filename: str, mime_type: str) -> str: + """Run local transcription on the clip.""" + cli = _PARAKEET_CLI + if not (cli and (os.path.isfile(cli) and os.access(cli, os.X_OK))): + raise TranscriptionFailed( + "Local dictation requires 'parakeet-mlx' installed and available in PATH. " + "Run: pip install parakeet-mlx", + retryable=False, + ) + + ext = filename.split(".", 1)[-1] if "." in filename else "wav" + with tempfile.TemporaryDirectory(prefix="deepcode-dictation-") as tmpdir: + input_file = Path(tmpdir) / f"input.{ext}" + input_file.write_bytes(audio) + + cmd = [ + cli, + str(input_file), + "--model", + self._model, + "--output-format", + "json", + "--output-dir", + tmpdir, + ] + env = dict(os.environ) + env["NUMBA_DISABLE_JIT"] = "1" + + try: + proc = subprocess.run( + cmd, + capture_output=True, + text=True, + env=env, + timeout=self._timeout_seconds, + check=False, + ) + except subprocess.TimeoutExpired as exc: + raise TranscriptionFailed( + f"Local dictation did not finish within {self._timeout_seconds:g}s", + retryable=True, + ) from exc + except Exception as exc: + raise TranscriptionFailed( + f"Local dictation process failed ({type(exc).__name__})", + retryable=True, + ) from exc + + if proc.returncode != 0: + err_msg = proc.stderr.strip() or "exit code " + str(proc.returncode) + # Keep error message concise and safe + first_err = err_msg.splitlines()[-1] if err_msg else "unknown error" + raise TranscriptionFailed( + f"Local dictation failed: {first_err}", + retryable=False, + ) + + json_out = Path(tmpdir) / "input.json" + if not json_out.exists(): + return "" + + try: + data = json.loads(json_out.read_text(encoding="utf-8")) + except ValueError as exc: + raise TranscriptionFailed( + "Local dictation returned invalid json", + retryable=False, + ) from exc + + text = data.get("text", "") + return text if isinstance(text, str) else "" diff --git a/docs/guide/dictation.md b/docs/guide/dictation.md index 71ac3284d..a252c9cef 100644 --- a/docs/guide/dictation.md +++ b/docs/guide/dictation.md @@ -5,15 +5,37 @@ DeepCode supports push-to-talk voice input directly in the composer. Spoken text ## Overview - **No audio leaves without configuration.** The microphone button only appears when a `dictation` block is declared in your configuration. -- **Local first.** With a local engine such as `mlx-audio`, recordings stay entirely on your machine. +- **Two operational modes:** + - **In-process local runner (`endpoint: "local"`):** Runs transcription directly on your machine via `parakeet-mlx` without starting or maintaining any background server. + - **OpenAI-compatible server (`endpoint: "http://..."`):** For remote endpoints or local servers such as `mlx_audio.server`. - **Composer integration.** The transcribed text lands in the composer draft at the current cursor position so you can review, edit, or append to it before starting or steering a turn. - **Escape to discard.** Pressing `Escape` (or clicking the discard button) while recording drops the audio immediately without making a request. --- -## Setting up a Local Parakeet Server +## Option 1: Zero-Server Local Dictation (Recommended) -You can run NVIDIA Parakeet locally on Apple Silicon using `mlx-audio`: +If you have `parakeet-mlx` installed on your machine (`pip install parakeet-mlx`), DeepCode can transcribe clips directly in-process without any daemon. + +Add to `~/.deepcode/deepcode_config.json`: + +```json +{ + "dictation": { + "endpoint": "local", + "model": "mlx-community/parakeet-tdt-0.6b-v3", + "language": "pt" + } +} +``` + +That's it! No daemon or separate terminal needs to be kept open. + +--- + +## Option 2: Setting up a Local Parakeet HTTP Server + +You can also run NVIDIA Parakeet as a standalone HTTP server using `mlx-audio`: ```bash # 1. Install mlx-audio in an isolated environment diff --git a/tests/test_dictation_service.py b/tests/test_dictation_service.py index 0502851d2..58c62bd1b 100644 --- a/tests/test_dictation_service.py +++ b/tests/test_dictation_service.py @@ -111,6 +111,20 @@ def test_status_reports_the_configured_endpoint(tmp_path): } +def test_status_and_transcribe_support_local_endpoint(tmp_path): + service = _service( + tmp_path, + {"dictation": {"endpoint": "local", "model": MODEL}}, + client_factory=lambda cfg, key, lang: type( + "StubLocal", (), {"transcribe": lambda *a, **k: "local ok"} + )(), + ) + + assert service.status()["available"] is True + res = service.transcribe(audio=_clip(), mime_type="audio/webm") + assert res == {"text": "local ok", "model": MODEL} + + def test_status_defaults_the_audio_cap(tmp_path): assert _service(tmp_path, _dictation_config()).status()["maxAudioSeconds"] == 120