diff --git a/.CLAUDE/context/feature-availability.md b/.CLAUDE/context/feature-availability.md new file mode 100644 index 00000000..ea3e84df --- /dev/null +++ b/.CLAUDE/context/feature-availability.md @@ -0,0 +1,375 @@ +# Feature availability + +Updated: 10 Aug 2026 + +Generated from the STT product catalogue. **Do not edit by hand**: regenerate when the catalogue changes. + +Speech to Text availability values only: features, add-ons, regions, and readiness. Structure and naming are in `product-architecture.md`. + +--- + +## What the tables contain + +The tables mix three kinds of item, bundled together so availability can be read in one place. The Items section at the foot of this file states the kind of every item. + +**Features** are characteristics, capabilities, and parameters of a model in an interaction pattern. They are configured within a transcription request. + +**Add-ons are not features.** Each is a separate product bolted onto transcription, producing an output derived from a completed transcript, and each is documented on its own page rather than in a feature list. Do not describe an add-on as a feature, and do not fold add-ons into feature documentation. See Level 3 in `product-architecture.md`. + +**Regions** are processing locations, an attribute of SaaS on Cloud only. + +--- + +## Values + +Availability depends on the combination of interaction pattern, model, and deployment. Check the combination, not the product name. + +| Value | Meaning | Authoring rule | +|---|---|---| +| Yes | Available now | Document it | +| No | Not available, and not on the roadmap | Do not document. State "not supported" only where readers repeatedly ask | +| n/a | Does not apply to this combination | Do not document, and do not list it as a gap | +| Not yet | Not available, on the roadmap | Do not document | +| TBD | Availability not established | Do not document, and do not imply availability either way | + +Rows that are n/a for every model in a table are omitted from that table. + +--- + +## Readiness + +Readiness is a separate question from availability: availability says whether something exists for a combination, readiness says whether that combination is usable and documentable. + +| Value | Deployment | Meaning | Authoring rule | +|---|---|---|---| +| GA | SaaS on Cloud | Generally available | Document fully | +| Preview | SaaS on Cloud | Usable, with limitations | Document with an Info admonition: for evaluation and feedback, not production-ready and not ready to scale | +| Released | on-prem | Available from a stated container release | Document, naming the release | +| In development | either | Not available | Do not document. No feature lists, no page stubs | + +Preview applies to SaaS on Cloud only, because on-prem ships as versioned containers: the question there is which release you need. + +| Interaction pattern | Model | SaaS on Cloud | On-prem | +|---|---|---|---| +| pre-recorded | Standard | GA | Released | +| pre-recorded | Enhanced | GA | Released | +| pre-recorded | Melia 1 | GA | Released | +| streaming | Standard | GA | Released | +| streaming | Enhanced | GA | Released | +| streaming | Melia 1 | Preview | In development | +| agent STT | Linden 1 | Preview | In development | + +--- + +## SaaS on Cloud + +### Pre-recorded + +| Item | Standard | Enhanced | Melia 1 | +|---|---|---|---| +| 56 languages | Yes | Yes | Yes | +| Mixed-language transcription | No | No | Yes | +| Language hints | No | No | Yes | +| Language labeling | No | No | Yes | +| Transcription language packs (including bilingual) | Yes | Yes | n/a | +| Automatic language identification | Yes | Yes | n/a | +| Custom dictionary | Yes | Yes | TBD | +| User context (prompting) | No | No | Not yet | +| Medical domain | No | Yes | Not yet | +| Output locale | Yes | Yes | Yes | +| Smart formatting | Yes | Yes | Yes | +| Entity detection (basic, legacy) | Yes | Yes | No | +| Entity detection (advanced) | TBD | TBD | Not yet | +| PII redaction | TBD | TBD | Not yet | +| Disfluency tagging | Yes | Yes | Not yet | +| Profanity tagging | Yes | Yes | Not yet | +| Text replacement (find and replace) | Yes | Yes | Not yet | +| Punctuation and casing | Yes | Yes | Yes | +| Timings, word-level | Yes | Yes | Yes | +| Timings, segment-level | Yes | Yes | Yes | +| Confidence scores | Yes | Yes | No | +| Speaker diarization | Yes | Yes | Yes | +| Channel diarization | Yes | Yes | Yes | +| Speaker identification | Yes | Yes | Not yet | +| Audio events | Yes | Yes | No | +| Audio filtering (volume filtering) | Yes | Yes | Not yet | +| Fetch URL | Yes | Yes | Yes | +| App usage tracking | Yes | Yes | Yes | +| Notifications | Yes | Yes | Yes | +| Translation | Yes | Yes | No | +| Chapters | Yes | Yes | No | +| Topics | Yes | Yes | No | +| Summaries | Yes | Yes | No | +| Sentiment | Yes | Yes | No | +| Audio alignment | Yes | Yes | No | +| EU | Yes | Yes | Yes | +| US | Yes | Yes | Yes | +| AUS | Yes | Yes | No | + +### Streaming + +| Item | Standard | Enhanced | Melia 1 | +|---|---|---|---| +| 56 languages | Yes | Yes | Yes | +| Mixed-language transcription | No | No | Yes | +| Language hints | No | No | Not yet | +| Language labeling | No | No | Yes | +| Transcription language packs (including bilingual) | Yes | Yes | n/a | +| Automatic language identification | No | No | n/a | +| Custom dictionary | Yes | Yes | TBD | +| User context (prompting) | No | No | Not yet | +| Medical domain | No | Yes | Not yet | +| Output locale | Yes | Yes | Yes | +| Smart formatting | Yes | Yes | Yes | +| Entity detection (basic, legacy) | Yes | Yes | No | +| Entity detection (advanced) | TBD | TBD | TBD | +| PII redaction | TBD | TBD | TBD | +| Disfluency tagging | Yes | Yes | Not yet | +| Profanity tagging | Yes | Yes | Not yet | +| Text replacement (find and replace) | Yes | Yes | Not yet | +| Punctuation and casing | Yes | Yes | Yes | +| Timings, word-level | Yes | Yes | Yes | +| Timings, segment-level | No | No | TBD | +| Confidence scores | Yes | Yes | No | +| Speaker diarization | Yes | Yes | Not yet | +| Channel diarization | Yes | Yes | Yes | +| Speaker identification | Yes | Yes | Not yet | +| Audio events | Yes | Yes | No | +| Audio filtering (volume filtering) | Yes | Yes | Not yet | +| Force end of utterance | Yes | Yes | Not yet | +| Voice activity detection (VAD) | No | No | No | +| Turn detection | Yes | Yes | No | +| Partials, word-level | Yes | Yes | Yes | +| Partials, segment-level | No | No | No | +| Translation | Yes | Yes | No | +| Chapters | No | No | No | +| Topics | No | No | No | +| Summaries | No | No | No | +| Sentiment | No | No | No | +| Audio alignment | No | No | No | +| EU | Yes | Yes | Yes | +| US | Yes | Yes | Yes | +| AUS | No | No | No | + +### Agent STT + +| Item | Linden 1 | +|---|---| +| 56 languages | Yes | +| Mixed-language transcription | No | +| Language hints | No | +| Language labeling | No | +| Transcription language packs (including bilingual) | Yes | +| Automatic language identification | No | +| Custom dictionary | Yes | +| User context (prompting) | No | +| Medical domain | Yes | +| Output locale | Yes | +| Smart formatting | Yes | +| Entity detection (basic, legacy) | No | +| Entity detection (advanced) | TBD | +| PII redaction | TBD | +| Disfluency tagging | No | +| Profanity tagging | No | +| Text replacement (find and replace) | Yes | +| Punctuation and casing | Yes | +| Timings, word-level | No | +| Timings, segment-level | Yes | +| Confidence scores | No | +| Speaker diarization | Yes | +| Channel diarization | No | +| Speaker identification | Yes | +| Audio events | No | +| Audio filtering (volume filtering) | No | +| Force end of utterance | Not yet | +| Voice activity detection (VAD) | Yes | +| Turn detection | Yes | +| Partials, word-level | No | +| Partials, segment-level | Yes | +| Translation | No | +| Chapters | No | +| Topics | No | +| Summaries | No | +| Sentiment | No | +| Audio alignment | No | +| EU | Yes | +| US | Yes | +| AUS | No | + + +--- + +## On-prem + +### Pre-recorded + +| Item | Standard | Enhanced | Melia 1 | +|---|---|---|---| +| 56 languages | Yes | Yes | Yes | +| Mixed-language transcription | No | No | Yes | +| Language hints | No | No | Yes | +| Language labeling | No | No | Yes | +| Transcription language packs (including bilingual) | Yes | Yes | n/a | +| Automatic language identification | Yes | Yes | n/a | +| Custom dictionary | Yes | Yes | TBD | +| User context (prompting) | No | No | Not yet | +| Medical domain | No | Yes | Not yet | +| Output locale | Yes | Yes | Yes | +| Smart formatting | Yes | Yes | Yes | +| Entity detection (basic, legacy) | Yes | Yes | No | +| Entity detection (advanced) | TBD | TBD | Not yet | +| PII redaction | TBD | TBD | Not yet | +| Disfluency tagging | Yes | Yes | Not yet | +| Profanity tagging | Yes | Yes | Not yet | +| Text replacement (find and replace) | Yes | Yes | Not yet | +| Punctuation and casing | Yes | Yes | Yes | +| Timings, word-level | Yes | Yes | Yes | +| Timings, segment-level | Yes | Yes | Yes | +| Confidence scores | Yes | Yes | No | +| Speaker diarization | Yes | Yes | Yes | +| Channel diarization | Yes | Yes | Yes | +| Speaker identification | Yes | Yes | Not yet | +| Audio events | Yes | Yes | No | +| Audio filtering (volume filtering) | Yes | Yes | Not yet | +| Fetch URL | Yes | Yes | Yes | +| Notifications | Yes | Yes | Yes | +| Translation | Yes | Yes | No | +| Chapters | No | No | No | +| Topics | No | No | No | +| Summaries | No | No | No | +| Sentiment | Yes | Yes | No | +| Audio alignment | No | No | No | + +### Streaming + +| Item | Standard | Enhanced | Melia 1 | +|---|---|---|---| +| 56 languages | Yes | Yes | Not yet | +| Mixed-language transcription | No | No | Not yet | +| Language hints | No | No | Not yet | +| Language labeling | No | No | Not yet | +| Transcription language packs (including bilingual) | Yes | Yes | n/a | +| Automatic language identification | No | No | n/a | +| Custom dictionary | Yes | Yes | TBD | +| User context (prompting) | No | No | Not yet | +| Medical domain | No | Yes | Not yet | +| Output locale | Yes | Yes | Not yet | +| Smart formatting | Yes | Yes | Not yet | +| Entity detection (basic, legacy) | Yes | Yes | No | +| Entity detection (advanced) | TBD | TBD | TBD | +| PII redaction | TBD | TBD | TBD | +| Disfluency tagging | Yes | Yes | Not yet | +| Profanity tagging | Yes | Yes | Not yet | +| Text replacement (find and replace) | Yes | Yes | Not yet | +| Punctuation and casing | Yes | Yes | Not yet | +| Timings, word-level | Yes | Yes | Not yet | +| Timings, segment-level | No | No | TBD | +| Confidence scores | Yes | Yes | No | +| Speaker diarization | Yes | Yes | Not yet | +| Channel diarization | Yes | Yes | Not yet | +| Speaker identification | Yes | Yes | Not yet | +| Audio events | Yes | Yes | No | +| Audio filtering (volume filtering) | Yes | Yes | Not yet | +| Force end of utterance | Yes | Yes | Not yet | +| Voice activity detection (VAD) | No | No | No | +| Turn detection | Yes | Yes | No | +| Partials, word-level | Yes | Yes | Not yet | +| Partials, segment-level | No | No | No | +| Translation | Yes | Yes | No | +| Chapters | No | No | No | +| Topics | No | No | No | +| Summaries | No | No | No | +| Sentiment | No | No | No | +| Audio alignment | No | No | No | + +### Agent STT + +| Item | Linden 1 | +|---|---| +| 56 languages | Not yet | +| Mixed-language transcription | No | +| Language hints | No | +| Language labeling | No | +| Transcription language packs (including bilingual) | Not yet | +| Automatic language identification | No | +| Custom dictionary | Not yet | +| User context (prompting) | No | +| Medical domain | Not yet | +| Output locale | Not yet | +| Smart formatting | Not yet | +| Entity detection (basic, legacy) | No | +| Entity detection (advanced) | TBD | +| PII redaction | TBD | +| Disfluency tagging | No | +| Profanity tagging | No | +| Text replacement (find and replace) | Not yet | +| Punctuation and casing | Not yet | +| Timings, word-level | No | +| Timings, segment-level | Not yet | +| Confidence scores | No | +| Speaker diarization | Not yet | +| Channel diarization | No | +| Speaker identification | Not yet | +| Audio events | No | +| Audio filtering (volume filtering) | No | +| Force end of utterance | Not yet | +| Voice activity detection (VAD) | Not yet | +| Turn detection | Not yet | +| Partials, word-level | No | +| Partials, segment-level | Not yet | +| Translation | No | +| Chapters | No | +| Topics | No | +| Summaries | No | +| Sentiment | No | +| Audio alignment | No | + +--- + +## Items + +| Item | Kind | Category | Description | +|---|---|---|---| +| 56 languages | feature | Language coverage | The supported transcription language set. | +| Mixed-language transcription | feature | No language selection needed | Uses one unified multilingual model to identify each language and switch seamlessly between languages mid-sentence. No language selection. | +| Language hints | feature | No language selection needed | Optionally list the languages you expect in the audio to improve accuracy and switching. Does not restrict the model to those languages. | +| Language labeling | feature | No language selection needed | Tags the specific languages spoken in audio. | +| Transcription language packs (including bilingual) | feature | Choose the language | Choose the language you want transcribed, or a bilingual pack. | +| Automatic language identification | feature | Choose the language | Automatically detects the spoken language and selects the matching language pack. | +| Custom dictionary | feature | Output, tuning, and formatting | Boosts accuracy for names, acronyms and domain terms. | +| User context (prompting) | feature | Output, tuning, and formatting | Supply free-form or structured information unique to the customer or session: boost words (custom dictionary), custom phrases, free-text documents, or natural-language instructions on formatting and domain hinting. | +| Medical domain | feature | Output, tuning, and formatting | Tunes recognition for healthcare audio: procedures, medications, conditions and anatomy. | +| Output locale | feature | Output, tuning, and formatting | Standardizes output spelling for a locale (en-GB vs en-US). | +| Smart formatting | feature | Output, tuning, and formatting | Converts spoken entities (numbers, dates, currencies, email addresses, time, measurements) into properly formatted written text. | +| Entity detection (basic, legacy) | feature | Output, tuning, and formatting | Identifies key words and phrases and tags them with their category, using the built-in formatting entity classes. | +| Entity detection (advanced) | feature | Output, tuning, and formatting | Broader entity tagging across sensitive-information classes (PII, PHI, PCI). | +| PII redaction | feature | Output, tuning, and formatting | Removes sensitive words from the transcript based on their entity tag, for GDPR and HIPAA workflows. | +| Disfluency tagging | feature | Output, tuning, and formatting | Flags hesitations and filler words such as “umm” in the transcript. | +| Profanity tagging | feature | Output, tuning, and formatting | Flags profanity in the transcript so it can be displayed or filtered differently. | +| Text replacement (find and replace) | feature | Output, tuning, and formatting | Replaces matched words or phrases in the transcript with your own wording. | +| Punctuation and casing | feature | Output, tuning, and formatting | Punctuates and capitalizes the transcript. | +| Timings, word-level | feature | Output, tuning, and formatting | Per-word start and end timestamps. | +| Timings, segment-level | feature | Output, tuning, and formatting | Start and end timestamps for each transcript segment. | +| Confidence scores | feature | Output, tuning, and formatting | Per-word confidence to speed human review. | +| Speaker diarization | feature | Transcription separation and speakers | Separates and labels speakers it has not seen before. | +| Channel diarization | feature | Transcription separation and speakers | Processes multiple audio channels separately. | +| Speaker identification | feature | Transcription separation and speakers | Labels known speakers across recordings. | +| Audio events | feature | Audio input | Labels non-speech sounds: laughter, applause, music. | +| Audio filtering (volume filtering) | feature | Audio input | Pre-processes input audio to remove low-volume background speech which might otherwise be detected and transcribed. | +| Fetch URL | feature | Audio input | Provides audio file for pre-recorded (Batch) job from a URL. | +| Force end of utterance | feature | Responding | Ends the current utterance on demand so the final transcript is returned without waiting for a natural pause. | +| Voice activity detection (VAD) | feature | Responding | Signals when speech starts and stops, as an acoustic indicator independent of the transcript. | +| Turn detection | feature | Responding | Detects the end of a speaker turn for responsive conversational apps. | +| Partials, word-level | feature | Responding | Interim transcripts emitted word by word while audio streams in, refined before the final. | +| Partials, segment-level | feature | Responding | Interim transcripts emitted as whole segments while audio streams in, refined before the final. | +| App usage tracking | feature | Operational | Returns app usage metadata in the API response, to attribute usage across applications. | +| Notifications | feature | Operational | Callbacks when a job completes. | +| Translation | add-on | Add-ons | Up to 5 target languages per job. 34 English-paired languages plus Norwegian Bokmål to Nynorsk. | +| Chapters | add-on | Add-ons | Splits audio into titled, summarized chapters. | +| Topics | add-on | Add-ons | Detects topics discussed, with timestamps. | +| Summaries | add-on | Add-ons | Auto-generated summary of the transcript. | +| Sentiment | add-on | Add-ons | Sentiment across the conversation. | +| Audio alignment | add-on | Add-ons | Aligns an existing transcript to audio timings. Enterprise only. | +| EU | region | Regions | European Union processing region. | +| US | region | Regions | United States processing region. | +| AUS | region | Regions | Australia processing region. | diff --git a/.CLAUDE/context/product-architecture.md b/.CLAUDE/context/product-architecture.md index 45dbb3b4..cc9a7a9a 100644 --- a/.CLAUDE/context/product-architecture.md +++ b/.CLAUDE/context/product-architecture.md @@ -1,74 +1,115 @@ # Product architecture -This file describes the Speechmatics applied product architecture for use as project context. Use it to ensure accuracy when writing or reviewing docs. +Updated: 10 Aug 2026 -Last updated: 3 July 2026. +This file describes the Speechmatics applied product architecture for use as project context. Use it to check accuracy when writing or reviewing docs. + +This file describes **structure only**: which entities exist, at which level, and under which name. It holds no availability values. Which features, add-ons, and regions are available for a given combination, and how ready that combination is, is in `feature-availability.md`. + +Written in en-US, matching the docs style guide. --- -## Speech to Text +## Two views of the same product -STT is structured as a layered decision tree. Each layer resolves one choice, taking the reader from the general API down to a specific packaged product, with deployment as an orthogonal final choice. +Speech to Text is described in two ways, for two different readers. Both are correct. Neither is a subset of the other. -The layers are: API, processing mode, interaction pattern, model variant, model sub-variant, packaging, and deployment. +**API view.** Batch and Realtime are the only processing modes: the transport details for accessing the service. This is the vocabulary of the API reference and the SDKs. -### Level 0: API +**Product view.** Three interaction patterns describe how audio is presented for processing. This is the vocabulary for onboarding and explanation, and it exists because "batch" is a poor descriptor of what the Batch API actually does. -One STT API: **Speech to Text**. +This crosswalk is the authoritative mapping between the naming systems in use. -### Level 1: Processing mode +| Interaction pattern | Processing mode | Transport | `session_type` | +|---|---|---|---| +| pre-recorded | Batch | REST API | `batch` | +| streaming | Realtime | WebSocket API | `realtime` | +| agent STT | Realtime | WebSocket API | `agent` | -How the API is consumed. +`session_type` is a field on the usage endpoint. Its values name interaction patterns, not processing modes: `realtime` means streaming, and agent STT has its own value, `agent`. Do not read `realtime` as covering all Realtime traffic. -- **Realtime** (streaming; persistent WebSocket; transcripts arrive as the audio plays) -- **Batch** (asynchronous; audio submitted as a file, transcript returned when ready) +--- + +## Speech to Text -### Level 2: Interaction pattern +Speech to Text resolves through six levels, each answering one question. -How audio is presented to the API for processing. Available patterns depend on the processing mode. +### Level 0: Service -- Realtime: **streaming** (from live audio input), **agent STT** (turn-based, from live audio input; coming soon) -- Batch: **pre-recorded** (file-based, asynchronous or synchronous using long-polling) +One service: **Speech to Text**. -agent STT is an interaction pattern on the Realtime API, purpose-built as the speech-to-text layer for voice-agent applications. It is coming soon. See "Notes for writers" for the distinction between agent STT and a voice agent. +### Level 1: Interaction pattern -### Level 3: Model variant +How audio is presented for processing. -The model used to transcribe. Available model variants depend on the interaction pattern. +- **pre-recorded**: file-based, asynchronous, or synchronous using long-polling +- **streaming**: from live audio input +- **agent STT**: turn-based, from live audio input + +Processing mode, transport, and `session_type` are attributes of the interaction pattern, fixed by the choice of pattern rather than selected separately. See the crosswalk above. + +### Level 2: Model + +The model used to transcribe. Available models depend on the interaction pattern. -- streaming: **Standard**, **Enhanced** -- agent STT: **Linden 1** (coming soon) - pre-recorded: **Standard**, **Enhanced**, **Melia 1** +- streaming: **Standard**, **Enhanced**, **Melia 1** +- agent STT: **Linden 1** + +**Features** are the characteristics, capabilities, and parameters of a model in a given interaction pattern. They are attributes of this level, not a level of their own: a feature has no meaning independent of the model and pattern it applies to, and its availability varies again by deployment. Values are in `feature-availability.md`. + +### Level 3: Add-ons -Notes: -- Standard prioritizes turnaround and cost; Enhanced prioritizes accuracy. -- Melia 1 is multilingual: no language pack selection, and it requires `"language": "multi"`. Batch only today. +Outputs derived from a completed transcript, selected in addition to transcription. -### Level 4: Model sub-variant +- Translation +- Chapters +- Topics +- Summaries +- Sentiment +- Audio alignment -A separately trained, use-case- or sector-specific extension of a parent model variant. A sub-variant is more than a packaging label, but it is not a standalone model variant of its own — it depends on its parent. +Add-ons are a level rather than an attribute because each produces a new output computed from the transcript, whereas a feature conditions the input or shapes the transcript itself. Chargeability is not the classifier. -- **Medical**: tuned for medical use cases. Available on Enhanced (both streaming and pre-recorded). -- Most model variants have no sub-variant. +Availability varies by interaction pattern, model, and deployment, and is in `feature-availability.md`. -### Level 5: Packaging +### Level 4: Packaging The packaged product a customer selects and contracts against. These are the names currently in use. -| Processing mode | Interaction pattern | Model variant | Model sub-variant | Current product name | -|---|---|---|---|---| -| Realtime | streaming | Standard | — | Realtime Standard | -| Realtime | streaming | Enhanced | — | Realtime Enhanced | -| Realtime | streaming | Enhanced | Medical | Realtime Enhanced Medical | -| Realtime | agent STT | Linden 1 | — | Agent STT Linden 1 (coming soon) | -| Batch | pre-recorded | Standard | — | Batch Standard | -| Batch | pre-recorded | Enhanced | — | Batch Enhanced | -| Batch | pre-recorded | Enhanced | Medical | Batch Enhanced Medical | -| Batch | pre-recorded | Melia 1 | — | Batch Melia 1 | +| Interaction pattern | Model | Current product name | +|---|---|---| +| pre-recorded | Standard | Batch Standard | +| pre-recorded | Enhanced | Batch Enhanced | +| pre-recorded | Melia 1 | Batch Melia 1 | +| streaming | Standard | Realtime Standard | +| streaming | Enhanced | Realtime Enhanced | +| streaming | Melia 1 | Realtime Melia 1 | +| agent STT | Linden 1 | Agent STT Linden 1 | -### Level 6: Deployment +Packaging names are led by processing mode, not interaction pattern. There are no packaged products for individual features or add-ons. -Deployment is orthogonal to the layers above and applies to all packaged products: **SaaS on Cloud**, **on-prem**, and **on-device** (coming soon). See "Notes for writers" for on-device coverage. +### Level 5: Deployment + +Deployment is orthogonal to the levels above and applies to all packaged products. + +- **SaaS on Cloud**: Speechmatics-hosted +- **on-prem**: container deployment or virtual appliance +- **on-device**: in development. Feature coverage is narrower than SaaS on Cloud and on-prem; check current documentation before describing on-device support. + +**Region** is an attribute of SaaS on Cloud only. The named regions are EU, US, and AUS. On-prem and on-device are customer-hosted, so regions do not apply to them. Which regions serve which pattern and model is in `feature-availability.md`. + +--- + +## Access surfaces + +How a developer reaches the service, orthogonal to the six levels in the same way deployment is. Any access surface can call any packaged product it supports. + +- **APIs**: the Batch and Realtime APIs directly. +- **SDKs**: Speechmatics-maintained libraries wrapping the APIs, per surface and language. +- **Integrations**: third-party platforms with Speechmatics built in, such as voice agent orchestrators and no-code automation tools. + +Access surfaces do not change what a product does or where it runs. Names are in `terminology.md`. --- @@ -76,9 +117,9 @@ Deployment is orthogonal to the layers above and applies to all packaged product TTS is a separate product line. It has three customer-facing levels. -### Level 0: API +### Level 0: Service -One TTS API. +One service: Text to Speech. ### Level 1: Model @@ -86,21 +127,15 @@ Current TTS model. ### Level 2: Voices -The model ships with four English voices: - -- Sarah -- Theo -- Jack -- Megan +The model ships with four English voices: Sarah, Theo, Jack, and Megan. --- ## Notes for writers -- **Use current product names.** The packaging names in the table above (Realtime Standard, Batch Enhanced, and so on) are the names currently in use and the only ones to put in docs. A target naming scheme led by interaction pattern (streaming, pre-recorded) is anticipated but not live. Do not preempt the rename or mix the two schemes in a page. -- **STT and TTS are separate product lines.** They share the API-at-top pattern but their mid-level semantics differ. STT Level 1 is processing mode; TTS Level 1 is model. Do not assume a uniform level model across the two. -- **Model sub-variants live at Level 4**, between model variant and packaging. A sub-variant (such as Medical) is a separately trained extension of its parent model variant, tuned for a use case or sector — more than a packaging label, but dependent on its parent rather than a standalone model. -- **agent STT is an interaction pattern on the Realtime API for building voice agents.** It is coming soon, with model variant Linden 1 and packaged product Agent STT Linden 1. A *voice agent* is a full conversational pipeline (STT + LLM + STT) and is a distinct concept that Speechmatics does not sell. agent STT provides the STT layer only. Never describe a Speechmatics product as a "voice agent." -- **Melia 1 is multilingual and Batch only today.** It requires `"language": "multi"` and has no language pack selection. -- **Deployment is an orthogonal axis.** SaaS on Cloud and on-prem are the current surfaces. On-device is coming soon and its feature coverage is currently narrower than Cloud and on-prem; check current documentation before describing on-device support. -- **The decision tree is a guide, not a strict path.** A reader's product selection may skip layers. +- **Use current packaging names.** The names in the Level 4 table are the only ones to put in docs. A naming scheme led by interaction pattern is anticipated but not live. Do not preempt the rename or mix the two schemes on one page. +- **Do not replace Batch and Realtime in reference content.** They are the only processing modes and they are entrenched in the API reference and the SDKs, including client names such as `BatchClient`. Reference content uses processing modes. +- **Use interaction patterns in explanation and onboarding content.** They exist to lower the entry barrier, because "batch" implies something different from what the Batch API does. Choose the vocabulary that matches the content type rather than mixing both in one section. +- **Medical is a feature, not a model.** Forbidden product-name variants are in `terminology.md`; which models offer the medical domain is in `feature-availability.md`. +- **agent STT is an interaction pattern, not a voice agent.** A voice agent is a full conversational pipeline (STT plus LLM plus TTS) and is a distinct concept that Speechmatics does not sell. agent STT provides the speech-to-text layer only. Never describe a Speechmatics product as a voice agent. +- **Do not assume a uniform level model across STT and TTS.** STT Level 1 is interaction pattern; TTS Level 1 is model. diff --git a/.CLAUDE/context/terminology.md b/.CLAUDE/context/terminology.md index 81c1c931..9346ecf9 100644 --- a/.CLAUDE/context/terminology.md +++ b/.CLAUDE/context/terminology.md @@ -1,114 +1,199 @@ # Terminology and product naming -Consistent terminology is the foundation of both human readability and AI retrieval. This file establishes the canonical form for every term that appears in the Speechmatics docs. +Updated: 10 Aug 2026 -For product structure (processing modes, models, SKU primitives, SKU names), see `product-architecture.md`. That file is the single source for how the product is structured; this file governs only the words used to name and describe it. +Canonical names, casing, and forbidden variants for every term in the Speechmatics docs. + +This file governs words only. Product structure is in `product-architecture.md`. Availability values and feature descriptions are in `feature-availability.md`. Do not restate either here. --- ## Capitalization -Capitalize named products and APIs. Use lowercase for capabilities and deployment methods. +Capitalize named services, APIs, models, and packaged products. Lowercase features, add-ons, capabilities, and deployment methods. | ✅ Do | ❌ Don't | |---|---| | Submit a job to the Batch API. | Submit a job to the batch API. | -| Batch transcription processes pre-recorded audio. | Batch Transcription processes pre-recorded audio. | | The Standard model is recommended for most use cases. | The standard model is recommended for most use cases. | +| Enable speaker diarization. | Enable Speaker Diarization. | +| Set the medical domain. | Set the Medical Domain. | -Deployment methods are common nouns: write "on-prem deployment", "on-device deployment", "container deployment", "virtual appliance" in lowercase. The named cloud surface "SaaS on Cloud" is the exception, capitalized as shown. +"SaaS on Cloud" is the one capitalized deployment method. --- -## Canonical product and API names +## Canonical service, API, and product names | Canonical form | Do not use | |---|---| -| Realtime API | Real-time API, Real Time API, RT API | -| Batch API | batch API | | Speech to Text | Speech-to-Text, Speech-To-Text, STT (except after first-use definition) | | Text to Speech | Text-to-Speech, Text-To-Speech, TTS (except after first-use definition) | -| Standard | standard (when referring to the model by name) | -| Enhanced | enhanced (when referring to the model by name) | +| Batch API | batch API | +| Realtime API | Real-time API, Real Time API, RT API | | Speechmatics | SM, speechmatics | +### Packaged products + +Canonical names are in `product-architecture.md`. Forbidden variants: + +| Do not use | Use instead | +|---|---| +| Agent Transcription API | Agent STT Linden 1 | + --- -## Speech to Text model and interaction terms +## Structural level names -These name the layers, model variants, and packaged products in the STT decision tree. See `product-architecture.md` for how the product is structured; this section governs only the words and their casing. +| Level | Canonical name | Notes | +|---|---|---| +| L1 | interaction pattern | Lowercase. Values: pre-recorded, streaming, agent STT. | +| L2 | model | Lowercase common noun. Named models are capitalized. | +| L3 | add-on | Lowercase. | -### Structural level names +"Model variant" and "model sub-variant" are retired. Do not use either. -| Level | Canonical name | Formerly | Notes | -|---|---|---|---| -| L2 | interaction pattern | — | Lowercase. Its values are lowercase: streaming, agent STT, pre-recorded. | -| L3 | model variant | Model | Lowercase common noun. Named variants (Standard, Enhanced, Melia 1, Linden 1) are capitalized. | -| L4 | model sub-variant | Modification | Lowercase common noun. A separately trained, sector-specific extension of a parent model variant, not a standalone model. | +--- -### Model variants and packaged products +## Models | Canonical form | Do not use | Notes | |---|---|---| -| Melia 1 | Melia-1, melia 1, Melia | Model variant. Space, no hyphen. The `model` config/API value is `melia-1` (hyphenated, lowercase) — use that form only in code, config, and API references. Multilingual; Batch only today. | -| Linden 1 | Linden-1, linden 1 | Model variant for the agent STT interaction pattern. Coming soon. | -| Medical | medical domain, Medical domain | Model sub-variant of Enhanced. Capitalized when naming the sub-variant or a product (Batch Enhanced Medical); lowercase as an ordinary adjective ("medical transcription"). | -| Agent STT Linden 1 | Agent Transcription API, Agent Transcription API v1 | Packaged product: agent STT on the Linden 1 model variant. Coming soon. | +| Standard | standard (when naming the model) | | +| Enhanced | enhanced (when naming the model) | | +| Melia 1 | Melia-1, melia 1, Melia | Config value `melia-1`, used only in code and API references. Requires `"language": "multi"` and has no language pack selection. | +| Linden 1 | Linden-1, linden 1 | Serves agent STT only. | -Standard and Enhanced are also model variants; their casing is in Canonical product and API names above. +--- -### Interaction patterns +## Interaction patterns | Canonical form | Do not use | Notes | |---|---|---| -| streaming | Streaming | Interaction pattern. Lowercase, except at the start of a sentence. | -| pre-recorded | Pre-recorded, prerecorded | Interaction pattern. Lowercase, hyphenated. | -| agent STT | Voice agent transcription, voice agent transcription, agent transcription | Interaction pattern on the Realtime API: the speech-to-text layer for building voice agents. Lowercase "agent", all-caps "STT". Capitalize "Agent" only at the start of a sentence or in the product name Agent STT Linden 1. Coming soon. | +| pre-recorded | Pre-recorded, prerecorded | Lowercase, hyphenated. | +| streaming | Streaming | Lowercase except at the start of a sentence. | +| agent STT | agent transcription, voice agent transcription | Lowercase "agent", all-caps "STT". Capitalize "Agent" only at the start of a sentence or in the product name Agent STT Linden 1. | -Speechmatics does not sell a "voice agent." A voice agent is a full conversational pipeline (STT + LLM + STT) and is a distinct concept Speechmatics does not offer. agent STT provides the STT layer only, for developers building voice agents. Never describe a Speechmatics product as a "voice agent." +Never write "voice agent" to describe a Speechmatics product. See `product-architecture.md` for why. --- -## Common terms and preferred forms +## `session_type` + +A field on the usage endpoint. Values are lowercase code identifiers: `batch`, `realtime`, `agent`. The mapping to interaction patterns is in `product-architecture.md`. + +Do not describe `realtime` as covering all Realtime traffic: it means streaming only. + +--- + +## Deployment terms | Preferred form | Do not use | Notes | |---|---|---| -| on-prem | on-premise | Preferred short form. "On-premises" is acceptable only when the topic warrants an explicit description. Never "on-premise." | -| on-device | On-device, on device | Deployment method. Lowercase, hyphenated. Coming soon; feature coverage is currently narrower than SaaS on Cloud and on-prem. | -| SaaS on Cloud | SaaS on cloud, cloud SaaS | Named cloud deployment surface. "SaaS" all-caps, "Cloud" capitalized. The exception to the lowercase deployment-method rule. | -| container deployment | Container deployment | Lowercase. A deployment method, not a product name. | -| virtual appliance | Virtual Appliance | Lowercase. A deployment method, not a product name. | -| portal | Portal | The self-service console at portal.speechmatics.com where users manage API keys, usage, and billing. Lowercase unless starting a sentence. | -| diarization | diarisation | en-US spelling. Lowercase. | -| speaker diarization | Speaker Diarization | Lowercase as a noun phrase. | -| channel diarization | Channel Diarization | Lowercase as a noun phrase. | -| WebSocket | Websocket, websocket, web socket | Capital W and S. | -| API key | api key, API Key | Lowercase "key." | -| JSON | json | All-caps. | -| REST | rest | All-caps. | -| SDK | sdk | All-caps. | -| job | Job | Lowercase common noun. | -| transcript | Transcript | Lowercase unless starting a sentence. | -| Mandarin | Chinese Mandarin, Mandarin Chinese | Language name, matching the canonical row on the languages page (`cmn`). Locale names are "Simplified Mandarin" (`cmn-Hans`) and "Traditional Mandarin" (`cmn-Hant`). | +| SaaS on Cloud | SaaS on cloud, cloud SaaS | The exception to the lowercase rule. | +| on-prem | on-premise | "On-premises" only where the topic warrants it. Never "on-premise". | +| on-device | On-device, on device | Lowercase, hyphenated. | +| container deployment | Container deployment | A deployment method, not a product name. | +| virtual appliance | Virtual Appliance | A deployment method, not a product name. | +| region | Region | Lowercase. | + +--- + +## Feature and add-on names + +Lowercase as common noun phrases, sentence case in tables. Link on first use rather than redefining. Descriptions and availability are in `feature-availability.md`. + +### Language + +| Preferred form | Do not use | +|---|---| +| mixed-language transcription | multilingual transcription, code-switching transcription | +| language hints | language hinting | +| language labeling | language labelling | +| transcription language packs (including bilingual) | bilingual packs, language pack combinations | +| automatic language identification | LID, auto language detection | +| language pack | Language Pack, languagepack | + +### Output, tuning, and formatting + +| Preferred form | Do not use | +|---|---| +| custom dictionary | Custom Dictionary | +| user context | prompting, context prompting | +| medical domain | Medical, Medical Domain, medical model | +| output locale | Output Locale | +| smart formatting | numeral formatting, Smart Formatting | +| entity detection (basic, legacy) | legacy entities | +| entity detection (advanced) | advanced entities | +| PII redaction | pii redaction | +| disfluency tagging | disfluency removal | +| profanity tagging | profanity filtering | +| text replacement (find and replace) | word replacement | +| punctuation and casing | Punctuation and Casing | +| word-level timings, segment-level timings | timings word-level, timings segment-level | +| confidence scores | Confidence Scores | + +### Speakers and audio + +| Preferred form | Do not use | +|---|---| +| speaker diarization | Speaker Diarization, diarization per speaker | +| channel diarization | Channel Diarization, diarization per channel | +| speaker identification | Speaker Identification | +| audio events | Audio Events | +| audio filtering (volume filtering) | Audio Filtering | +| fetch URL | Fetch Url, fetch-url | + +### Conversational + +| Preferred form | Do not use | +|---|---| +| turn detection | Turn Detection | +| voice activity detection (VAD) | Voice Activity Detection (undefined) | +| force end of utterance | FEOU, Force End of Utterance | +| word-level partials, segment-level partials | partials word-level, partials segment-level | + +Define VAD on first use, then use the abbreviation. Do not use FEOU in user-facing content. API message names keep their exact casing in code font: `EndOfUtterance`, `StartOfTurn`, `EndOfTurn`, `AddPartialSegment`. + +### Operational and add-ons + +| Preferred form | Do not use | +|---|---| +| app usage tracking | App Usage Tracking | +| notifications | Notifications, callbacks | +| translation | Translation | +| chapters | auto chapters, Auto Chapters | +| topics | topic detection | +| summaries | summarization | +| sentiment | sentiment analysis | +| audio alignment | alignment, forced alignment | --- -## Feature and concept terms +## Integrations and SDKs + +Third-party names keep their own casing. Do not normalize them. + +| Preferred form | Do not use | +|---|---| +| LiveKit | Livekit, Live Kit | +| Pipecat | PipeCat, Pipe Cat | +| Vapi | VAPI, vapi | +| Zapier | zapier | +| Voice SDK | voice sdk, VoiceSDK, Voice sdk | +| .NET | dotnet, DotNet, .Net | -Customer-facing features and concepts that appear in the docs. Lowercase as common noun phrases unless starting a sentence. Each has a dedicated page or section; link on first use rather than redefining inline. +Name an SDK by the surface it calls, then the language in parentheses: "Realtime (Python)", "Batch (JavaScript)". Do not invent a shorter form. Never state an SDK version in docs prose: versions belong in the repository. + +--- + +## Platform organization terms | Preferred form | Do not use | Notes | |---|---|---| -| translation | Translation | Lowercase. Converting transcript output into a target language. | -| speaker identification | Speaker Identification | Lowercase. Distinct from speaker diarization: identification labels known speakers; diarization separates unknown ones. | -| custom dictionary | Custom Dictionary | Lowercase. Adds words to the recognition vocabulary for a given input. | -| audio events | Audio Events | Lowercase. Non-speech sounds (such as laughter or music) labeled in the transcript. | -| audio filtering | Audio Filtering | Lowercase. | -| language pack | Language Pack, languagepack | Lowercase. The model assets for a given language. | -| language identification | Language Identification, LID | Lowercase. Define on first use if abbreviating; prefer the full form. | -| feature discovery | Feature Discovery | Lowercase. The endpoint returning current capability metadata. | -| smart formatting | numeral formatting, Smart Formatting | Lowercase. The feature that converts spoken entities (numbers, dates, currencies, times, measurements, and more) into their written form. Avoid "numeral formatting": it is too narrow, implying numbers only and excluding non-numeric entity classes such as dates, times, and email addresses. | -| entity | Entity | Lowercase. A spoken value with a conventional written form (a number, date, currency, time, measurement, and so on) that smart formatting detects and converts. `enable_entities` exposes each entity's structure in the JSON output. | +| workspace | Workspace | Lowercase. A container for organizing work in the portal. | +| project | Project | Lowercase. A way of organizing work within a workspace, also exposed through the API. | +| management token | Management Token | Lowercase. | --- @@ -116,33 +201,54 @@ Customer-facing features and concepts that appear in the docs. Lowercase as comm | Preferred form | Do not use | Notes | |---|---|---| -| ASR | asr | All-caps. Industry term for the underlying technology. In user-facing content, prefer "Speech to Text" when naming the Speechmatics product; use ASR only when referring to the general technology or matching an established external term. | -| WER (Word Error Rate) | wer, word error rate (undefined) | All-caps abbreviation. Define on first use: full term followed by the abbreviation in parentheses. The standard accuracy metric; can exceed 100% because insertions are counted. | -| accuracy | Accuracy | Lowercase common noun. A characteristic of a model, not a tier or product. Not a synonym for WER: accuracy reflects the reader's overall experience (recognition, punctuation, formatting, diarization), whereas WER measures recognition only. | +| ASR | asr | Use only for the general technology or an established external term. Prefer "Speech to Text" when naming the Speechmatics product. | +| WER (Word Error Rate) | wer, word error rate (undefined) | Define on first use. Can exceed 100% because insertions are counted. | +| accuracy | Accuracy | A characteristic of a model, not a tier. Not a synonym for WER. | +| diarization | diarisation | en-US spelling. | +| Mandarin | Chinese Mandarin, Mandarin Chinese | Locale names: Simplified Mandarin (`cmn-Hans`), Traditional Mandarin (`cmn-Hant`). | + +--- + +## Other common terms + +| Preferred form | Do not use | +|---|---| +| portal | Portal | +| API key | api key, API Key | +| WebSocket | Websocket, websocket, web socket | +| JSON, REST, SDK | json, rest, sdk | +| job | Job | +| transcript | Transcript | +| feature discovery | Feature Discovery | +| entity | Entity | --- ## Abbreviations -Define an abbreviation on first use: full term followed by the abbreviation in parentheses. Use only the abbreviation for all subsequent references on the same page. +Define on first use: full term, then the abbreviation in parentheses. Use the abbreviation for all later references on the same page. ❌ Speech to Text converts audio to text. Speechmatics offers Speech to Text via the Realtime and Batch APIs. ✅ Speech to Text (STT) converts audio to text. Speechmatics offers STT via the Realtime and Batch APIs. -Do not define abbreviations more recognizable than their expanded form: API, JSON, SDK, REST, WebSocket. - --- ## Terms not used in user-facing content -Do not use internal codenames, deprecated product names, or internal system names in user-facing content. If you are unsure whether a term is internal, check with the docs platform owners before using it. +Internal codenames, model architecture names, deprecated product names, and marketing-only constructs must not appear in docs. -Example: legacy platform names such as "V1 SaaS" or "AWS SaaS" must not appear. Use the current product surface name instead. +| Do not use | Why | Use instead | +|---|---|---| +| Limina | Internal codename | entity detection (advanced) | +| Ursa, Ursa 2, AED | Internal model architecture names | Name the model: Standard, Enhanced, Melia 1, Linden 1 | +| Enhanced Medical, Batch Enhanced Medical, Realtime Enhanced Medical | Marketing constructs, not packaged products | Enhanced with the medical domain | +| V1 SaaS, AWS SaaS | Legacy platform names | SaaS on Cloud | +| Micro-Batch | Not a tracked interaction pattern | pre-recorded | -### Deprecated: Flow +If you are unsure whether a term is internal, ask before using it. Do not add any term in this table to `custom-words.txt`: a dictionary entry invites use. -"Flow" (and the lowercase "flow" when used as the product or feature name) referred to a standalone voice agent product and API that has been removed from the Speechmatics offering. It is deprecated and must not be documented, referenced, or reintroduced in any user-facing content. +### Deprecated: Flow -If "Flow" appears in the codebase, existing docs, release notes, or source material, treat it as a deprecated reference: remove it rather than carrying it forward. Where the surrounding content still needs a conversational speech-to-text reference, point to agent STT — the interaction pattern on the Realtime API that provides the speech-to-text layer for building voice agents (see `product-architecture.md`). Do not carry Flow's framing forward: Speechmatics does not sell a "voice agent" (a full STT + LLM + STT pipeline), so never describe agent STT, or any Speechmatics product, as a voice agent. +Flow referred to a standalone voice agent product that has been removed from the Speechmatics offering. It must not be documented, referenced, or reintroduced. Where content still needs a conversational speech-to-text reference, point to agent STT. -This rule targets the product/feature named Flow. It does not apply to ordinary uses of the word in phrases such as "authentication flow" or "data flow", which are unaffected. +This rule targets the product named Flow. Ordinary uses such as "authentication flow" or "data flow" are unaffected. diff --git a/.CLAUDE/docs-restructure-plan.md b/.CLAUDE/docs-restructure-plan.md new file mode 100644 index 00000000..57c09521 --- /dev/null +++ b/.CLAUDE/docs-restructure-plan.md @@ -0,0 +1,128 @@ +# Documentation restructure: scoping the problem + +**Prepared for:** CPO +**From:** DevX (Matt and Pete) +**Date:** 18 August 2026 +**Status:** Early-stage experiment. Not a committed plan, roadmap, or timeline. + +We picked one section of the docs, Speech to Text, and rebuilt it end to end against a +different structure, on a branch, to find out three things: what the real problems are, +what a fix looks like in practice, and roughly how much work is involved. We then looked at +Deployments through the same lens without rebuilding it, to check whether the same problems +and the same fix apply there too. + +The purpose right now is to scope this properly and work out a delivery estimate, not to +announce a rollout. This document sets out the problems we found and what we propose doing +about them. + +The test rebuild is on a branch with a working preview, not merged and not going to be, so +it can be reviewed as a concrete example rather than a description: +https://docs-git-docs-restructure-test-speechmatics.vercel.app + +--- + +## Foundations + +Three things ground every proposal below. Without them, this would just be moving files +around. + +- **The Diátaxis framework** decides what a page should be: a tutorial, a how-to, a + reference, or an explanation. A page that mixes types is a defect, not a style choice. +- **Our product architecture** is the canonical model of what we sell: interaction pattern + (pre-recorded, streaming, agent STT) by model by deployment. This is what "correct" + means when we say a page is wrong. +- **The style guide** sets the editorial rules: terminology, naming, table limits, tone. + This is what "well-written" means, independent of structure. + +--- + +## The problems + +### Speech to Text is organised by transport, not by task + +- URLs are built around **processing modes** (Batch, Realtime), which describe how audio + reaches us, not the job a customer is doing. Two different jobs, streaming and agent STT, + share one transport, so "Realtime" alone never tells a reader which one they're using. +- What a feature supports was stated **six different ways** — comparison tables, prose, + per-page tabs, and most damagingly, implied only by which folder a page lived in. + Language identification, subtitles, and the whole speech-intelligence set read as + pre-recorded-only purely because they sat under `batch/`. Nothing on those pages said so. +- We were **selling capabilities the docs didn't describe**. Agent STT and the Linden 1 + model lived only on an unlisted page, under a folder name our own terminology forbids. + Melia 1 for streaming, in Preview with customers, had no public documentation at all. + +### Deployments has the same root problem, plus its own + +- **SaaS on Cloud has no real home.** The section gives it three bullet points, then sends + readers elsewhere. On-prem gets 34 of the section's 37 pages. +- **On-prem duplicates itself.** Licensing, security, language identification, and GPU are + each documented twice, once for container deployment and once for the virtual appliance. + Networking exists as two separate documents, both only in the virtual appliance branch. + Monitoring and scaling are virtual-appliance only; troubleshooting is container-only. +- **The vocabulary is a release behind.** 95 uses of "Batch" and 88 of "Realtime" against 8 + of "pre-recorded" and 3 of "streaming" — and zero mentions of agent STT. "Operating + point", a term we retired, still appears 14 times. +- **On-prem readiness is unanswerable.** Anything available on-prem is supposed to name the + container release it arrives in. No page names one. +- **Its feature matrix is a sixth, undocumented availability mechanism** — and it's mostly + right. We checked all 21 rows against the real source of truth: 19 agree. + +--- + +## What we're doing + +The response in both areas is the same move: **organise around the product architecture's +own axes** — interaction pattern, then model, then deployment — instead of around transport +or delivery mechanism. + +### Speech to Text + +- `pre-recorded/`, `streaming/`, and `agent-stt/` replace `batch/` and `realtime/`. The old + terms stay in the API reference and in prose, per the terminology rules. +- **One availability reference**, derived from a single source of truth, covering SaaS on + Cloud and on-prem, retiring the six competing mechanisms. +- **Agent STT promoted** to a first-class, public section, marked Preview. +- **Melia 1 for streaming published**, marked Preview. +- **Features separated from add-ons** — model behaviour is not the same thing as a billable + extra, and the two were mixed together. + +### Deployments + +- Split by **deployment first** (SaaS on Cloud, on-prem), then by **task** rather than by + delivery mechanism, so a topic like licensing has one home instead of several. +- **Consolidate the duplicated on-prem topics** — licensing, security, language ID, GPU, + and networking. +- **A new page naming which models run on which on-prem release** — the readiness question + that's currently unanswerable. +- **Bring the vocabulary current** with the rest of the docs. +- **Fold the deployments feature matrix into the single availability reference**, corrected, + rather than deleting it — most of it was right and it was doing real work. + +--- + +## What this means for the estimate + +Things that affect scope and timeline, surfaced by doing the test rebuild rather than +assumed in advance: + +- **The data behind availability has its own problems.** The catalogue that feeds the + single source of truth has 10 known errors and can't express two dimensions we need — + language and latency. That's a fix outside documentation, and it sits upstream of every + availability claim in both sections. +- **On-prem container release numbers don't exist anywhere yet.** The new page above can't + be written until they do. +- **Two Preview products have real content gaps, not just structural ones.** Agent STT has + no runnable example and no limits documented. Melia 1 streaming has no feedback route, + which our own Preview rules require. +- **Deployments hasn't been reviewed by anyone who owns it.** What's above is our read of + the content alone. +- **Redirects can only be verified against a live deployment**, not a local build — worth + building into any timeline for the real rollout. +- **Content preservation is checkable mechanically** — every identifier in a retired page + can be asserted present in its replacement — which is reassuring for scope, since it means + "don't lose anything" is a testable gate, not a hope. + +--- + +This is the input we're using to size the remaining work. Deployments in particular still +needs a review from someone who owns that content before any estimate on it is reliable. diff --git a/.CLAUDE/restructure-debt.md b/.CLAUDE/restructure-debt.md new file mode 100644 index 00000000..db68b90f --- /dev/null +++ b/.CLAUDE/restructure-debt.md @@ -0,0 +1,205 @@ +# Restructure debt and open flags + +Branch: `docs-restructure-test` · PR [#302](https://github.com/speechmatics/docs/pull/302) (**must never merge**) +Updated: 12 Aug 2026 + +Everything outstanding from the STT restructure, in one place. Nothing here blocks the +preview. `New` marks something this work introduced or discovered; `Pre-existing` marks +something that was already true on `main`. + +Recovery point: tag `pre-restructure` is the last green pre-move state. + +--- + +## 1. Source of truth: the product catalogue + +`feature-availability.md` is generated from the upstream STT product catalogue. A wrong +value can only be fixed upstream and regenerated. Hand-edits are reverted by the next +regeneration, and `CLAUDE.md` forbids them. + +**`feature-availability.md` currently carries one hand-edit** — on-prem pre-recorded +notifications, `No` → `Yes`, made at the owner's request. It reverts on regeneration. + +Corrections the catalogue needs: + +| # | Item | Catalogue says | Should say | Source | +|---|---|---|---|---| +| 1 | on-prem pre-recorded notifications | `No` | `Yes` | Owner confirmed; the deployments matrix was right | +| 2 | streaming app usage tracking | absent from Items (implies `n/a`) | `Yes` | `sm-app` is documented and works | +| 3 | agent STT force end of utterance | `Not yet` | `Yes` | `ForceEndOfUtterance` shipped; it *is* the `external` profile | +| 4 | agent STT entity detection (basic) | `No` | `Yes` | `enable_entities` shipped | +| 5 | agent STT audio filtering | `No` | `Yes` | `volume_threshold` shipped | +| 6 | Feature discovery | absent from Items entirely | present | Live endpoint, has a docs page | +| 7 | Tracking | absent from Items entirely | present | Documented `JobConfig` object | +| 8 | streaming Melia 1 custom dictionary | `TBD` | `Not yet` | PR #296 draft roadmaps it — `New` | +| 9 | streaming Melia 1 translation | `No` | `Not yet` | PR #296 draft roadmaps it — `New` | +| 10 | streaming Melia 1 audio events | `No` | `Not yet` | PR #296 draft roadmaps it — `New` | + +Items 8–10 change no published output — all three values mean "do not document" — but the +values are wrong. + +### Dimensions the catalogue does not have + +- **No language dimension.** Per-language limits have nowhere to live: sentiment is + English-only, chapters has 5 unsupported languages, language identification has 10, + smart formatting has 17. This is the structural reason DEL-33895 exists. +- **No latency-control Item.** `New` — so the Melia 1 streaming `max_delay` / + `max_delay_mode` gap and its ~4s average finals latency have no home. Both are currently + documented as prose on the feature page. + +### Values needing confirmation + +- On-prem container release numbers for **every** `Released` combination. The readiness + rule requires naming the release; no page names one. +- Which `No` items do readers repeatedly ask about? The rule permits a "not supported" + statement only with that evidence, so today none are stated. +- Feature discovery and speaker focus are live and documented but absent from the + catalogue. Add them, or publish without an availability statement? + +--- + +## 2. Content gaps needing an owner + +- **agent STT has no runnable quickstart code.** The source page had only an Academy link. + Needs Python and JavaScript samples written and executed. +- **agent STT has no limits content at all** — no concurrency, session duration, or + retention — while streaming documents all three. Confirm this is a real gap, not an + omission. +- **Melia 1 streaming has no feedback route.** `New` — the readiness rule for Preview + requires inviting evaluation and feedback. The PR #296 draft's route could not ship: the + URL was the literal placeholder `https://REPLACE-WITH-FEEDBACK-FORM-URL`, tied to an + 11 September 2026 deadline. Decide the durable route, and whether the demo at + `melia-rt-preview.speechmatics.cloud` is public enough to link. +- **`streaming/ffmpeg-audio.mdx`** keeps the per-OS FFmpeg install and device enumeration + (documented nowhere else), but its legacy CLI recipe still targets `speechmatics-python` + and needs rewriting against `speechmatics-rt`. Needs an owner to write and test. +- **Voice SDK preset names disagree.** The shipped `quickstart.py` uses lowercase + `preset="scribe"`; the prose lists `SCRIBE`. Verify against the SDK. +- **`batch/usage.mdx` was merged into `administration/usage.mdx`.** Needs that page + owner's agreement. +- **Dropped from PR #296, restore if wanted:** `New` — the qualitative smart-formatting + claim "improved accuracy, e.g. recognition of email/web addresses" for Melia 1 + streaming. Left out as unverifiable, not as a rules violation. + +--- + +## 3. Structural decisions for DevX (Matt and Pete) + +- **PR [#296](https://github.com/speechmatics/docs/pull/296) now duplicates published + content.** `New` — its `docs/private/melia-1-realtime.mdx` covers the same ground as + `/speech-to-text/features/mixed-language-transcription`, + `/speech-to-text/models#melia-1-streaming` and `/get-started/authentication`. If both + land, the content exists twice, once robots-disallowed. Decide whether #296 closes in + favour of the public pages or stays as the customer handout. +- **`private/next-gen-model.mdx`** publishes "Q3 2026" for unreleased work in a public + repo, against the style guide. PR #296 replaces that line but keeps the "codenamed Omni" + framing, and points at a private page that may go away. +- **Publishing agent STT out of `/private/` makes it crawlable.** `robots.txt` disallows + `/private/`. That is a disclosure decision for whoever owns the Preview launch. +- **The "one availability mechanism" rule is stated too broadly.** `New` — as written it + forbids "a table whose axes are interaction pattern, model, deployment, region, or + readiness" anywhere outside `availability/`. That does not distinguish an availability + matrix from a legitimate side-by-side comparison, and + `agent-stt/index.mdx` has a `| Model | Standard, Enhanced, Melia 1 | Linden 1 |` row in + a streaming-vs-agent-STT comparison table that sits in the grey zone. Sharpen the rule. +- **No `pre-recorded/index.mdx` or `streaming/index.mdx`.** `New` — the plan's IA + specified index pages for both. They were deliberately not created: the content would + duplicate `interaction-patterns.mdx` and would breach the style guide's rule against + pages that only describe relationships between other pages. Model choice is now + signposted from the pattern sidebars instead. Confirm that stands. +- **The 5-column table cap** in `docs-style-guide.md` needs revisiting; the comparison + tables in PR #278 already exceed it. +- **Deployments is out of scope** — 37 pages, 5739 lines. Its matrix in + `deployments/index.md` is retained and corrected, not deleted: 19 of 21 rows agree with + `feature-availability.md`, and the 2 that diverge are catalogue gaps (items 6 and 7 + above), not rogue content. + +--- + +## 4. Build and tooling + +| Status | Issue | +|---|---| +| Pre-existing | **Mermaid renders to nothing site-wide**, including `api-ref/realtime-transcription-websocket`. Prose equivalents currently carry the information. | +| Pre-existing | **`npm run spellcheck`'s `**/*.md` glob scans `build/`** — 12 of its 14 hits are generated output. Spellcheck is not in CI. Deferred as low priority. | +| Pre-existing | **Old `.md` mirror URLs 404** after the renames, because the Vercel sources do not match `.md`. Needs wildcard entries appended *after* all exact entries. `/llms.txt` itself is clean. | +| Pre-existing | **`spec/realtime.yaml` holds ~20 absolute and protocol-relative doc links**, invisible to the link checker, rendering into the busiest API page. Two use `http://`. | +| Pre-existing | `docusaurus.config.ts` uses the deprecated `siteConfig.onBrokenMarkdownLinks`; migrate to `markdown.hooks.onBrokenMarkdownLinks` before Docusaurus v4. | +| Pre-existing | postcss-calc parse warnings on the Radix CSS. Cosmetic. | +| Pre-existing | Docusaurus 3.9.2, with 3.10.2 available. | +| New | **Sidebar `type: "link"` hrefs bypass `onBrokenLinks`.** Two now carry fragments (`models#pre-recorded-models`, `models#streaming-models`). A checker exists but lives in a scratchpad; promote it into the repo as a real gate if the pattern stays. | +| Pre-existing | **A stale redirect destination survives in `old-redirects.json`:** `/features/accuracy-language-packs` → `/speech-to-text/languages#multilingual-speech-to-text`, an anchor that no longer exists. Harmless at runtime — `redirects.json` supplies the same source with the correct `#bilingual-and-multi-language-packs` anchor and is emitted first, so Vercel's first-match-wins takes the good one. The stale entry is unreachable dead weight. Fixing it means editing a legacy historical file. | +| Pre-existing | **Seven trailing-slash destinations remain in `vercel.json`**, all from the legacy files (`/features`, `/features-other`, `/speech-capabilities` → `/speech-to-text/`; `/on-prem/virtual-appliance`; `/on-prem/containers/usage/what-data`; plus two external pypi.org URLs where it does not matter). Each internal one costs one extra normalisation hop. | + +Baselines to hold rather than fix, all pre-existing: `npx tsc` 27 errors, `biome check` 5. +Never run `biome --write` on `vercel.json` — `sync-redirects` writes it without a trailing +newline, so the two fight forever. + +--- + +## 5. Verification that only a deployment can do + +`vercel.json` and `middleware.ts` are inert under `docusaurus start`, so redirect +*behaviour* cannot be tested locally. **Run on 12 Aug 2026 against the preview — all +passed**, and it caught the trailing-slash extra hop that no local check could see: + +| Legacy URL | Result | +|---|---| +| `/voice-agents-flow/setup` | 1 hop → `/speech-to-text/agent-stt/quickstart` — was three hops | +| `/on-prem/containers` | 1 hop → `/deployments/container/accessing-images` — was a production 404 | +| `/introduction/rt-guide` | 1 hop → `/speech-to-text/streaming/quickstart` | +| `/features-other/tracking` | 1 hop → `/speech-to-text/pre-recorded/output#tracking-metadata` | +| `/features-other/auto-chapters` | 1 hop → `/speech-to-text/add-ons/chapters` | +| `/get-started/quickstart` | 1 hop → `/` | +| `/speech-to-text/batch/`, `/realtime/` | 2 hops, both to the right page — hop 1 is trailing-slash normalisation of the *visitor's* URL, which is unavoidable | + +Still outstanding: the remaining legacy Flow sources, and one `.md` mirror URL to confirm +whether wildcard entries are needed. **Algolia needs reindexing after deploy.** + +### Two things that are still measured wrong + +- **`check-redirects.ts` does not validate fragments** — it resolves the destination path + after normalisation, so a destination pointing at a non-existent anchor passes. The + fragment-aware sweep has to be run separately over all 890 entries. +- **Duplicate normalized sources are 10, not the 4 recorded earlier.** Eight are benign by + design: `normalizePath` strips fragments from sources, so a plain entry and its fragment + variants collapse to one source and only the first can ever match. The plain entry is + correctly ordered first in every case. Fragment-specific server-side redirects remain + impossible; `src/theme/Root.tsx` is the client-side shim for the two that matter. +- A fragment-aware destination checker must special-case **static assets** — `/batch.yaml` + and `/management.yaml` are real files, not pages with an `index.html`. + +--- + +## 6. Jira + +| Ticket | Action | +|---|---| +| DEL-33243 | Close as superseded. Its instruction is now forbidden by the readiness rules, and its blanket "not yet" framing was wrong for most of its scope: most items are permanent `No`, one is `TBD`, and only 5 of ~12 are actually `Not yet`. Its canonical target `models.mdx#compare-the-models` was deleted and has since been restored pointing at the new by-pattern section. | +| DEL-33895 | Keep In Progress. Four documentation ACs are already satisfied by existing content; the remainder is engineering (language count, per-language vs per-entity-class, Romanian/Turkish/Arabic). Retarget to `features/formatting.mdx` and `features/entity-detection.mdx`. Rooted in the catalogue's missing language dimension. | +| DEL-34047 | Keep open, retarget AC2 to `/speech-to-text/pre-recorded/input`. AC1 is a spec-side fix only (`spec/batch.yaml` or `scripts/generate-batch-spec.ts`); the generated page is gitignored. It was silently blocked by the `build:jobs-api-ref` rimraf bug, fixed in this branch. | +| DEL-34274 | Update the description: its premise is stale, since Melia 1 is GA on pre-recorded rather than Batch-only early access. Still correctly blocked, on Melia becoming the **default** and on the context files recording it. Retarget to `features/formatting.mdx` and `features/entity-detection.mdx`. | + +--- + +## 7. Resolved this session, for the record + +So these are not re-reported as open: + +- **Melia 1 streaming Preview endpoint** — was unknown, blocking the streaming half of + mixed-language transcription. It is `wss://preview.rt.speechmatics.com/v2`, France for + EU and Oregon for US, SaaS only, existing API keys and `rt` temporary keys both work. + It is the same host that serves agent STT. +- **`build:jobs-api-ref` cleaned the wrong directory** — `rimraf docs/api-ref/jobs` while + the plugin `outputDir` is `docs/api-ref/batch`, so spec changes never reached the + generated pages. Fixed. +- **`#compare-the-models`** — lost when the model comparison table was deleted, now + restored on the by-pattern "Choose a model" section. +- Pre-existing bugs fixed in passing: 6 protocol-relative `//api-ref` links; the + `/deployements/` typo that 404'd `/on-prem/containers` in production; 3 duplicate + `/voice-agents-flow` entries; 5 stale URLs in code comments; 2 redirect fragments + pointing at anchors that never existed; `spec/realtime.yaml`'s `#max-speakers` and + `#prefer-current-speaker` plus an `http://` link; invalid JSON (`"speaker": 'S1'`); a + second H1 in sentiment; an empty date range (2023-01-01 to 2023-01-01, should be 2024); + duplicate `default` on two TabItems; 8 inert `quickstart=` props; 4 over-length card + descriptions; 8 two-hop redirect chains. diff --git a/CLAUDE.md b/CLAUDE.md index 4f0202ea..21e38086 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,14 +1,63 @@ # Speechmatics docs +Updated: 10 Aug 2026 + This repository contains the source code and content for the Speechmatics Docs site (docs.speechmatics.com). It documents integrations with Speechmatics APIs, libraries and SDKs, as well as short technical guides and API references. Use it both to work on the site and to create or review documentation content. To start (content tasks): paste your notes, rough draft, or brief. The assistant proposes a plan before writing unless you ask it to write immediately. -## Project files +--- + +## Context files + +| File | Owns | +|---|---| +| docs-style-guide.md | Writing standards, content types, editorial governance | +| .CLAUDE/context/terminology.md | Canonical names, casing, forbidden variants | +| .CLAUDE/context/product-architecture.md | Product structure, levels, naming crosswalk | +| .CLAUDE/context/feature-availability.md | Availability and readiness values, feature descriptions. Generated: do not edit by hand | + +Consult the file that owns a topic instead of copying its content elsewhere. If two files disagree, the owning file is correct and the other is a bug: fix it rather than working around it. + +**Atlassian (Confluence and Jira).** Use only when the context files do not cover the question, or when a contributor asks. Always state that the information comes from Confluence or Jira and may be outdated. Never present it as authoritative. If a release ticket contradicts a context file, stop and flag it. Do not publish either version. + +--- + +## Ownership and review + +Every page is owned by whoever added it. The owner keeps it accurate when the product changes, and every change is reviewed before it goes live. + +When a feature ships, changes, or is deprecated, the responsible team identifies affected pages and updates them before or at release. Documentation updates are not a post-release task. -docs-style-guide.md — writing standards, content types, and governance -.CLAUDE/context/terminology.md — canonical product names, preferred terms, and terms to avoid -.CLAUDE/context/product-architecture.md — STT and TTS product structure for accuracy checks +Major changes and structural decisions (content location, categorization, information architecture, and the context files) are owned by Matt and Pete at DevX. If in doubt whether a change is structural, consult them before making it. + +Editorial governance for individual pages, including the pre-publication quality checklist, when not to create a page, and how to handle deprecated content, is in docs-style-guide.md. + +### Maintaining the context files + +`feature-availability.md` is generated from the STT product catalogue. Regenerate it rather than editing it, and do not hand-correct a value: fix the catalogue and regenerate. + +The other three files are hand-maintained. A change to any of them is a structural change. + +--- + +## Behavioral rules + +**Apply the style guide automatically.** Every document produced follows [docs-style-guide.md](docs-style-guide.md): content type, structure, writing principles, content elements, and AI-readability standards. Do not wait to be asked. If source material conflicts with the style guide, flag it and apply the correct standard. + +**Check terminology.** Verify every product name, API name, model name, feature name, and technical term against terminology.md. Flag deviations in source material before producing output. + +**Verify product claims.** Check statements about structure against product-architecture.md. Before documenting any feature, add-on, or region, check its availability and readiness in feature-availability.md and follow the authoring rules stated there. Do not reproduce claims that conflict with either file. + +**Default mode is planning.** When given partial input, before writing: identify the Diátaxis content type, propose a structure (title, H2 sections, approximate scope), flag any terminology or style guide issues in the source material, then ask for confirmation before proceeding. + +**Auto mode.** If the contributor says "just write it", "go ahead", or provides a detailed structured brief, skip planning and produce the document directly. + +**Word count discipline.** Produce the minimum words the reader needs to act. Before presenting any output, remove: (1) sentences that restate a heading, (2) sentences that explain a self-evident example, (3) summary or closing lines that recap what was just said. + +**Flag violations.** Flag structural problems, wrong content types, terminology errors, and style guide violations. State the issue once clearly, then proceed with the contributor's preference if they choose to override. + +--- ## Tech stack @@ -50,14 +99,3 @@ The project uses cspell (`cspell.json`) for spell checking. Add new technical te ## API reference We generate API reference pages from the YAML files in the spec/ folder. There are scripts in package.json which handle this process. Check there if you need more information on their workings. - -## Behavioral rules - -**Apply the style guide automatically.** Every document produced follows [docs-style-guide.md](docs-style-guide.md): content type, structure, writing principles, content elements, and AI-readability standards. Do not wait to be asked. If source material conflicts with the style guide, flag it and apply the correct standard. -**Check terminology.** Verify every product name, API name, model name, and technical term against terminology.md. Flag deviations in source material before producing output. -**Verify product claims.** Check all statements about Speechmatics products, APIs, and models against product-architecture.md. Do not reproduce claims that conflict with it. -**Atlassian (Confluence and Jira).** Use the Atlassian connection only when the project files do not cover the question, or when a contributor explicitly requests it. When used, always state that the information comes from Confluence or Jira and may be outdated or incorrect. Never present it as authoritative. Project files always take precedence over Atlassian content. -**Default mode — planning.** When given partial input, before writing:1. Identify the Diátaxis content type.2. Propose a structure: title, H2 sections, approximate scope.3. Flag any terminology or style guide issues in the source material.4. Ask for confirmation before proceeding. -**Auto mode.** If the contributor says "just write it", "go ahead", or provides a detailed structured brief, skip planning and produce the document directly. -**Word count discipline.** Produce the minimum words the reader needs to act. Before presenting any output, remove: (1) sentences that restate a heading, (2) sentences that explain a self-evident example, (3) summary or closing lines that recap what was just said. -**Flag violations.** Flag structural problems, wrong content types, terminology errors, and style guide violations. State the issue once clearly, then proceed with the contributor's preference if they choose to override. diff --git a/docs-style-guide.md b/docs-style-guide.md index 2ca7f1ae..9ab7d5cc 100644 --- a/docs-style-guide.md +++ b/docs-style-guide.md @@ -2,11 +2,13 @@ # Speechmatics Docs Style Guide +Updated: 10 Aug 2026 + Source of truth for writing, structuring, and maintaining content on docs.speechmatics.com. -**Scope:** docs.speechmatics.com only. Does not govern speechmatics.com, portal.speechmatics.com, or Confluence. +**Scope:** docs.speechmatics.com only. -**Owners:** Matt and Pete at DevX. Flag gaps or propose changes in the shared Claude project. +**Ownership:** Matt and Pete at DevX. See Ownership and review in CLAUDE.md. **Using this guide:** Start with Content types before writing anything. Use the quality checklist in Governance before publishing. @@ -219,7 +221,7 @@ The page's main topic. Maximum 5 words. Sentence case only. Must make sense out ## Description -A single sentence summarizing what the reader will do or learn. Appears below the title. Also used as the frontmatter `description` field for search and AI retrieval. +A single sentence summarizing what the reader will do or learn. Appears below the title. Second-person imperative, active voice, present tense, sentence case. Maximum 158 characters. @@ -244,7 +246,7 @@ A concise label for a major section. Sentence case only. Must name the subject e ## Subheading (H3+) -Use for finer distinctions within a section. Never immediately follow a heading — always insert at least one paragraph of prose between an H2 and an H3. +Use for finer distinctions within a section. Never immediately follow a heading: always insert at least one paragraph of prose between an H2 and an H3. --- @@ -291,6 +293,17 @@ Use for comparisons, structured properties, and listings where column relationsh - **Accessibility:** include a brief `aria-label` or caption. - **Size:** maximum 5 columns and 10 rows. Split larger datasets. - **Caption** (optional): one sentence, maximum 80 characters. + +### Catalogue tables + +A catalogue table lists every member of a set for lookup, not comparison. It may exceed 10 rows if it meets all three conditions: + +1. Maximum 3 columns. +2. It is the only table on the page. +3. Lookup is the page's entire purpose. + +Splitting a catalogue table fragments the lookup and makes it harder to scan, so the row cap does not apply. The 5-column cap still applies, and comparison tables never qualify. + --- ## Card @@ -345,16 +358,13 @@ AI-readability is not a separate concern from good writing. Both require clear s ## Frontmatter -Every page requires `title` and `description` in frontmatter. +Every page requires `title` in frontmatter. `title` must accurately reflect the page content. - -`description` is a single benefit-led sentence. Same rules as the Description content element. Maximum 158 characters. Do not leave empty or auto-generated. - + ```yaml --- title: Configure speaker diarization -description: Enable the Speechmatics API to identify and label individual speakers in a transcript. --- ``` @@ -383,7 +393,7 @@ One concept per H2 section. Avoid forward and backward references ("as mentioned ## Terminology consistency -Use one term per concept. Do not introduce a term by one name and later refer to it by another on the same page. Full rules are in the terminology file. +Use one term per concept. Do not introduce a term by one name and later refer to it by another on the same page. Full rules are in `.CLAUDE/context/terminology.md`. ❌ "Speaker diarization separates a transcript... The diarisation feature can be enabled..." → ✅ "Speaker diarization (referred to as diarization throughout this page) separates a transcript into distinct speakers." @@ -426,7 +436,6 @@ Every page must satisfy the following before going live. **Structure** - Content type is identifiable (the arbitration rule passes) - Page title follows naming conventions -- Frontmatter `description` is present and complete - Content is not duplicated on another page **Content** - All code snippets have a context sentence @@ -450,13 +459,9 @@ Every page must satisfy the following before going live. --- -## Ownership - -Every page has an owner responsible for keeping it accurate when the product changes. - -When a feature ships, changes, or is deprecated, the responsible team identifies affected pages and updates them before or at release. Documentation updates are not a post-release task. +## Unreleased work -Structural decisions — content location, categorization, information architecture — are escalated to the docs platform owners: Matt and Pete at DevX. This responsibility will transfer to a Documentation Lead if that role is created. +Never publish a date for unreleased work. This repository is public. --- @@ -472,4 +477,4 @@ Structural decisions — content location, categorization, information architect ## Proposing changes -Contact Matt or Pete at DevX directly with the proposed edit and a one-sentence rationale. Do not edit the project files or instructions before the proposal has been approved! \ No newline at end of file +Contact Matt or Pete at DevX directly with the proposed edit and a one-sentence rationale. Do not edit the context files or instructions before the proposal has been approved! \ No newline at end of file diff --git a/docs/administration/projects.mdx b/docs/administration/projects.mdx index b0f6f771..efb55a33 100644 --- a/docs/administration/projects.mdx +++ b/docs/administration/projects.mdx @@ -5,7 +5,7 @@ description: Organize work inside a workspace, scope API keys to projects, and t # Projects -A project is an organizational unit inside a [workspace](/administration/workspaces-concepts). API keys, the transcripts they produce, and any voice agent configurations belong to a project. Any member of the workspace can access any project within it. +A project is an organizational unit inside a [workspace](/administration/workspaces-concepts). API keys and the transcripts they produce belong to a project. Any member of the workspace can access any project within it. Each workspace starts with a default project, which cannot be deleted. A workspace can have up to 1000 projects. If you need more, use [temporary keys](/get-started/authentication) or contact [support@speechmatics.com](mailto:support@speechmatics.com). diff --git a/docs/administration/usage.mdx b/docs/administration/usage.mdx index f6cfb362..481116cb 100644 --- a/docs/administration/usage.mdx +++ b/docs/administration/usage.mdx @@ -3,6 +3,9 @@ title: Usage description: Track transcription usage by processing mode and model in the portal and through the API. --- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + # Usage :::info @@ -24,14 +27,48 @@ The models shown depend on the processing mode: - **Realtime**: Standard and Enhanced. - **Batch**: Standard, Enhanced, and Melia 1. -Medical and agent variants are reported under their underlying model, not as separate lines. For example, Batch usage on the medical variant appears under Enhanced. +Usage on the medical domain and on agent STT is reported under the underlying model, not as a separate line. For example, Batch usage with the medical domain appears under Enhanced. ## Track usage with the API -To retrieve usage data programmatically, use the Usage API. See [Get usage statistics](/api-ref/batch/get-usage-statistics) in the API reference for endpoint details. +To retrieve usage data programmatically, use the Usage API. Usage is reported as the number of jobs successfully processed in a time period, along with the total duration of the corresponding audio files. The Usage API currently reports Batch usage only. +Usage is queried with an API key, and is returned for the account associated with that key. All usage is reported on a Coordinated Universal Time (UTC) calendar day basis, and usage for the current UTC day is **not** included in the results. Deleting a completed job has no effect on reported usage. + +### Request usage + +This request retrieves usage for all jobs from the beginning of 1 January 2023 until the end of 31 December 2023. More precisely, it includes all jobs completed successfully at or after 2023-01-01T00:00Z and before 2024-01-01T00:00Z. + +The `since` and `until` parameters are both optional: + +- If `since` is omitted, usage is reported from the day on which the first job was successfully processed for the account. +- If `until` is omitted, usage is reported until the end of the previous UTC day. + + + + +```bash +curl -L -X GET "https://eu1.asr.api.speechmatics.com/v2/usage?since=2023-01-01&until=2023-12-31" \ + -H "Authorization: Bearer $API_KEY" +``` + + + + +```powershell +curl -L -X GET "https://eu1.asr.api.speechmatics.com/v2/usage?since=2023-01-01&until=2023-12-31" ` + -H "Authorization: Bearer $API_KEY" +``` + + + + +### Response + +The response is a JSON object containing usage information. See [Get usage statistics](/api-ref/batch/get-usage-statistics) in the API reference for full details. + ## Next steps - [Billing](/administration/billing): understand how usage translates into charges. diff --git a/docs/deployments/container/batch-persistent-worker.mdx b/docs/deployments/container/batch-persistent-worker.mdx index 1479c163..9da2850e 100644 --- a/docs/deployments/container/batch-persistent-worker.mdx +++ b/docs/deployments/container/batch-persistent-worker.mdx @@ -126,7 +126,7 @@ asyncio.run(main()) ### Fetching audio from a URL -As well as uploading audio files directly, you can have the worker fetch them from a remote URL by adding a `fetch_data` section to the job config. You can find an example of this below. The full config options are documented [here](/speech-to-text/batch/input#fetch-url). +As well as uploading audio files directly, you can have the worker fetch them from a remote URL by adding a `fetch_data` section to the job config. You can find an example of this below. The full config options are documented [here](/speech-to-text/pre-recorded/input#fetch-url). ```bash curl -X POST address.of.container:PORT/v2/jobs \ diff --git a/docs/deployments/container/gpu-translation.mdx b/docs/deployments/container/gpu-translation.mdx index b0d1dbd9..b0418060 100644 --- a/docs/deployments/container/gpu-translation.mdx +++ b/docs/deployments/container/gpu-translation.mdx @@ -87,7 +87,7 @@ Available GPU memory after models loaded: 35GB Batch and Realtime (RT) transcribers handle sending requests to the Translation Inference Server. To run a transcription job with Translation, follow the [instructions for running the CPU Container](/deployments/container/cpu-speech-to-text) and additionally: - Set the environment variable `SM_TRANSLATION_ENDPOINT` in the transcriber to the GRPC endpoint of the running Translation Inference Server, in the form `:` where the port is the one bound to port 8001 of the Translation Docker Container (see [running the image](#running-the-image)) -- Include a `translation_config` inside of your job config. [More details](/speech-to-text/features/translation) +- Include a `translation_config` inside of your job config. [More details](/speech-to-text/add-ons/translation) - Use a transcriber version 10.3.0 or newer - Ensure you use a license which allows Translation diff --git a/docs/deployments/container/language-id.mdx b/docs/deployments/container/language-id.mdx index a4f5d1f5..9c5742da 100644 --- a/docs/deployments/container/language-id.mdx +++ b/docs/deployments/container/language-id.mdx @@ -9,7 +9,7 @@ keywords: [speechmatics, container, language ID, lang ID, language, ASR, SaaS, o This guide will walk you through the steps needed to deploy the Speechmatics Batch Language Identification Container. :::tip -Looking for how to use this in cloud SaaS? See the documentation [here](/speech-to-text/batch/language-identification). +Looking for how to use this on SaaS on Cloud? See the documentation [here](/speech-to-text/pre-recorded/language-identification). ::: This Container will allow you to predict the most likely, predominant language spoken in a media file. You can use the predicted language to select the correct transcriber when the language spoken in your file is unknown. diff --git a/docs/deployments/index.md b/docs/deployments/index.md index 0c45d215..3be5ae69 100644 --- a/docs/deployments/index.md +++ b/docs/deployments/index.md @@ -12,8 +12,8 @@ Leverage Speechmatics’ cloud services for easy, scalable, and fully managed sp The best way to get started using Speechmatics' cloud services is: - Create an account in our [Portal](https://portal.speechmatics.com/) -- Check out our [Realtime transcription](/speech-to-text/realtime/quickstart.mdx) -- Check out our [Batch transcription](/speech-to-text/batch/quickstart.mdx) +- Check out our [Realtime transcription](/speech-to-text/streaming/quickstart) +- Check out our [Batch transcription](/speech-to-text/pre-recorded/quickstart) ## On-prem @@ -32,23 +32,23 @@ Feature availability varies depending on the deployment method you choose. Below | Feature | Modes | Deployments | | ------------------------------------------------------------------------------------- | --------------- | ------------- | | [Multilingual speech to text](/speech-to-text/languages#bilingual-and-multi-language-packs) | Batch, Realtime | SaaS, On-prem | -| [Alignment](/speech-to-text/batch/alignment) | Batch | SaaS | +| [Alignment](/speech-to-text/add-ons/audio-alignment) | Batch | SaaS | | [Audio events](/speech-to-text/features/audio-events) | Batch, Realtime | SaaS, On-prem | | [Audio filtering](/speech-to-text/features/audio-filtering) | Batch, Realtime | SaaS, On-prem | -| [Auto chapters](/speech-to-text/batch/speech-intelligence/auto-chapters) | Batch | SaaS | +| [Auto chapters](/speech-to-text/add-ons/chapters) | Batch | SaaS | | [Custom dictionary](/speech-to-text/features/custom-dictionary) | Batch, Realtime | SaaS, On-prem | -| [Diarization](/speech-to-text/features/diarization) | Batch, Realtime | SaaS, On-prem | -| [Disfluencies and word replacement](/speech-to-text/formatting#disfluencies) | Batch, Realtime | SaaS, On-prem | +| [Diarization](/speech-to-text/features/speaker-diarization) | Batch, Realtime | SaaS, On-prem | +| [Disfluencies and word replacement](/speech-to-text/features/disfluencies-and-profanities#disfluencies) | Batch, Realtime | SaaS, On-prem | | [Feature discovery](/speech-to-text/features/feature-discovery) | Batch, Realtime | SaaS | -| [Fetch URL](/speech-to-text/batch/input#fetch-url) | Batch | SaaS, On-Prem | -| [Language identification](/speech-to-text/batch/language-identification) | Batch | SaaS | -| [Notifications](/speech-to-text/batch/notifications.md) | Batch | SaaS, On-prem | -| [Punctuation settings](/speech-to-text/formatting#punctuation) | Batch, Realtime | SaaS, On-prem | -| [Sentiment analysis](/speech-to-text/batch/speech-intelligence/sentiment-analysis) | Batch | SaaS, On-prem | -| [Smart formatting](/speech-to-text/formatting#smart-formatting) | Batch, Realtime | SaaS, On-prem | +| [Fetch URL](/speech-to-text/pre-recorded/input#fetch-url) | Batch | SaaS, On-prem | +| [Language identification](/speech-to-text/pre-recorded/language-identification) | Batch | SaaS, On-prem | +| [Notifications](/speech-to-text/pre-recorded/notifications) | Batch | SaaS, On-prem | +| [Punctuation settings](/speech-to-text/features/formatting#punctuation) | Batch, Realtime | SaaS, On-prem | +| [Sentiment analysis](/speech-to-text/add-ons/sentiment) | Batch | SaaS, On-prem | +| [Smart formatting](/speech-to-text/features/formatting#smart-formatting) | Batch, Realtime | SaaS, On-prem | | [Speaker identification](/speech-to-text/features/speaker-identification) | Batch, Realtime | SaaS, On-prem | -| [Summarization](/speech-to-text/batch/speech-intelligence/summarization) | Batch | SaaS | -| [Topic detection](/speech-to-text/batch/speech-intelligence/topic-detection) | Batch | SaaS | -| [Tracking](/speech-to-text/batch/output#tracking-metadata) | Batch, Realtime | SaaS, On-prem | -| [Translation](/speech-to-text/features/translation) | Batch, Realtime | SaaS, On-prem | -| [Turn detection](/speech-to-text/realtime/turn-detection) | Realtime | SaaS, On-prem | +| [Summarization](/speech-to-text/add-ons/summaries) | Batch | SaaS | +| [Topic detection](/speech-to-text/add-ons/topics) | Batch | SaaS | +| [Tracking](/speech-to-text/pre-recorded/output#tracking-metadata) | Batch, Realtime | SaaS, On-prem | +| [Translation](/speech-to-text/add-ons/translation) | Batch, Realtime | SaaS, On-prem | +| [Turn detection](/speech-to-text/features/turn-detection) | Realtime | SaaS, On-prem | diff --git a/docs/deployments/kubernetes/index.mdx b/docs/deployments/kubernetes/index.mdx index 5b8b25d4..93615e1e 100644 --- a/docs/deployments/kubernetes/index.mdx +++ b/docs/deployments/kubernetes/index.mdx @@ -25,4 +25,4 @@ Using Helm, customers can customize deployments through configurable values, man ## Supported applications Speechmatics Kubernetes deployment supports the following applications: - - [Realtime](/speech-to-text/realtime/quickstart): Stream audio from an input device or file and receive real-time transcription updates as audio is processed. \ No newline at end of file + - [Realtime](/speech-to-text/streaming/quickstart): Stream audio from an input device or file and receive real-time transcription updates as audio is processed. \ No newline at end of file diff --git a/docs/deployments/virtual-appliance/administration/language-identification.mdx b/docs/deployments/virtual-appliance/administration/language-identification.mdx index f223a516..672b0546 100644 --- a/docs/deployments/virtual-appliance/administration/language-identification.mdx +++ b/docs/deployments/virtual-appliance/administration/language-identification.mdx @@ -22,4 +22,4 @@ If you set `max_thread_count` to 2 and `memory_per_thread` to 3Gi, 6Gi will be r assigned to each thread. -You can learn more about Language Identification in the [SaaS documentation](/speech-to-text/batch/language-identification). +You can learn more about Language Identification in the [SaaS documentation](/speech-to-text/pre-recorded/language-identification). diff --git a/docs/get-started/authentication.mdx b/docs/get-started/authentication.mdx index b34a6ed9..67c5fd1b 100644 --- a/docs/get-started/authentication.mdx +++ b/docs/get-started/authentication.mdx @@ -1,12 +1,13 @@ --- +title: Authentication toc_max_heading_level: 3 -description: 'Learn about how the Speechmatics API handles authentication' +description: Authenticate requests to the Speechmatics APIs with an API key or a temporary key. keywords: [batch, file, transcription, speech recognition, asr, api, jwt] --- import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# Authentication +# Authentication Your API key must be used with any interaction with the Speechmatics API to authenticate to the service. Any interaction without this key will receive a `HTTP 401 - Unauthorized` response. @@ -24,7 +25,7 @@ Enter a name for your API key and then store it somewhere safe when you have gen -Your API key needs to be included in the header of all requests to the Speechmatics Jobs API. For example: +Your API key needs to be included in the header of all requests to the Batch API. For example: ```bash curl -X GET "https://eu1.asr.api.speechmatics.com/v2/jobs/" \ @@ -32,7 +33,7 @@ curl -X GET "https://eu1.asr.api.speechmatics.com/v2/jobs/" \ ``` - + For server-side calls, your API key must be provided in the header of the [WebSocket connection request](/api-ref/realtime-transcription-websocket#handshake-responses). For example: ```bash @@ -55,16 +56,18 @@ wss://eu.rt.speechmatics.com/v2?jwt=$TEMP_KEY ## Supported endpoints -All production endpoints are active and highly available. You can use multiple regions to balance requests or provide a failover in the event of disruption to one region. +The Batch and Realtime endpoints below are active and highly available. You can use multiple regions to balance requests or provide a failover in the event of disruption to one region. Combinations in Preview are served from a separate host, `preview.rt.speechmatics.com`, and the production availability statement does not apply to them. + +The three tabs map to the three interaction patterns: Batch serves pre-recorded, Realtime serves streaming, and agent STT has its own host. See [Understanding interaction patterns](/speech-to-text/interaction-patterns). :::info If you are an enterprise customer and would like to use a different region than the one specified in your contract, please contact your account manager or [support](https://support.speechmatics.com). ::: - - + + -Speechmatics Batch SaaS supports the following endpoints for production use: +The Batch API on SaaS on Cloud supports the following endpoints for production use: | Customer type | Region | Endpoint | | ------------- | --------------- | -------------------------------- | @@ -79,13 +82,13 @@ Jobs are created in the region corresponding to the endpoint used. You must use The Melia 1 model is available in the EU1 and US1 regions only. For details, refer to [Models](/speech-to-text/models#melia-1). :::warning -The EU2 and US2 Batch SaaS endpoints are provided for enterprise customer high availability and failover purposes only. Jobs created in these environments will not be visible in the Portal. +The EU2 and US2 Batch endpoints are provided for enterprise customer high availability and failover purposes only. Jobs created in these environments will not be visible in the Portal. ::: - + -Speechmatics Realtime SaaS supports the following endpoints for production use: +The Realtime API on SaaS on Cloud supports the following endpoints for production use: | Region | Endpoint | | ------------ | ------------------------------ | @@ -95,10 +98,31 @@ Speechmatics Realtime SaaS supports the following endpoints for production use: `global.rt.speechmatics.com` automatically routes each connection to the nearest region for lowest latency, so you don't have to select one. The regional endpoints remain available if you want to pin to a specific region. +Streaming transcription is not available in the AUS region. + +Streaming with the Melia 1 model is in Preview on a separate endpoint, `wss://preview.rt.speechmatics.com/v2`, served from the EU and the US. Your existing API keys and `rt` temporary keys work against it. See [Mixed-language transcription](/speech-to-text/features/mixed-language-transcription). + :::warning `global.rt.speechmatics.com` may route a connection to any region. If you have data residency or compliance requirements, use a regional endpoint (`eu.rt.speechmatics.com` or `us.rt.speechmatics.com`) instead. ::: + + +Agent STT is in [Preview](/speech-to-text/agent-stt/#preview-limitations) and uses a separate preview endpoint. The production availability statement above does not apply to it. + +| Interaction pattern | Endpoint | +| --- | --- | +| agent STT | **preview.rt.speechmatics.com** | + +Select a [turn detection profile](/speech-to-text/agent-stt/profiles) in the connection path: + +```bash +wss://preview.rt.speechmatics.com/v2/agent/ +``` + +Audio is processed in the EU or the US. You cannot pin a region during Preview, and agent STT is not available in the AUS region. + + ## Temporary keys @@ -151,6 +175,9 @@ For example, if you created a temporary key associated with a given `client_ref` curl -X GET "https://eu1.asr.api.speechmatics.com/v2/jobs/" \ -H "Authorization: Bearer $TEMP_KEY" ``` + +The same `type=rt` temporary key authenticates agent STT sessions. + @@ -184,7 +211,7 @@ Note that when starting a Realtime transcription session in the browser, you mus | Request Body Parameters | Required | Description | |--------------|-----------|----------------------------------------------------------- -| `ttl` | Yes | Integer: 60-86400. The temporary key's time to live in seconds. **Note** We suggest using the shortest TTL possible to minimise security risks. +| `ttl` | Yes | Integer: 60-86400. The temporary key's time to live in seconds. **Note** We suggest using the shortest TTL possible to minimize security risks. | `client_ref` | No | String. When provided, `batch` tokens can only create and retrieve jobs with that reference; without it, they can access any job. **Must** be set when temporary keys are exposed to end-users to prevent accessing another user's data. `client_ref` is ignored when requesting `rt` tokens. | `region` | No | **Deprecated.** String: `eu` (default), `usa`, or `au` (Batch transcription only). Region is determined by the [endpoint you call](#supported-endpoints). Still accepted for backward compatibility, but it has no effect. diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx deleted file mode 100644 index 355756f0..00000000 --- a/docs/get-started/quickstart.mdx +++ /dev/null @@ -1,78 +0,0 @@ ---- -pagination_prev: null -pagination_next: null -description: Take your first steps with the Speechmatics API. ---- -import { LinkCard } from "@site/src/theme/LinkCard"; -import DevIcon from '@site/src/components/devicon'; -import { Flex, Grid } from "@radix-ui/themes"; -import { ChevronsRightIcon, FileAudio, BotMessageSquare, BookOpen, Braces, Speech, GraduationCap } from "lucide-react"; -import Head from '@docusaurus/Head'; - - - - - -# Quickstart - -:::tip -The easiest way to try our products is via the [web portal](https://portal.speechmatics.com/). -::: - -## Explore our developer guides - - - } - href="/speech-to-text/realtime/quickstart" - /> - } - href="/speech-to-text/batch/quickstart" - /> - } - href="/voice-agents/overview" - /> - } - href="/text-to-speech/quickstart" - /> - - -## Quicklinks - - - } - href="/api-ref" - /> - } - href="https://github.com/speechmatics/speechmatics-js-sdk" - /> - } - href="https://github.com/speechmatics/speechmatics-python-sdk" - /> - } - href="https://github.com/speechmatics/speechmatics-academy" - /> - \ No newline at end of file diff --git a/docs/get-started/sidebar.ts b/docs/get-started/sidebar.ts index 817736f4..5f704995 100644 --- a/docs/get-started/sidebar.ts +++ b/docs/get-started/sidebar.ts @@ -7,12 +7,7 @@ export default { { type: "doc", id: "index", - label: "Introduction", - }, - { - type: "doc", - id: "get-started/quickstart", - label: "Quickstart", + label: "Welcome", }, { type: "doc", diff --git a/docs/index.mdx b/docs/index.mdx index f8ea1f02..8ca33ef3 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -9,48 +9,56 @@ pagination_next: null import { LinkCard } from "@site/src/theme/LinkCard"; import { ChevronsRightIcon, FileAudio, BotMessageSquare, Speech, Package } from "lucide-react"; -import { Flex, Grid } from "@radix-ui/themes"; +import { Grid } from "@radix-ui/themes"; # Welcome to Speechmatics -Speechmatics provides enterprise-grade APIs for Speech to Text, Text to Speech, and building voice agents. +Speechmatics provides APIs for Speech to Text and Text to Speech. Pick a starting point below. ## Developer quickstart - + + } + title="Transcribe a file" + description="Send an audio file and get a complete transcript back." + direction="column" + href="/speech-to-text/pre-recorded/quickstart" + /> } - title="Transcribe in real time" - description="Instantly convert streaming audio to text with Realtime processing" + title="Transcribe live audio" + description="Stream live audio and receive transcripts as you speak." direction="column" - href="/speech-to-text/realtime/quickstart" + href="/speech-to-text/streaming/quickstart" /> } - title="Transcribe a file" - description="Use Batch processing to accurately turn your audio files into text" + icon={} + title="Transcribe for an agent" + badgeText="Preview" + description="Get turn-based transcripts ready to pass to a language model." direction="column" - href="/speech-to-text/batch/quickstart" + href="/speech-to-text/agent-stt/quickstart" /> } title="Generate speech from text" - description="Use our Text to Speech API to generate speech from text" + description="Turn text into speech with four English voices." direction="column" href="/text-to-speech/quickstart" /> - + ## Build with Speechmatics } - title="Build a voice agent" - description="Use our integrations to build voice agents with ease" - href="/voice-agents/overview" + icon={} + title="Use an integration" + description="Connect through LiveKit, Pipecat, Vapi, or Zapier." + href="/integrations-and-sdks/" /> } @@ -64,23 +72,23 @@ Pick a starting point below. @@ -90,12 +98,17 @@ Pick a starting point below. + diff --git a/docs/integrations-and-sdks/index.mdx b/docs/integrations-and-sdks/index.mdx index c1a1ea5f..49122ab6 100644 --- a/docs/integrations-and-sdks/index.mdx +++ b/docs/integrations-and-sdks/index.mdx @@ -1,6 +1,6 @@ --- title: Integrations and SDKs — Overview -description: Discover which integrations and SDKs to add Speechmatics' STT, TTS or voice agents to your applications. +description: Discover which integrations and SDKs add Speechmatics Speech to Text and Text to Speech to your applications. --- import { LinkCard } from "@site/src/theme/LinkCard"; @@ -20,7 +20,7 @@ Use this page to quickly choose how you connect to Speechmatics: through popular ## Voice agent orchestrator integrations -Choose an integration to build accurate, low-latency voice agents rapidly with the Speechmatics API available in these frameworks. +Choose an integration to build voice agents on the Speechmatics API, available in these frameworks. } - href="https://github.com/speechmatics/speechmatics-python-sdk/tree/main/sdk/voice" + href="/integrations-and-sdks/voice-sdk/" /> } - quickstart="/voice-sdk" - href="https://github.com/speechmatics/speechmatics-python-sdk/tree/main/sdk/voice" + href="/integrations-and-sdks/voice-sdk/" /> } - quickstart="/rt-stt-sdk" href="https://github.com/speechmatics/speechmatics-python-sdk/tree/main/sdk/rt" /> } - quickstart="/python-batch-stt-sdk" href="https://github.com/speechmatics/speechmatics-python-sdk/tree/main/sdk/batch" /> } - quickstart="/js-rt-stt-sdk" href="https://github.com/speechmatics/speechmatics-js-sdk/tree/main/packages/real-time-client" /> } - quickstart="/js-batch-stt-sdk" href="https://github.com/speechmatics/speechmatics-js-sdk/tree/main/packages/batch-client" /> } - quickstart="/dotnet-rt-stt-sdk" href="https://github.com/speechmatics/speechmatics-dotnet/blob/main/README.md" /> } - quickstart="/rust-stt-sdk" href="https://github.com/speechmatics/speechmatics-rs/blob/main/README.md" /> @@ -90,7 +83,6 @@ Each SDK card provides simple installation steps and practical examples to get y title="TTS (Python)" description="Convert text to speech" icon={} - quickstart="/python-tts-sdk" href="https://github.com/speechmatics/speechmatics-python-sdk/tree/main/sdk/tts" /> diff --git a/docs/integrations-and-sdks/sidebar.ts b/docs/integrations-and-sdks/sidebar.ts index 24a0dd79..668a1447 100644 --- a/docs/integrations-and-sdks/sidebar.ts +++ b/docs/integrations-and-sdks/sidebar.ts @@ -24,6 +24,11 @@ export default { id: "integrations-and-sdks/zapier", label: "Zapier", }, + { + type: "doc", + id: "integrations-and-sdks/voice-sdk/index", + label: "Voice SDK", + }, { type: "doc", id: "integrations-and-sdks/sdks", diff --git a/docs/voice-agents/assets/additional-vocab.py b/docs/integrations-and-sdks/voice-sdk/assets/additional-vocab.py similarity index 100% rename from docs/voice-agents/assets/additional-vocab.py rename to docs/integrations-and-sdks/voice-sdk/assets/additional-vocab.py diff --git a/docs/voice-agents/assets/advanced-config.py b/docs/integrations-and-sdks/voice-sdk/assets/advanced-config.py similarity index 100% rename from docs/voice-agents/assets/advanced-config.py rename to docs/integrations-and-sdks/voice-sdk/assets/advanced-config.py diff --git a/docs/voice-agents/assets/basic-config.py b/docs/integrations-and-sdks/voice-sdk/assets/basic-config.py similarity index 100% rename from docs/voice-agents/assets/basic-config.py rename to docs/integrations-and-sdks/voice-sdk/assets/basic-config.py diff --git a/docs/voice-agents/assets/config-overlays.py b/docs/integrations-and-sdks/voice-sdk/assets/config-overlays.py similarity index 100% rename from docs/voice-agents/assets/config-overlays.py rename to docs/integrations-and-sdks/voice-sdk/assets/config-overlays.py diff --git a/docs/voice-agents/assets/config-serialization.py b/docs/integrations-and-sdks/voice-sdk/assets/config-serialization.py similarity index 100% rename from docs/voice-agents/assets/config-serialization.py rename to docs/integrations-and-sdks/voice-sdk/assets/config-serialization.py diff --git a/docs/voice-agents/assets/custom-config.py b/docs/integrations-and-sdks/voice-sdk/assets/custom-config.py similarity index 100% rename from docs/voice-agents/assets/custom-config.py rename to docs/integrations-and-sdks/voice-sdk/assets/custom-config.py diff --git a/docs/voice-agents/assets/event-subscription.py b/docs/integrations-and-sdks/voice-sdk/assets/event-subscription.py similarity index 100% rename from docs/voice-agents/assets/event-subscription.py rename to docs/integrations-and-sdks/voice-sdk/assets/event-subscription.py diff --git a/docs/voice-agents/assets/known-speakers.py b/docs/integrations-and-sdks/voice-sdk/assets/known-speakers.py similarity index 100% rename from docs/voice-agents/assets/known-speakers.py rename to docs/integrations-and-sdks/voice-sdk/assets/known-speakers.py diff --git a/docs/voice-agents/assets/quickstart.py b/docs/integrations-and-sdks/voice-sdk/assets/quickstart.py similarity index 88% rename from docs/voice-agents/assets/quickstart.py rename to docs/integrations-and-sdks/voice-sdk/assets/quickstart.py index 5b8aabbc..e0bd9adf 100644 --- a/docs/voice-agents/assets/quickstart.py +++ b/docs/integrations-and-sdks/voice-sdk/assets/quickstart.py @@ -4,7 +4,7 @@ from speechmatics.voice import VoiceAgentClient, AgentServerMessageType async def main(): - """Stream microphone audio to Speechmatics Voice Agent using 'scribe' preset""" + """Stream microphone audio to the Realtime API using 'scribe' preset""" # Audio configuration SAMPLE_RATE = 16000 # Hz @@ -17,7 +17,7 @@ async def main(): preset=PRESET ) - # Print finalised segments of speech with speaker ID + # Print finalized segments of speech with speaker ID @client.on(AgentServerMessageType.ADD_SEGMENT) def on_segment(message): for segment in message["segments"]: @@ -31,7 +31,7 @@ def on_segment(message): print("Error: Microphone not available") return - # Connect to the Voice Agent + # Connect to the Realtime API await client.connect() # Stream microphone audio (interruptible using keyboard) diff --git a/docs/voice-agents/assets/smart-turn.py b/docs/integrations-and-sdks/voice-sdk/assets/smart-turn.py similarity index 100% rename from docs/voice-agents/assets/smart-turn.py rename to docs/integrations-and-sdks/voice-sdk/assets/smart-turn.py diff --git a/docs/voice-agents/assets/speaker-focus-handler.py b/docs/integrations-and-sdks/voice-sdk/assets/speaker-focus-handler.py similarity index 100% rename from docs/voice-agents/assets/speaker-focus-handler.py rename to docs/integrations-and-sdks/voice-sdk/assets/speaker-focus-handler.py diff --git a/docs/voice-agents/assets/speaker-focus.py b/docs/integrations-and-sdks/voice-sdk/assets/speaker-focus.py similarity index 100% rename from docs/voice-agents/assets/speaker-focus.py rename to docs/integrations-and-sdks/voice-sdk/assets/speaker-focus.py diff --git a/docs/voice-agents/voice-sdk.mdx b/docs/integrations-and-sdks/voice-sdk/index.mdx similarity index 92% rename from docs/voice-agents/voice-sdk.mdx rename to docs/integrations-and-sdks/voice-sdk/index.mdx index f352b780..216fc6bc 100644 --- a/docs/voice-agents/voice-sdk.mdx +++ b/docs/integrations-and-sdks/voice-sdk/index.mdx @@ -1,5 +1,6 @@ --- -description: Learn how to use the Voice SDK. +title: Voice SDK +description: Use the Voice SDK to segment transcripts and detect turns over the Realtime API. --- import CodeBlock from '@theme/CodeBlock'; @@ -21,12 +22,14 @@ import pythonAdvancedConfig from "./assets/advanced-config.py?raw" # Voice SDK -## Overview +## What the Voice SDK provides The Voice SDK is a Python library that provides additional features optimized for conversational AI, built on top of our Realtime API. We use it to build our integrations, and it is also available for you to use. +The Voice SDK does its segmentation and turn detection SDK-side, over the [streaming](/speech-to-text/streaming/quickstart) interaction pattern. If you want turn detection to run server-side instead, with transcripts delivered as complete speaker turns, use [agent STT](/speech-to-text/agent-stt/). + - **Intelligent segmentation**: groups words into meaningful speech segments per speaker. - **Turn detection**: automatically detects when speakers finish talking. - **Speaker management**: focus on or ignore specific speakers in multi-speaker scenarios. @@ -90,7 +93,7 @@ pip install speechmatics-voice[smart] ### Quickstart -Here's how to stream microphone audio to the Voice Agent and transcribe finalized segments of speech, with speaker ID: +Here's how to stream microphone audio to the Realtime API and transcribe finalized segments of speech, with speaker ID: {pythonQuickstart} @@ -201,7 +204,7 @@ To change the language (or any other preset setting), use a custom configuration ### Custom configuration -For more control, you can also specify custom configurations or use presets as a starting point and customise with overlays: +For more control, you can also specify custom configurations or use presets as a starting point and customize with overlays: @@ -211,7 +214,7 @@ Specify configurations in a `VoiceAgentConfig` object: -Use presets as a starting point and customise with overlays: +Use presets as a starting point and customize with overlays: {pythonVoiceConfigOverlays} @@ -235,12 +238,12 @@ See [supported languages and locales](/speech-to-text/languages). ### Model selection `operating_point` (OperatingPoint, default: ENHANCED) -Select an accuracy level. -Options: STANDARD or ENHANCED. +Select the model. `STANDARD` selects the Standard model and `ENHANCED` selects the Enhanced model. +In the API, [`model`](/speech-to-text/models#specify-a-model) replaces `operating_point`; the SDK still takes `operating_point`. `domain` (str, default: None) -Domain-specific model (e.g., "finance", "medical"). -See the [Enhanced Medical model](/speech-to-text/models#healthcare-domain). +Tune recognition for a domain. `medical` is the only supported value. +See the [medical domain](/speech-to-text/features/medical-domain). ### Vocabulary @@ -267,7 +270,7 @@ Audio encoding format. `max_delay` (float, default: 1.0) Maximum transcription delay in seconds for word emission. -Turn detection ensures finalisation latency is not affected. +Turn detection ensures finalization latency is not affected. ### Basic diarization diff --git a/docs/integrations-and-sdks/zapier.mdx b/docs/integrations-and-sdks/zapier.mdx index 206a3efe..a1a2514c 100644 --- a/docs/integrations-and-sdks/zapier.mdx +++ b/docs/integrations-and-sdks/zapier.mdx @@ -84,8 +84,8 @@ Then configure your transcript output using these settings: | Setting | What it does | Learn more | |---|---|---| | **Language hints** | **Optional.** Speechmatics auto-detects the language by default. Use hints to narrow detection to specific languages — search the dropdown by name, or type a language code. | [Languages](/speech-to-text/languages) | -| **Speaker diarization** | Separates the transcript by speaker. | [Speaker diarization](/speech-to-text/features/diarization) | -| **Output format** | Controls the shape of the returned transcript. | [Output formats](/speech-to-text/batch/output) | +| **Speaker diarization** | Separates the transcript by speaker. | [Speaker diarization](/speech-to-text/features/speaker-diarization) | +| **Output format** | Controls the shape of the returned transcript. | [Output formats](/speech-to-text/pre-recorded/output) | Once you've configured your settings, click the **Test** tab, then click **Test Step**. Zapier validates your configuration and runs it. @@ -112,6 +112,6 @@ Still stuck? See [Zapier support](https://help.zapier.com/hc/en-us) or [contact ## Next steps - [Explore all supported languages](/speech-to-text/languages) -- [Configure speaker diarization](/speech-to-text/features/diarization) -- [Choose an output format](/speech-to-text/batch/output) -- [Learn what the Batch API supports](/speech-to-text/batch/quickstart) +- [Configure speaker diarization](/speech-to-text/features/speaker-diarization) +- [Choose an output format](/speech-to-text/pre-recorded/output) +- [Learn what the Batch API supports](/speech-to-text/pre-recorded/quickstart) diff --git a/docs/private/next-gen-model.mdx b/docs/private/next-gen-model.mdx index 31c9e8ac..f7a9cfc8 100644 --- a/docs/private/next-gen-model.mdx +++ b/docs/private/next-gen-model.mdx @@ -63,7 +63,7 @@ Example config with language hints: } ``` -- You can specify any number of Speechmatics [supported language ISO codes](../speech-to-text/languages#transcription-languages) +- You can specify any number of Speechmatics [supported language ISO codes](../speech-to-text/languages/transcription) - For monolingual audio files, specifying the single language present can also improve WER by around 2% relative ### Language labelling @@ -97,7 +97,7 @@ Two new properties have been introduced into the returned job metadata based on | ------------- | --------------- |----- | | Modes | Batch only | Q3: Realtime mode available | | Accuracy - Monolingual audio | Matching Standard model on general conversation | Q3: Improved recognition of domain specific terms and proper nouns | -| Accuracy - Multilingual audio | Multilingual files fully supported across all [supported languages](../speech-to-text/languages#transcription-languages) | Q2: Improved code-switching accuracy | +| Accuracy - Multilingual audio | Multilingual files fully supported across all [supported languages](../speech-to-text/languages/transcription) | Q2: Improved code-switching accuracy | | Accuracy - Alphanumerics | ✅ Matching Enhanced model accuracy, though formatting consistency not yet implemented | Q3: Further improved accuracy and formatting consistency | | Additional context | Custom dictionary not supported | Q3: Support for additional context, similar to custom dictionary | | Language labelling | Language labelling at the granularity of around 14 seconds | Q2: Word level language labelling | diff --git a/docs/private/voice-agent-api.mdx b/docs/private/voice-agent-api.mdx deleted file mode 100644 index 2926052c..00000000 --- a/docs/private/voice-agent-api.mdx +++ /dev/null @@ -1,726 +0,0 @@ ---- -description: Early access to the Voice Agent API — a turn-based API built for voice agents ---- - - -# Voice Agent API - -:::warning -- The Voice Agent API is a preview offering and should **not be used for live production traffic**. The system will be less stable than our production endpoints and features may change. -- There are no uptime or performance SLAs. -- There are no data residency guarantees. Data processing may occur in both US and EU regions. -- Preview features may be cancelled at any time or never be released publicly. -::: - -## Introduction - -The Voice Agent API is a WebSocket API for building voice agents. Stream audio in and receive speaker-labelled, turn-based transcription back — clean, punctuated, and ready to pass directly to an LLM. - -Turn detection runs server-side. Choose a [profile](#profiles) based on your use case and the API handles when to finalise each speaker's turn. - -**Looking for code examples?** See working examples in [Speechmatics Academy](https://github.com/speechmatics/speechmatics-academy/tree/main/basics/11-voice-api-explorer) for Python and JavaScript. - ---- - -## Profiles - -Profiles are pre-configured turn detection modes. Each profile sets the right defaults for your use case — you choose one when connecting, include it in your endpoint URL, and the server handles the rest. - -| Profile | Turn detection | Best for | -|---------|---------------|----------| -| `adaptive` | Adapts to speaker pace and hesitation | General conversational agents | -| `agile` | VAD-based silence detection | Speed-first use cases | -| `smart` | `adaptive` + ML acoustic turn prediction | High-stakes conversations | -| `external` | Manual — you trigger turn end | Push-to-talk, custom VAD, LLM-driven | - -### `adaptive` - -**Endpoint:** `/v2/agent/adaptive` - -Adapts to each speaker's pace over the course of a conversation. It adjusts the turn-end threshold based on speech rate and disfluencies (e.g. hesitations, filler words), waiting longer for speakers who tend to pause mid-thought. - -**Best for:** General conversational voice agents. - -**Languages:** All supported languages. Disfluency detection is available for the [supported languages](https://docs.speechmatics.com/speech-to-text/formatting#supported-languages-for-disfluencies) — other languages fall back to speech-rate adaptation. - -**Trade-off:** Latency varies by speaker. - -### `agile` - -**Endpoint:** `/v2/agent/agile` - -Uses voice activity detection (VAD) to detect silence and finalise turns as quickly as possible. The lowest latency profile. - -**Best for:** Use cases where response speed is the top priority and occasional mid-speech finalisations are acceptable. - -**Languages:** All supported languages. - -**Trade-off:** Because it relies on silence, it may finalise a turn while the speaker is still mid-sentence — for example, during a natural pause. This can result in additional downstream LLM calls. - -### `smart` - -**Endpoint:** `/v2/agent/smart` - -Builds on `adaptive` with an additional ML model that analyses acoustic cues to predict whether a speaker has genuinely finished their turn. The most conservative profile — least likely to interrupt. - -**Best for:** High-stakes conversations where cutting off the user is costly — finance, healthcare, legal. - -**Languages:** Arabic, Bengali, Chinese, Danish, Dutch, English, Finnish, French, German, Hindi, Indonesian, Italian, Japanese, Korean, Marathi, Norwegian, Polish, Portuguese, Russian, Spanish, Turkish, Ukrainian, Vietnamese. - -**Trade-off:** Higher latency than `adaptive`. - -### `external` - -**Endpoint:** `/v2/agent/external` - -Turn detection is fully manual. The server accumulates audio and transcript until you send a `ForceEndOfUtterance` message, at which point it finalises everything spoken up to that point and emits an `AddSegment`. - -**Best for:** Push-to-talk interfaces, custom VAD pipelines, or setups where an LLM decides when to respond. - -**Languages:** All supported languages. - -**Trade-off:** You are responsible for all turn detection logic. - ---- - -## Session Flow - -Every session follows the same structure: connect, start recognition, stream audio, receive turn events, close. - -```mermaid -sequenceDiagram - participant C as Client - participant S as Server - - C->>S: Connect to endpoint with profile via WebSocket - C->>S: StartRecognition - S-->>C: RecognitionStarted - - loop Audio Stream - C->>S: Audio frames (binary) - S-->>C: AudioAdded - S-->>C: SpeechStarted - S-->>C: StartOfTurn - S-->>C: SpeakerStarted - S-->>C: AddPartialSegment (repeating) - S-->>C: SpeakerMetrics (repeating) - S-->>C: EndOfTurnPrediction (adaptive, smart) - S-->>C: SmartTurnResult (smart only) - S-->>C: SpeechEnded - S-->>C: EndOfUtterance - S-->>C: SpeakerEnded - S-->>C: AddSegment - S-->>C: EndOfTurn - - opt Mid-session controls - C->>S: ForceEndOfUtterance (external only) - C->>S: UpdateSpeakerFocus - C->>S: GetSpeakers - S-->>C: SpeakersResult - end - end - - C->>S: EndOfStream - S-->>C: EndOfTranscript -``` - -`SessionMetrics` is emitted every 5 seconds independently of turn boundaries. - -For a full reference of all messages, see [Messages Overview](#messages-overview). - ---- - -## Getting Started - -### 1. Connect - -Open a WebSocket connection to the preview endpoint. To do this, you must specify the [profile](#profiles) to use: - -``` -wss://preview.rt.speechmatics.com/v2/agent/ -``` - -### 2. Authenticate - -Authenticate every connection using one of the following: - -| Method | Format | -|--------|--------| -| Header (API key) | `Authorization: Bearer ` | -| Header (JWT) | `Authorization: Bearer ` | -| Query parameter (API key) | `?api_key=` | -| Query parameter (JWT) | `?jwt=` | - -See [Authentication](/get-started/authentication) for details including temporary keys. - -### 3. Start the session - -Send [`StartRecognition`](#startrecognition) as your first message: - -```json -{ - "message": "StartRecognition", - "transcription_config": { - "language": "en" - } -} -``` -For all configuration options, see [Configuration](#configuration). - -The server responds with `RecognitionStarted` when the session is ready. You should wait for this message before sending audio. - - -### 4. Stream audio and handle responses - -Send audio as binary WebSocket frames. Turn events will arrive in real time as the API processes speech — see [Session Flow](#session-flow) for the full message sequence. - ---- - -## Configuration - -Configuration is passed in [`StartRecognition`](#startrecognition) and is split across two levels of the payload: `audio_format` (top-level) and `transcription_config`. - -**`audio_format`** - -:::warning -Only `pcm_s16le` at `8000` or `16000` Hz is supported. Other encodings (e.g. `pcm_f32le`, `mulaw`) and sample rates (e.g. `44100`) may be silently accepted by the API but will not produce correct output. -::: - -| Field | Notes | -|-------|-------| -| `type` | Must be `raw` | -| `encoding` | Must be `pcm_s16le` (16-bit signed little-endian PCM) | -| `sample_rate` | Must be `8000` or `16000` | - -Example: `{"type":"raw","encoding":"pcm_s16le","sample_rate":16000}` - -**`transcription_config`** - -| Field | Default | Notes | -|-------|---------|-------| -| `language` | `en` | All supported languages | -| `output_locale` | — | Output locale (e.g. `en-US`) | -| `additional_vocab` | — | Custom vocabulary entries | -| `punctuation_overrides` | — | Custom punctuation rules | -| `domain` | — | Domain-specific model (e.g. `medical`) | -| `enable_entities` | `false` | Entity detection | -| `enable_partials` | `true` | Emit partial segments during speech | -| `diarization` | `speaker` | Speaker diarization; `none` to disable | -| `volume_threshold` | — | Minimum audio volume to process | - -**`transcription_config.speaker_diarization_config`** - -Note: The following require `diarization: speaker` to be set. -| Field | Default | Notes | -|-------|---------|-------| -| `max_speakers` | — | Maximum number of speakers to track | -| `speaker_sensitivity` | — | Sensitivity of speaker separation | -| `prefer_current_speaker` | — | Bias toward the most recently active speaker | -| `known_speakers` | — | Pre-enrolled speaker identifiers for cross-session recognition (see [Speaker ID](#speaker-id)) | - -**Not supported — will be rejected if present** - -| Field | Notes | -|-------|-------| -| `translation_config` | Not supported on this endpoint | -| `audio_events_config` | Not supported on this endpoint | - ---- - -## Messages Overview - -All messages exchanged during a Voice Agent API session. For payload details, see the API Reference sections. - -### Client → Server - -| Message | When to send | -|---------|-------------| -| [`StartRecognition`](#startrecognition) | First message after connecting. Starts the session and passes configuration. | -| Audio frames | Binary WebSocket frames containing raw PCM audio, sent continuously. | -| [`ForceEndOfUtterance`](#forceendofutterance) | `external` profile only. Triggers immediate turn finalisation. | -| [`UpdateSpeakerFocus`](#updatespeakerfocus) | Any time during the session. Changes which speakers are in focus. | -| [`GetSpeakers`](#getspeakers) | Any time during the session. Requests voice identifiers for diarized speakers. | -| [`EndOfStream`](#endofstream) | When there is no more audio to send. | - -### Server → Client - -**Core turn events** — the messages your agent logic acts on - -| Message | Profile | When it's emitted | -|---------|---------|------------------| -| [`StartOfTurn`](#startofturn) | All | A speaker begins a new turn | -| [`AddPartialSegment`](#addpartialsegment) | All | Interim transcript update; each replaces the previous | -| [`AddSegment`](#addsegment) | All | Final transcript for the turn — pass this to your LLM | -| [`EndOfTurn`](#endofturn) | All | Turn complete; your agent can now respond | - -**Turn prediction** — early signals you can use to prepare a response - -| Message | Profile | When it's emitted | -|---------|---------|------------------| -| [`EndOfTurnPrediction`](#endofturnprediction) | `adaptive`, `smart` | The model predicts the current turn will end soon | -| [`SmartTurnResult`](#smartturnresult) | `smart` only | High-confidence acoustic prediction of turn completion | - -**Speech and speaker activity** - -| Message | Profile | When it's emitted | -|---------|---------|------------------| -| [`SpeechStarted`](#speechstarted--speechended) | All | Voice activity detected in the audio stream | -| [`SpeechEnded`](#speechstarted--speechended) | All | Voice activity stopped | -| [`SpeakerStarted`](#speakerstarted--speakerended) | All | A specific diarized speaker began talking | -| [`SpeakerEnded`](#speakerstarted--speakerended) | All | A specific diarized speaker stopped talking | -| [`SpeakersResult`](#speakersresult) | All | Response to `GetSpeakers` | - -**Session lifecycle** - -| Message | When it's emitted | -|---------|------------------| -| `RecognitionStarted` | Session ready; emitted in response to `StartRecognition` | -| `AudioAdded` | Audio frame acknowledged | -| `EndOfTranscript` | Session closing; emitted by the proxy after `EndOfStream` | - -**Metrics and diagnostics** - -| Message | When it's emitted | -|---------|------------------| -| [`SessionMetrics`](#sessionmetrics) | Session stats; emitted every 5 seconds and at session end | -| [`SpeakerMetrics`](#speakermetrics) | Per-speaker word count and volume; emitted on each recognised word | - -**Shared messages with the RT API** - messages shared with the RT API. See the [RT API Reference](/api-ref) for full payload details. - -| Message | When it's emitted | -|---------|------------------| -| `EndOfUtterance` | Silence threshold reached; precedes turn finalisation | -| `Info` | Non-critical informational message | -| `Warning` | Non-fatal issue (e.g. unsupported config field ignored) | -| `Error` | Fatal error; connection will close | - ---- - -## API Reference - Client Messages - -#### StartRecognition - -The first message you send after connecting. Starts the recognition session and passes configuration. -The server responds with `RecognitionStarted`. - -```json -{ - "message": "StartRecognition", - "audio_format": { - "type": "raw", - "encoding": "pcm_s16le", - "sample_rate": 16000 - }, - "transcription_config": { - "language": "en" - } -} -``` - -For all configuration options, see [Configuration](#configuration). - -#### EndOfStream - -Send when you have finished streaming audio. The server finalises any remaining transcript and then emits `EndOfTranscript`. - -`last_seq_no` is the sequence number of the last audio frame you sent. -```json -{ - "message": "EndOfStream", - "last_seq_no": 1234 -} -``` - -#### ForceEndOfUtterance - -Only applies to the `external` profile. Immediately ends the current turn — the server finalises all audio received so far and emits a single `AddSegment` containing the complete transcript for that turn, followed by `EndOfTurn`. - -Use this wherever your application decides a turn is complete: on button release (push-to-talk), on VAD silence, or on an LLM signal. - -```json -{ - "message": "ForceEndOfUtterance" -} -``` - -#### UpdateSpeakerFocus - -Updates which speakers are in focus, mid-session. Takes effect immediately. See [Speaker Focus](#speaker-focus) for full details. - -```json -{ - "message": "UpdateSpeakerFocus", - "speaker_focus": { - "focus_speakers": ["S1"], - "ignore_speakers": [], - "focus_mode": "retain" - } -} -``` - -#### GetSpeakers - -Requests voice identifiers for all speakers diarized so far in the session. The server responds with a `SpeakersResult` message. See [Speaker ID](#speaker-id) for full details. - -```json -{ - "message": "GetSpeakers" -} -``` - ---- - -## API Reference - Server Messages - -This section covers Voice Agent API-specific messages only. For shared messages (`RecognitionStarted`, `AudioAdded`, `AddPartialTranscript`, `AddTranscript`, `EndOfUtterance`, `EndOfTranscript`, `Info`, `Warning`, `Error`), see the [RT API reference](/api-ref). - -#### StartOfTurn - -Emitted when a speaker begins a new turn. Use this to signal to your agent that it should stop speaking if it currently is. - -```json -{ - "message": "StartOfTurn", - "turn_id": 42 -} -``` - -**Fields:** -- `turn_id` — monotonically increasing integer; pairs with the corresponding `EndOfTurn` - -#### EndOfTurn - -Emitted when turn detection decides the speaker has finished. This is the trigger for your agent to respond. The finalised transcript for the turn is in the preceding `AddSegment`. - -```json -{ - "message": "EndOfTurn", - "turn_id": 42, - "metadata": { - "start_time": 0.84, - "end_time": 3.24 - } -} -``` - -**Fields:** -- `turn_id` — matches the `StartOfTurn` for this turn -- `metadata.start_time` / `metadata.end_time` — audio time range for the turn, in seconds from session start - -#### AddPartialSegment - -Interim transcript update emitted continuously while the speaker is talking. Each new `AddPartialSegment` replaces the previous one — do not concatenate them. - -```json -{ - "message": "AddPartialSegment", - "segments": [ - { - "speaker_id": "S1", - "is_active": true, - "timestamp": "2025-01-01T12:00:00.000+00:00", - "language": "en", - "text": "Good evening", - "is_eou": false, - "metadata": { - "start_time": 0.84, - "end_time": 1.24 - } - } - ], - "metadata": { - "start_time": 0.84, - "end_time": 1.24, - "processing_time": 0.23 - } -} -``` - -#### AddSegment - -The final, complete transcript for a turn. Emitted just before `EndOfTurn`. This is the stable output to pass to your LLM — do not use `AddPartialSegment` for this. - -In multi-speaker scenarios, a single `AddSegment` may contain segments from multiple speakers, returned in time order. - -```json -{ - "message": "AddSegment", - "segments": [ - { - "speaker_id": "S1", - "is_active": true, - "timestamp": "2025-01-01T12:00:00.000+00:00", - "language": "en", - "text": "Good evening.", - "is_eou": true, - "metadata": { - "start_time": 0.84, - "end_time": 1.56 - } - } - ], - "metadata": { - "start_time": 0.84, - "end_time": 1.56, - "processing_time": 0.25 - } -} -``` - -**Segment fields:** -- `speaker_id` — speaker label (e.g. `S1`, `S2`, or a custom label if using [Speaker ID](#speaker-id)) -- `is_active` — `true` if this speaker is in your current focus list; `false` if they are a background speaker (see [Speaker Focus](#speaker-focus)) -- `is_eou` — `true` on final segments, `false` on partials -- `text` — clean, punctuated transcript text -- `metadata.start_time` / `metadata.end_time` — time range of this segment in seconds from session start - -**Message-level fields:** -- `metadata.processing_time` — transcription latency in seconds for this message - -#### SpeakerStarted / SpeakerEnded - -Emitted when a specific speaker starts or stops being heard. These are voice activity events — they fire based on detected speech, independently of turn boundaries. - -```json -{ - "message": "SpeakerStarted", - "speaker_id": "S1", - "is_active": true, - "time": 0.84, - "metadata": { "start_time": 0.84, "end_time": 0.84 } -} -``` - -```json -{ - "message": "SpeakerEnded", - "speaker_id": "S1", - "is_active": true, - "time": 3.24, - "metadata": { "start_time": 0.84, "end_time": 3.24 } -} -``` - -**Fields:** -- `speaker_id` — the speaker whose activity changed -- `is_active` — whether this speaker is in your current focus list -- `time` — seconds from session start when the activity was detected -- `metadata.start_time` — when this speaker started their current speaking interval -- `metadata.end_time` — when this speaker stopped speaking (`SpeakerEnded` only) - -#### SessionMetrics - -Emitted every 5 seconds and once at the end of the session. - -```json -{ - "message": "SessionMetrics", - "total_time": 4.6, - "total_time_str": "00:00:04", - "total_bytes": 148480, - "processing_time": 0.295 -} -``` - -#### SpeakerMetrics - -Emitted each time a speaker produces a recognised word. - -```json -{ - "message": "SpeakerMetrics", - "speakers": [ - { - "speaker_id": "S1", - "word_count": 6, - "last_heard": 2.36, - "volume": 5.2 - } - ] -} -``` - -#### SpeakersResult - -Emitted in response to `GetSpeakers`. Contains voice identifiers for all diarized speakers so far. See [Speaker ID](#speaker-id) for how to store and use these. - -```json -{ - "message": "SpeakersResult", - "speakers": [ - { "label": "S1", "speaker_identifiers": [""] }, - { "label": "S2", "speaker_identifiers": [""] } - ] -} -``` - -#### EndOfTurnPrediction - -Emitted by `adaptive` and `smart` profiles when the model predicts the current turn is about to end. Can be used to begin preparing a response before `EndOfTurn` arrives, reducing perceived latency. - -```json -{ - "message": "EndOfTurnPrediction", - "turn_id": 2, - "predicted_wait": 0.73, - "metadata": { - "ttl": 0.73, - "reasons": ["not__ends_with_eos"] - } -} -``` - -**Fields:** -- `turn_id` — the turn this prediction applies to -- `predicted_wait` — estimated seconds until the turn ends -- `metadata.ttl` — time to live; how long this prediction remains valid -- `metadata.reasons` — internal signals that contributed to the prediction - -#### SmartTurnResult - -:::warning -This message is currently emitted as `SmartTurnResult` during preview. It will be renamed to `SmartTurnPrediction` at GA. -::: - -Emitted by the `smart` profile only. A higher-confidence acoustic prediction of turn completion, based on the ML model that analyses vocal cues. - -```json -{ - "message": "SmartTurnResult", - "prediction": { - "prediction": true, - "probability": 0.979, - "processing_time": 0.128 - }, - "metadata": { - "start_time": 0.0, - "end_time": 2.2, - "language": "en", - "speaker_id": "S1", - "total_time": 2.2 - } -} -``` - -**Fields:** -- `prediction.prediction` — `true` if the model predicts the turn is complete -- `prediction.probability` — confidence score (0–1) -- `prediction.processing_time` — time taken by the ML model in seconds -- `metadata.start_time` / `metadata.end_time` — audio window analysed -- `metadata.total_time` — total session time at point of prediction -- `metadata.speaker_id` — speaker being analysed (`null` if not yet identified) - -#### SpeechStarted / SpeechEnded - -Voice activity detection events. Emitted when speech is first detected in the audio stream (`SpeechStarted`) or stops (`SpeechEnded`). These fire independently of speaker identity and turn boundaries. - -```json -{ - "message": "SpeechStarted", - "probability": 0.508, - "transition_duration_ms": 192.0, - "metadata": { - "start_time": 2.1, - "end_time": 2.1 - } -} -``` - -```json -{ - "message": "SpeechEnded", - "probability": 0.307, - "transition_duration_ms": 192.0, - "metadata": { - "start_time": 0.4, - "end_time": 2.5 - } -} -``` - -**Fields:** -- `probability` — VAD confidence score (0–1) -- `transition_duration_ms` — duration of the speech/silence transition in milliseconds -- `metadata.start_time` — when speech began (`SpeechStarted`: same as `end_time`; `SpeechEnded`: when the speaking interval started) -- `metadata.end_time` — when the event was detected - ---- - -## Features - -### Speaker Focus - -Speaker focus lets you control which speakers' output your agent acts on. By default, all detected speakers are active and their transcripts are included in `AddSegment` output. - -Speaker IDs (`S1`, `S2`, etc.) are assigned automatically when diarization is enabled, and persist for the lifetime of the session. Send `UpdateSpeakerFocus` at any point during the session to change who is in focus — the new config takes effect immediately and replaces the previous one. - -```json -{ - "message": "UpdateSpeakerFocus", - "speaker_focus": { - "focus_speakers": ["S1"], - "ignore_speakers": ["S3"], - "focus_mode": "retain" - } -} -``` - -**Fields:** - -- `focus_speakers` — speaker IDs to treat as active. Their segments appear with `is_active: true`. -- `ignore_speakers` — speaker IDs to exclude entirely. Their speech is dropped and does not affect turn detection. -- `focus_mode` — what happens to speakers who are neither in `focus_speakers` nor `ignore_speakers`: - - `retain` — they remain in the output as passive speakers (`is_active: false`) - - `ignore` — they are excluded from the output entirely - -### Speaker ID - -Speaker ID lets you recognise the same person across separate sessions. At the end of a session, you can retrieve voice identifiers for each speaker and store them. In future sessions, pass those identifiers into `StartRecognition` and the system will tag matching speakers with a consistent label rather than a generic `S1`, `S2`. - -#### Getting identifiers - -Send [`GetSpeakers`](#getspeakers) at any point during a session to retrieve identifiers for all diarized speakers so far. The server responds with a [`SpeakersResult`](#speakersresult) message. - -Store the `speaker_identifiers` values from the response. These are opaque tokens tied to a speaker's voice profile — treat them as credentials and store them securely. - -#### Using identifiers in future sessions - -Pass stored identifiers into `StartRecognition` via `transcription_config.known_speakers`. You can assign any label: - -```json -{ - "message": "StartRecognition", - "transcription_config": { - "language": "en", - "known_speakers": [ - { "label": "Alice", "speaker_identifiers": [""] }, - { "label": "Bob", "speaker_identifiers": [""] } - ] - } -} -``` - -When those speakers are detected, their segments will carry `"Alice"` or `"Bob"` as the `speaker_id` instead of generic labels. Any unrecognised speakers are still assigned generic labels (`S1`, `S2`, etc.). - ---- - -## Code Examples - -For working code examples in Python and JavaScript, see the [Speechmatics Academy](https://github.com/speechmatics/speechmatics-academy/tree/main/basics/11-voice-api-explorer). - ---- - -## Feedback - -This is a preview and your feedback shapes what goes to GA (General Availability). -We'd love to hear from you — Tell us what works well, which features you use, whether something didn't work as expected, a profile that behaved differently than you anticipated, or a feature you'd want before we ship this more broadly. - -Specific areas of interest: - -- Integration experience (documentation, SDKs, API messages/metadata) -- Accuracy and latency (including data capture if it's relevant. E.g. phone numbers, spell outs of names/account numbers) -- Turn detection and experience with different profiles -- Any missing capabilities which would make your product better -- What would stop you using this in production - -We'd love to get on a call with you to discuss your feedback in person, or you can [fill in this form](https://docs.google.com/forms/d/e/1FAIpQLSc-6GQXYx_0M-X0Uu_uB_4XyDL009jMv3hBJAFw7kD98AILJg/viewform). You can also reach us via your Speechmatics contact or the channel shared in your preview welcome email. diff --git a/docs/speech-to-text/batch/speech-intelligence/assets/auto-chapters.py b/docs/speech-to-text/add-ons/assets/auto-chapters.py similarity index 96% rename from docs/speech-to-text/batch/speech-intelligence/assets/auto-chapters.py rename to docs/speech-to-text/add-ons/assets/auto-chapters.py index 84a18d4c..4b76bc5b 100644 --- a/docs/speech-to-text/batch/speech-intelligence/assets/auto-chapters.py +++ b/docs/speech-to-text/add-ons/assets/auto-chapters.py @@ -30,7 +30,7 @@ print(f"job {job_id} submitted successfully, waiting for transcript") # Note that in production, you should set up notifications instead of polling. - # Notifications are described here: https://docs.speechmatics.com/speech-to-text/batch/notifications + # Notifications are described here: https://docs.speechmatics.com/speech-to-text/pre-recorded/notifications transcript = client.wait_for_completion(job_id, transcription_format="json-v2") # highlight-start chapters = transcript["chapters"] diff --git a/docs/speech-to-text/batch/speech-intelligence/assets/interview-informative-brief-bullets.txt b/docs/speech-to-text/add-ons/assets/interview-informative-brief-bullets.txt similarity index 100% rename from docs/speech-to-text/batch/speech-intelligence/assets/interview-informative-brief-bullets.txt rename to docs/speech-to-text/add-ons/assets/interview-informative-brief-bullets.txt diff --git a/docs/speech-to-text/batch/speech-intelligence/assets/interview-informative-brief-paragraphs.txt b/docs/speech-to-text/add-ons/assets/interview-informative-brief-paragraphs.txt similarity index 100% rename from docs/speech-to-text/batch/speech-intelligence/assets/interview-informative-brief-paragraphs.txt rename to docs/speech-to-text/add-ons/assets/interview-informative-brief-paragraphs.txt diff --git a/docs/speech-to-text/batch/speech-intelligence/assets/interview-informative-detailed-bullets.txt b/docs/speech-to-text/add-ons/assets/interview-informative-detailed-bullets.txt similarity index 100% rename from docs/speech-to-text/batch/speech-intelligence/assets/interview-informative-detailed-bullets.txt rename to docs/speech-to-text/add-ons/assets/interview-informative-detailed-bullets.txt diff --git a/docs/speech-to-text/batch/speech-intelligence/assets/interview-informative-detailed-paragraphs.txt b/docs/speech-to-text/add-ons/assets/interview-informative-detailed-paragraphs.txt similarity index 100% rename from docs/speech-to-text/batch/speech-intelligence/assets/interview-informative-detailed-paragraphs.txt rename to docs/speech-to-text/add-ons/assets/interview-informative-detailed-paragraphs.txt diff --git a/docs/speech-to-text/batch/speech-intelligence/assets/sentiment.py b/docs/speech-to-text/add-ons/assets/sentiment.py similarity index 97% rename from docs/speech-to-text/batch/speech-intelligence/assets/sentiment.py rename to docs/speech-to-text/add-ons/assets/sentiment.py index 8ee99e93..65526c85 100644 --- a/docs/speech-to-text/batch/speech-intelligence/assets/sentiment.py +++ b/docs/speech-to-text/add-ons/assets/sentiment.py @@ -30,7 +30,7 @@ print(f"job {job_id} submitted successfully, waiting for transcript") # Note that in production, you should set up notifications instead of polling. - # Notifications are described here: https://docs.speechmatics.com/speech-to-text/batch/notifications + # Notifications are described here: https://docs.speechmatics.com/speech-to-text/pre-recorded/notifications transcript = client.wait_for_completion(job_id, transcription_format="json-v2") # highlight-start sentiment = transcript["sentiment_analysis"] diff --git a/docs/speech-to-text/batch/speech-intelligence/assets/summarization.py b/docs/speech-to-text/add-ons/assets/summarization.py similarity index 96% rename from docs/speech-to-text/batch/speech-intelligence/assets/summarization.py rename to docs/speech-to-text/add-ons/assets/summarization.py index 3eab0c1e..01ce9e99 100644 --- a/docs/speech-to-text/batch/speech-intelligence/assets/summarization.py +++ b/docs/speech-to-text/add-ons/assets/summarization.py @@ -30,7 +30,7 @@ print(f"job {job_id} submitted successfully, waiting for transcript") # Note that in production, you should set up notifications instead of polling. - # Notifications are described here: https://docs.speechmatics.com/speech-to-text/batch/notifications + # Notifications are described here: https://docs.speechmatics.com/speech-to-text/pre-recorded/notifications transcript = client.wait_for_completion(job_id, transcription_format="json-v2") # highlight-start summary = transcript["summary"]["content"] diff --git a/docs/speech-to-text/batch/speech-intelligence/assets/tech-support-call-conversational-brief-bullets.txt b/docs/speech-to-text/add-ons/assets/tech-support-call-conversational-brief-bullets.txt similarity index 100% rename from docs/speech-to-text/batch/speech-intelligence/assets/tech-support-call-conversational-brief-bullets.txt rename to docs/speech-to-text/add-ons/assets/tech-support-call-conversational-brief-bullets.txt diff --git a/docs/speech-to-text/batch/speech-intelligence/assets/tech-support-call-conversational-brief-paragraphs.txt b/docs/speech-to-text/add-ons/assets/tech-support-call-conversational-brief-paragraphs.txt similarity index 100% rename from docs/speech-to-text/batch/speech-intelligence/assets/tech-support-call-conversational-brief-paragraphs.txt rename to docs/speech-to-text/add-ons/assets/tech-support-call-conversational-brief-paragraphs.txt diff --git a/docs/speech-to-text/batch/speech-intelligence/assets/tech-support-call-conversational-detailed-bullets.txt b/docs/speech-to-text/add-ons/assets/tech-support-call-conversational-detailed-bullets.txt similarity index 100% rename from docs/speech-to-text/batch/speech-intelligence/assets/tech-support-call-conversational-detailed-bullets.txt rename to docs/speech-to-text/add-ons/assets/tech-support-call-conversational-detailed-bullets.txt diff --git a/docs/speech-to-text/batch/speech-intelligence/assets/tech-support-call-conversational-detailed-paragraphs.txt b/docs/speech-to-text/add-ons/assets/tech-support-call-conversational-detailed-paragraphs.txt similarity index 100% rename from docs/speech-to-text/batch/speech-intelligence/assets/tech-support-call-conversational-detailed-paragraphs.txt rename to docs/speech-to-text/add-ons/assets/tech-support-call-conversational-detailed-paragraphs.txt diff --git a/docs/speech-to-text/batch/speech-intelligence/assets/topics.py b/docs/speech-to-text/add-ons/assets/topics.py similarity index 97% rename from docs/speech-to-text/batch/speech-intelligence/assets/topics.py rename to docs/speech-to-text/add-ons/assets/topics.py index 909f8356..08fa3429 100644 --- a/docs/speech-to-text/batch/speech-intelligence/assets/topics.py +++ b/docs/speech-to-text/add-ons/assets/topics.py @@ -31,7 +31,7 @@ print(f"job {job_id} submitted successfully, waiting for transcript") # Note that in production, you should set up notifications instead of polling. - # Notifications are described here: https://docs.speechmatics.com/batch-transcription/notifications + # Notifications are described here: https://docs.speechmatics.com/speech-to-text/pre-recorded/notifications transcript = client.wait_for_completion(job_id, transcription_format="json-v2") # highlight-start topics_detected = transcript["topics"] diff --git a/docs/speech-to-text/batch/speech-intelligence/assets/webinar-informative-brief-bullets.txt b/docs/speech-to-text/add-ons/assets/webinar-informative-brief-bullets.txt similarity index 100% rename from docs/speech-to-text/batch/speech-intelligence/assets/webinar-informative-brief-bullets.txt rename to docs/speech-to-text/add-ons/assets/webinar-informative-brief-bullets.txt diff --git a/docs/speech-to-text/batch/speech-intelligence/assets/webinar-informative-brief-paragraphs.txt b/docs/speech-to-text/add-ons/assets/webinar-informative-brief-paragraphs.txt similarity index 100% rename from docs/speech-to-text/batch/speech-intelligence/assets/webinar-informative-brief-paragraphs.txt rename to docs/speech-to-text/add-ons/assets/webinar-informative-brief-paragraphs.txt diff --git a/docs/speech-to-text/batch/speech-intelligence/assets/webinar-informative-detailed-bullets.txt b/docs/speech-to-text/add-ons/assets/webinar-informative-detailed-bullets.txt similarity index 100% rename from docs/speech-to-text/batch/speech-intelligence/assets/webinar-informative-detailed-bullets.txt rename to docs/speech-to-text/add-ons/assets/webinar-informative-detailed-bullets.txt diff --git a/docs/speech-to-text/batch/speech-intelligence/assets/webinar-informative-detailed-paragraphs.txt b/docs/speech-to-text/add-ons/assets/webinar-informative-detailed-paragraphs.txt similarity index 100% rename from docs/speech-to-text/batch/speech-intelligence/assets/webinar-informative-detailed-paragraphs.txt rename to docs/speech-to-text/add-ons/assets/webinar-informative-detailed-paragraphs.txt diff --git a/docs/speech-to-text/batch/alignment.mdx b/docs/speech-to-text/add-ons/audio-alignment.mdx similarity index 92% rename from docs/speech-to-text/batch/alignment.mdx rename to docs/speech-to-text/add-ons/audio-alignment.mdx index 1d74a181..c366a634 100644 --- a/docs/speech-to-text/batch/alignment.mdx +++ b/docs/speech-to-text/add-ons/audio-alignment.mdx @@ -13,6 +13,12 @@ This allows users to determine when exactly a given word was spoken in the conte If you do not have access to use the alignment feature, and you would like to, please reach out to [Support](https://support.speechmatics.com) or speak to your Account Manager. +## Availability + +Add-ons produce an output derived from a completed transcript, and are selected in addition to transcription. + +Audio alignment is available with the Standard and Enhanced models for pre-recorded transcription on SaaS on Cloud. It is available to Enterprise customers only. See [Feature availability](/speech-to-text/availability/). + ## Supported Formats The input text file must be UTF-8 encoded plain text file. Characters outside this format will mean the job is rejected. @@ -78,7 +84,7 @@ The job config must state that the job type is `alignment`, the language of the ## Retrieving alignment jobs -Checking status of Alignment jobs is done in the same way as transcription jobs. This is described on [this page](/speech-to-text/batch/output). +Checking status of Alignment jobs is done in the same way as transcription jobs. This is described on [this page](/speech-to-text/pre-recorded/output). An aligned file can be retrieved from the `/v2/jobs//alignment` endpoint. By default, the `word_start_and_end` alignment format is returned. This can be overridden using the `tags` query string parameter: @@ -106,7 +112,7 @@ Use the following endpoints to retrieve the inputs files used for an alignment j - `/v2/jobs//text`: to get the text file submitted - `/v2/jobs//data`: to get the audio file submitted -Note that Alignment follows Speechmatics' [Data Retention Limits](/speech-to-text/batch/limits#data-retention-limits). +Note that Alignment follows Speechmatics' [Data Retention Limits](/speech-to-text/pre-recorded/limits#data-retention-limits). ## Fetching files from an online location diff --git a/docs/speech-to-text/batch/speech-intelligence/auto-chapters.mdx b/docs/speech-to-text/add-ons/chapters.mdx similarity index 89% rename from docs/speech-to-text/batch/speech-intelligence/auto-chapters.mdx rename to docs/speech-to-text/add-ons/chapters.mdx index aea13592..53dcbde4 100644 --- a/docs/speech-to-text/batch/speech-intelligence/auto-chapters.mdx +++ b/docs/speech-to-text/add-ons/chapters.mdx @@ -33,7 +33,7 @@ import { pick } from "lodash"; Speechmatics enables you to identify distinct, continuous chapters in your audio. With just a single API call, you can quickly transcribe and extract the timings, title and summary for each chapter. -If you're new to Speechmatics, please see our guide on [Transcribing a File](/speech-to-text/batch/quickstart). Once you are set up, include the following config to enable Chapters: +If you're new to Speechmatics, please see our guide on [Transcribing a File](/speech-to-text/pre-recorded/quickstart). Once you are set up, include the following config to enable Chapters: ```json { @@ -48,6 +48,12 @@ If you're new to Speechmatics, please see our guide on [Transcribing a File](/sp } ``` +## Availability + +Add-ons produce an output derived from a completed transcript, and are selected in addition to transcription. + +Chapters are available with the Standard and Enhanced models for pre-recorded transcription on SaaS on Cloud. See [Feature availability](/speech-to-text/availability/). + ## Example Python client example to return Chapters for a batch file. @@ -126,8 +132,8 @@ Chapters are non-overlapping and exhaustive i.e. the chapters cover the entire a ## Considerations - Chapters works best for longer-form content. A minimum of 10 minutes of transcribed speech is recommended for best results -- Chapters works best with Speaker or Channel [Diarization](/speech-to-text/features/diarization) enabled -- Punctuation plays a significant role in the quality of the Chapters. It is therefore recommended to avoid disabling any end of sentence [punctuation](/speech-to-text/formatting#punctuation) or reducing the [punctuation sensitivity](/speech-to-text/formatting#punctuation) +- Chapters works best with Speaker or Channel [Diarization](/speech-to-text/features/speaker-diarization) enabled +- Punctuation plays a significant role in the quality of the Chapters. It is therefore recommended to avoid disabling any end of sentence [punctuation](/speech-to-text/features/formatting#punctuation) or reducing the [punctuation sensitivity](/speech-to-text/features/formatting#punctuation) ## Error responses diff --git a/docs/speech-to-text/batch/speech-intelligence/sentiment-analysis.mdx b/docs/speech-to-text/add-ons/sentiment.mdx similarity index 83% rename from docs/speech-to-text/batch/speech-intelligence/sentiment-analysis.mdx rename to docs/speech-to-text/add-ons/sentiment.mdx index 66c123ff..5652ee02 100644 --- a/docs/speech-to-text/batch/speech-intelligence/sentiment-analysis.mdx +++ b/docs/speech-to-text/add-ons/sentiment.mdx @@ -11,9 +11,9 @@ import batchSpec from "!openapi-schema-loader!@site/spec/batch.yaml" # Sentiment analysis -Speechmatics enables you to understand the sentiment of what was spoken in your audio. With just a single API call, you can quickly transcribe and extract the sentiment, either positive, negative or neutral to quickly detect areas of conversation to analyse further. +Speechmatics enables you to understand the sentiment of what was spoken in your audio. With just a single API call, you can quickly transcribe and extract the sentiment, either positive, negative or neutral to quickly detect areas of conversation to analyze further. -If you're new to Speechmatics, please see our guide on [Transcribing a File](/speech-to-text/batch/quickstart). Once you are set up, include the following config to enable Sentiment: +If you're new to Speechmatics, please see our guide on [Transcribing a File](/speech-to-text/pre-recorded/quickstart). Once you are set up, include the following config to enable Sentiment: ```json { @@ -26,6 +26,12 @@ If you're new to Speechmatics, please see our guide on [Transcribing a File](/sp } ``` +## Availability + +Add-ons produce an output derived from a completed transcript, and are selected in addition to transcription. + +Sentiment is available with the Standard and Enhanced models for pre-recorded transcription, on SaaS on Cloud and on-prem. See [Feature availability](/speech-to-text/availability/). + ## Example Python client example to return transcription with sentiment analysis for an audio file. @@ -102,11 +108,11 @@ Segments of the transcript are labeled with either positive, negative or neutral - Sentiment is only supported for English - Typically, segments equate to full sentences. However, in cases of short utterances composed of 1-2 words, these can be combined with a neighboring sentence to form a segment -- Punctuation plays a significant role in the accuracy of the Sentiment. It is therefore recommended to avoid disabling any end of sentence [punctuation](/speech-to-text/formatting#punctuation) or reducing the [punctuation sensitivity](/speech-to-text/formatting#punctuation) to ensure the best possible results -- Speaker or channel labels will only be available if [Diarization](/speech-to-text/features/diarization) is enabled. By default when Diarization is not enabled "speaker":"UU" is returned to indicate that speakers of each segment are unknown +- Punctuation plays a significant role in the accuracy of the Sentiment. It is therefore recommended to avoid disabling any end of sentence [punctuation](/speech-to-text/features/formatting#punctuation) or reducing the [punctuation sensitivity](/speech-to-text/features/formatting#punctuation) to ensure the best possible results +- Speaker or channel labels will only be available if [Diarization](/speech-to-text/features/speaker-diarization) is enabled. By default when Diarization is not enabled "speaker":"UU" is returned to indicate that speakers of each segment are unknown - Speaker Change is not supported in combination with Sentiment -# Error responses +## Error responses ### Unsupported language diff --git a/docs/speech-to-text/batch/speech-intelligence/summarization.mdx b/docs/speech-to-text/add-ons/summaries.mdx similarity index 93% rename from docs/speech-to-text/batch/speech-intelligence/summarization.mdx rename to docs/speech-to-text/add-ons/summaries.mdx index 51e00d89..77d44390 100644 --- a/docs/speech-to-text/batch/speech-intelligence/summarization.mdx +++ b/docs/speech-to-text/add-ons/summaries.mdx @@ -43,7 +43,7 @@ import SchemaNode from "@theme/Schema" Speechmatics enables you to generate a concise summary from your audio. With just a single API call, you can quickly transcribe and summarize content, making content review simpler and more efficient. -If you're new to Speechmatics, please see our guide on [Transcribing a File](/speech-to-text/batch/quickstart). Once you are set up, include the following config to enable Summaries: +If you're new to Speechmatics, please see our guide on [Transcribing a File](/speech-to-text/pre-recorded/quickstart). Once you are set up, include the following config to enable Summaries: ```json { @@ -56,6 +56,12 @@ If you're new to Speechmatics, please see our guide on [Transcribing a File](/sp } ``` +## Availability + +Add-ons produce an output derived from a completed transcript, and are selected in addition to transcription. + +Summaries are available with the Standard and Enhanced models for pre-recorded transcription on SaaS on Cloud. See [Feature availability](/speech-to-text/availability/). + ## Example Python example to summarize a filed with the default parameters: diff --git a/docs/speech-to-text/batch/speech-intelligence/topic-detection.mdx b/docs/speech-to-text/add-ons/topics.mdx similarity index 96% rename from docs/speech-to-text/batch/speech-intelligence/topic-detection.mdx rename to docs/speech-to-text/add-ons/topics.mdx index b7a15eb2..aa2d2f41 100644 --- a/docs/speech-to-text/batch/speech-intelligence/topic-detection.mdx +++ b/docs/speech-to-text/add-ons/topics.mdx @@ -24,7 +24,7 @@ import topicsPythonSample from "./assets/topics.py"; Speechmatics enables you to detect topics from your audio. With just a single API call, you can quickly transcribe and identify key topics with the corresponding segments of audio, allowing you to tag conversations or identify trends and patterns. -If you're new to Speechmatics, please see our guide on [Transcribing a File](/speech-to-text/batch/quickstart). Once you are set up, include the following config to enable Topics: +If you're new to Speechmatics, please see our guide on [Transcribing a File](/speech-to-text/pre-recorded/quickstart). Once you are set up, include the following config to enable Topics: ```json { @@ -40,6 +40,12 @@ If you're new to Speechmatics, please see our guide on [Transcribing a File](/sp } ``` +## Availability + +Add-ons produce an output derived from a completed transcript, and are selected in addition to transcription. + +Topics are available with the Standard and Enhanced models for pre-recorded transcription on SaaS on Cloud. See [Feature availability](/speech-to-text/availability/). + ## Example Python client example to detect topics in a file for Batch with the default parameters. diff --git a/docs/speech-to-text/features/translation.mdx b/docs/speech-to-text/add-ons/translation.mdx similarity index 92% rename from docs/speech-to-text/features/translation.mdx rename to docs/speech-to-text/add-ons/translation.mdx index 407dabab..f01220e0 100644 --- a/docs/speech-to-text/features/translation.mdx +++ b/docs/speech-to-text/add-ons/translation.mdx @@ -26,6 +26,12 @@ import transcriptResponseSchema from "!openapi-schema-loader!@site/spec/batch.ya Translate your audio into multiple languages through a single API call, with over 30 languages supported. +## Availability + +Add-ons produce an output derived from a completed transcript, and are selected in addition to transcription. + +Translation is available with the Standard and Enhanced models for pre-recorded and streaming transcription, on SaaS on Cloud and on-prem. It is not available for agent STT. See [Feature availability](/speech-to-text/availability/). + ## Use cases - Translate audio files for international distribution @@ -36,7 +42,7 @@ Translate your audio into multiple languages through a single API call, with ove Enable translation when processing a file or in real-time in SaaS and [on-prem deployment](/deployments/container/gpu-translation). -New to Speechmatics? See our guides on [transcribing a file](/speech-to-text/batch/quickstart) or [transcribing in real-time](/speech-to-text/realtime/quickstart). Once set up, add the following configuration to enable translation: +New to Speechmatics? See our guides on [transcribing a file](/speech-to-text/pre-recorded/quickstart) or [transcribing in real-time](/speech-to-text/streaming/quickstart). Once set up, add the following configuration to enable translation: ```json { @@ -215,7 +221,7 @@ with BatchClient(settings) as client: print(f'job {job_id} submitted successfully, waiting for transcript') # Note that in production, you should set up notifications instead of polling. - # Notifications are described here: https://docs.speechmatics.com/features-other/notifications + # Notifications are described here: https://docs.speechmatics.com/speech-to-text/pre-recorded/notifications transcript = client.wait_for_completion(job_id, transcription_format='json-v2') for language in TRANSLATION_LANGUAGES: # Print the translation for each language from the JSON @@ -236,7 +242,7 @@ with BatchClient(settings) as client: -Python client example to translate a file in real-time, see [here](/speech-to-text/realtime/quickstart) for more examples of Realtime Transcription +Python client example to translate a file in real-time, see [here](/speech-to-text/streaming/quickstart) for more examples of Realtime Transcription ```python showLineNumbers import speechmatics @@ -362,7 +368,7 @@ In batch, you can also translate from Norwegian Bokmål to Nynorsk. Follow these guidelines to achieve optimal translation results: - **Use the enhanced model** — Higher transcription accuracy directly leads to better translations -- **Keep punctuation enabled** — Maintain all [punctuation settings](/speech-to-text/formatting#punctuation) at default levels for optimal translation quality +- **Keep punctuation enabled** — Maintain all [punctuation settings](/speech-to-text/features/formatting#punctuation) at default levels for optimal translation quality - **Consider processing times** — Each additional target language increases processing time in batch jobs - **Plan for connection closing** — Realtime sessions may have a 5-second delay when finalizing translations @@ -373,8 +379,8 @@ Be aware of these limitations: - **Reduced metadata** — Certain features (timestamps, confidence scores, word tagging, and regional spelling) are only available in the original language ::: -## Next Steps +## Next steps - Visit the [Speechmatics Academy](https://github.com/speechmatics/speechmatics-academy/tree/main/basics/05-translation) to learn how to use translation in your projects. - [Try the portal](https://portal.speechmatics.com/) to see how translation works with your own audio. -- Use [diarization](/speech-to-text/features/diarization) to enhance your translations with speaker information. +- Use [diarization](/speech-to-text/features/speaker-diarization) to enhance your translations with speaker information. diff --git a/docs/speech-to-text/agent-stt/configuration.mdx b/docs/speech-to-text/agent-stt/configuration.mdx new file mode 100644 index 00000000..1fccd79e --- /dev/null +++ b/docs/speech-to-text/agent-stt/configuration.mdx @@ -0,0 +1,68 @@ +--- +title: Agent STT configuration +sidebar_label: Configuration +description: Look up every audio format and transcription option an agent STT session accepts. +toc_max_heading_level: 3 +--- + +Configuration is passed in `StartRecognition` and is split across two levels of the payload: `audio_format` at the top level, and `transcription_config`. + +Agent STT is in [Preview](/speech-to-text/agent-stt/#preview-limitations). + +## Audio format {#audio-format} + +:::warning +Only `pcm_s16le` at 8000 Hz or 16000 Hz is supported. Other encodings such as `pcm_f32le` and `mulaw`, and other sample rates such as 44100 Hz, may be silently accepted by the API but will not produce correct output. +::: + +| Field | Notes | +|---|---| +| `type` | Must be `raw` | +| `encoding` | Must be `pcm_s16le` (16-bit signed little-endian PCM) | +| `sample_rate` | Must be `8000` or `16000` | + +For example: + +```json +{ "type": "raw", "encoding": "pcm_s16le", "sample_rate": 16000 } +``` + +## Transcription config + +The `transcription_config` object accepts the following fields. + +| Field | Default | Notes | +|---|---|---| +| `language` | `en` | All supported languages | +| `output_locale` | — | Output locale, for example `en-US` | +| `additional_vocab` | — | Custom vocabulary entries | +| `punctuation_overrides` | — | Custom punctuation rules | +| `domain` | — | Domain-specific tuning, for example `medical` | +| `enable_entities` | `false` | Entity detection | +| `enable_partials` | `true` | Emit partial segments during speech | +| `diarization` | `speaker` | Speaker diarization; set to `none` to disable | +| `volume_threshold` | — | Minimum audio volume to process | + +For the languages available, see [Transcription languages](/speech-to-text/languages/transcription). For the medical domain, see [Medical domain](/speech-to-text/features/medical-domain). + +## Speaker diarization config + +The `transcription_config.speaker_diarization_config` object accepts the following fields. All of them require `diarization` to be set to `speaker`. + +| Field | Default | Notes | +|---|---|---| +| `max_speakers` | — | Maximum number of speakers to track | +| `speaker_sensitivity` | — | Sensitivity of speaker separation | +| `prefer_current_speaker` | — | Bias toward the most recently active speaker | +| `known_speakers` | — | Pre-enrolled speaker identifiers for cross-session recognition | + +For how to obtain and use `known_speakers`, see [Speaker focus and identification](/speech-to-text/agent-stt/speaker-focus). + +## Unsupported fields {#unsupported-fields} + +The following fields are rejected if present in an agent STT request. + +| Field | Notes | +|---|---| +| `translation_config` | Not supported on agent endpoints | +| `audio_events_config` | Not supported on agent endpoints | diff --git a/docs/speech-to-text/agent-stt/index.mdx b/docs/speech-to-text/agent-stt/index.mdx new file mode 100644 index 00000000..cca51efe --- /dev/null +++ b/docs/speech-to-text/agent-stt/index.mdx @@ -0,0 +1,81 @@ +--- +title: How agent STT works +sidebar_label: Overview +description: Understand how agent STT returns turn-based transcripts for conversational applications. +pagination_prev: null +--- + +Agent STT is a WebSocket interaction pattern for conversational applications. You stream audio in and receive speaker-labeled, turn-based transcription back: clean, punctuated, and ready to pass directly to a language model. + +Turn detection runs on the server. You choose a [profile](/speech-to-text/agent-stt/profiles) based on your use case, and the server decides when to finalize each speaker's turn. + +:::info Agent STT is in Preview +Agent STT is available on SaaS on Cloud for evaluation and feedback. It is not production-ready and not ready to scale. See [Preview limitations](#preview-limitations). +::: + +## What agent STT provides + +Agent STT uses the Linden 1 model and gives you four things a raw transcript stream does not: + +- **Turn detection** — the server decides when a speaker has finished talking, so you know when to respond +- **Segmentation** — transcripts arrive as complete speaker turns rather than as a word-by-word stream +- **Diarization** — each segment carries a speaker label, and speakers persist for the session +- **Speaker focus** — choose which speakers your application acts on and which it ignores + +## How agent STT differs from streaming + +Both agent STT and streaming send live audio over a WebSocket, and both use the Realtime processing mode. They differ in what comes back and in who decides when a turn has ended. + +| | Streaming | Agent STT | +|---|---|---| +| Transcript arrives as | words, refined by partials | complete speaker turns | +| Timings and partials | word-level | segment-level | +| Turn detection | you configure a silence trigger | the server runs a profile | +| Model | Standard, Enhanced, Melia 1 | Linden 1 | + +Use streaming when you want the raw word-by-word stream and your own segmentation. Use agent STT when you want each finished turn handed to you ready for a language model. + +Agent STT provides the speech-to-text layer only. It is not a conversational pipeline: it does not include a language model or speech synthesis. + +Three separate things configure turn detection across Speechmatics surfaces, and they are not interchangeable: + +- **Agent STT profiles** (`adaptive`, `agile`, `smart`, `external`) — server-side, selected in the endpoint path +- **Voice SDK presets** (`FAST`, `FIXED`, `ADAPTIVE`, `SMART_TURN`, `EXTERNAL`, `SCRIBE`, `CAPTIONS`) — client-side, over the Realtime API +- **Realtime API `conversation_config`** — a silence threshold you set yourself + +The only genuine correspondence is between the agent STT `external` profile and the Voice SDK `EXTERNAL` preset: both hand the turn boundary to your application. Otherwise, do not map one name onto another. + +## Where agent STT runs + +Agent STT runs on SaaS on Cloud. Audio is processed in the EU or the US. You cannot select a region during Preview, and agent STT is not available in the AUS region. + +For the features Linden 1 supports, see [Feature availability](/speech-to-text/availability/#agent-stt-features). + +## Preview limitations + +Agent STT is a preview offering with the following limitations. + +- Do not send live production traffic. The service is less stable than production endpoints. +- There are no uptime or performance SLAs. +- There are no data residency guarantees. Audio may be processed in the EU or the US. +- Features may change. Preview features may be withdrawn, or may never be released publicly. + +## Give feedback + +This is a preview, and your feedback shapes what goes to general availability. Tell us what works well, which features you use, whether something did not work as expected, a profile that behaved differently from what you anticipated, or a feature you would want before we ship more broadly. + +Specific areas of interest: + +- Integration experience, including documentation, SDKs, and API messages and metadata +- Accuracy and latency, including data capture where relevant, such as phone numbers or spelled-out names and account numbers +- Turn detection, and your experience with the different profiles +- Any missing capabilities that would make your product better +- What would stop you using this in production + +To talk to us in person, contact your Speechmatics contact or use the channel shared in your preview welcome email. You can also [fill in this form](https://docs.google.com/forms/d/e/1FAIpQLSc-6GQXYx_0M-X0Uu_uB_4XyDL009jMv3hBJAFw7kD98AILJg/viewform). + +## Next steps + +- [Quickstart](/speech-to-text/agent-stt/quickstart) — open a session and receive your first turn +- [Turn detection profiles](/speech-to-text/agent-stt/profiles) — choose a profile +- [Agent STT messages](/speech-to-text/agent-stt/messages) — every message in a session diff --git a/docs/speech-to-text/agent-stt/messages.mdx b/docs/speech-to-text/agent-stt/messages.mdx new file mode 100644 index 00000000..7dc3ae17 --- /dev/null +++ b/docs/speech-to-text/agent-stt/messages.mdx @@ -0,0 +1,466 @@ +--- +title: Agent STT messages +sidebar_label: Messages +description: Look up every client and server message in an agent STT session, with example payloads. +toc_max_heading_level: 3 +--- + +Every agent STT session follows the same structure: connect, start recognition, stream audio, receive turn events, close. + +Agent STT is in [Preview](/speech-to-text/agent-stt/#preview-limitations). + +## Session flow + +```mermaid +sequenceDiagram + participant C as Client + participant S as Server + + C->>S: Connect to endpoint with profile via WebSocket + C->>S: StartRecognition + S-->>C: RecognitionStarted + + loop Audio Stream + C->>S: Audio frames (binary) + S-->>C: AudioAdded + S-->>C: SpeechStarted + S-->>C: StartOfTurn + S-->>C: SpeakerStarted + S-->>C: AddPartialSegment (repeating) + S-->>C: SpeakerMetrics (repeating) + S-->>C: EndOfTurnPrediction (adaptive, smart) + S-->>C: SmartTurnResult (smart only) + S-->>C: SpeechEnded + S-->>C: EndOfUtterance + S-->>C: SpeakerEnded + S-->>C: AddSegment + S-->>C: EndOfTurn + + opt Mid-session controls + C->>S: ForceEndOfUtterance (external only) + C->>S: UpdateSpeakerFocus + C->>S: GetSpeakers + S-->>C: SpeakersResult + end + end + + C->>S: EndOfStream + S-->>C: EndOfTranscript +``` + +In sequence, a session runs as follows. + +1. The client connects to the endpoint for its chosen profile over a WebSocket, then sends `StartRecognition`. The server replies `RecognitionStarted`. +2. The client streams binary audio frames. The server acknowledges each with `AudioAdded`. +3. When speech is detected the server emits `SpeechStarted`, then `StartOfTurn`, then `SpeakerStarted` for the speaker who is talking. +4. While the speaker continues, the server repeatedly emits `AddPartialSegment` with the interim transcript, and `SpeakerMetrics` with per-speaker counts. The `adaptive` and `smart` profiles also emit `EndOfTurnPrediction`; `smart` additionally emits `SmartTurnResult`. +5. As the turn closes, the server emits `SpeechEnded`, `EndOfUtterance`, and `SpeakerEnded`, then `AddSegment` with the final transcript, then `EndOfTurn`. +6. At any point mid-session the client may send `ForceEndOfUtterance` (the `external` profile only), `UpdateSpeakerFocus`, or `GetSpeakers`, which the server answers with `SpeakersResult`. +7. When the client has no more audio it sends `EndOfStream`, and the server closes with `EndOfTranscript`. + +`SessionMetrics` is emitted every 5 seconds, independently of turn boundaries. + +## Messages sent by the client + +| Message | When to send | +|---|---| +| [`StartRecognition`](#startrecognition) | First message after connecting. Starts the session and passes configuration. | +| Audio frames | Binary WebSocket frames containing raw PCM audio, sent continuously. | +| [`ForceEndOfUtterance`](#forceendofutterance) | `external` profile only. Triggers immediate turn finalization. | +| [`UpdateSpeakerFocus`](#updatespeakerfocus) | Any time during the session. Changes which speakers are in focus. | +| [`GetSpeakers`](#getspeakers) | Any time during the session. Requests voice identifiers for diarized speakers. | +| [`EndOfStream`](#endofstream) | When there is no more audio to send. | + +## Messages sent by the server + +These are the messages your application logic acts on. + +| Message | Profile | When it is emitted | +|---|---|---| +| [`StartOfTurn`](#startofturn) | All | A speaker begins a new turn | +| [`AddPartialSegment`](#addpartialsegment) | All | Interim transcript update; each replaces the previous | +| [`AddSegment`](#addsegment) | All | Final transcript for the turn — pass this to your language model | +| [`EndOfTurn`](#endofturn) | All | Turn complete; your application can now respond | + +Two messages predict the end of a turn early, so you can start preparing a response. + +| Message | Profile | When it is emitted | +|---|---|---| +| [`EndOfTurnPrediction`](#endofturnprediction) | `adaptive`, `smart` | The model predicts the current turn will end soon | +| [`SmartTurnResult`](#smartturnresult) | `smart` only | High-confidence acoustic prediction of turn completion | + +These messages report speech and speaker activity, independently of turn boundaries. + +| Message | Profile | When it is emitted | +|---|---|---| +| [`SpeechStarted`](#speechstarted--speechended) | All | Voice activity detected in the audio stream | +| [`SpeechEnded`](#speechstarted--speechended) | All | Voice activity stopped | +| [`SpeakerStarted`](#speakerstarted--speakerended) | All | A specific diarized speaker began talking | +| [`SpeakerEnded`](#speakerstarted--speakerended) | All | A specific diarized speaker stopped talking | +| [`SpeakersResult`](#speakersresult) | All | Response to `GetSpeakers` | + +These messages track the session lifecycle. + +| Message | When it is emitted | +|---|---| +| `RecognitionStarted` | Session ready; emitted in response to `StartRecognition` | +| `AudioAdded` | Audio frame acknowledged | +| `EndOfTranscript` | Session closing; emitted after `EndOfStream` | + +These messages carry metrics and diagnostics. + +| Message | When it is emitted | +|---|---| +| [`SessionMetrics`](#sessionmetrics) | Session stats; emitted every 5 seconds and at session end | +| [`SpeakerMetrics`](#speakermetrics) | Per-speaker word count and volume; emitted on each recognized word | + +## Messages shared with the Realtime API + +These messages are shared with the Realtime API. For full payload details, see the [Realtime API reference](/api-ref/realtime-transcription-websocket). + +| Message | When it is emitted | +|---|---| +| `EndOfUtterance` | Silence threshold reached; precedes turn finalization | +| `Info` | Non-critical informational message | +| `Warning` | Non-fatal issue, for example an unsupported config field being ignored | +| `Error` | Fatal error; the connection will close | + +`RecognitionStarted`, `AudioAdded`, `AddPartialTranscript`, `AddTranscript` and `EndOfTranscript` are also shared with the Realtime API. + +## Client message payloads + +### StartRecognition + +The first message you send after connecting. Starts the recognition session and passes configuration. The server responds with `RecognitionStarted`. + +```json +{ + "message": "StartRecognition", + "audio_format": { + "type": "raw", + "encoding": "pcm_s16le", + "sample_rate": 16000 + }, + "transcription_config": { + "language": "en" + } +} +``` + +For all configuration options, see [Agent STT configuration](/speech-to-text/agent-stt/configuration). + +### EndOfStream + +Send when you have finished streaming audio. The server finalizes any remaining transcript and then emits `EndOfTranscript`. `last_seq_no` is the sequence number of the last audio frame you sent. + +```json +{ + "message": "EndOfStream", + "last_seq_no": 1234 +} +``` + +### ForceEndOfUtterance + +Applies to the `external` profile only. Immediately ends the current turn: the server finalizes all audio received so far and emits a single `AddSegment` containing the complete transcript for that turn, followed by `EndOfTurn`. + +Send this wherever your application decides a turn is complete: on button release for push-to-talk, on VAD silence, or on a signal from your language model. + +```json +{ + "message": "ForceEndOfUtterance" +} +``` + +### UpdateSpeakerFocus + +Updates which speakers are in focus, mid-session. Takes effect immediately. See [Speaker focus and identification](/speech-to-text/agent-stt/speaker-focus) for full details. + +```json +{ + "message": "UpdateSpeakerFocus", + "speaker_focus": { + "focus_speakers": ["S1"], + "ignore_speakers": [], + "focus_mode": "retain" + } +} +``` + +### GetSpeakers + +Requests voice identifiers for all speakers diarized so far in the session. The server responds with `SpeakersResult`. + +```json +{ + "message": "GetSpeakers" +} +``` + +## Server message payloads + +### StartOfTurn + +Emitted when a speaker begins a new turn. Use this to signal to your application that it should stop speaking if it currently is. + +```json +{ + "message": "StartOfTurn", + "turn_id": 42 +} +``` + +- `turn_id` — monotonically increasing integer; pairs with the corresponding `EndOfTurn` + +### EndOfTurn + +Emitted when turn detection decides the speaker has finished. This is the trigger for your application to respond. The finalized transcript for the turn is in the preceding `AddSegment`. + +```json +{ + "message": "EndOfTurn", + "turn_id": 42, + "metadata": { + "start_time": 0.84, + "end_time": 3.24 + } +} +``` + +- `turn_id` — matches the `StartOfTurn` for this turn +- `metadata.start_time` and `metadata.end_time` — audio time range for the turn, in seconds from session start + +### AddPartialSegment + +Interim transcript update, emitted continuously while the speaker is talking. Each new `AddPartialSegment` replaces the previous one; do not concatenate them. + +```json +{ + "message": "AddPartialSegment", + "segments": [ + { + "speaker_id": "S1", + "is_active": true, + "timestamp": "2025-01-01T12:00:00.000+00:00", + "language": "en", + "text": "Good evening", + "is_eou": false, + "metadata": { + "start_time": 0.84, + "end_time": 1.24 + } + } + ], + "metadata": { + "start_time": 0.84, + "end_time": 1.24, + "processing_time": 0.23 + } +} +``` + +### AddSegment + +The final, complete transcript for a turn, emitted just before `EndOfTurn`. This is the stable output to pass to your language model; do not use `AddPartialSegment` for this. + +In multi-speaker scenarios, a single `AddSegment` may contain segments from multiple speakers, returned in time order. + +```json +{ + "message": "AddSegment", + "segments": [ + { + "speaker_id": "S1", + "is_active": true, + "timestamp": "2025-01-01T12:00:00.000+00:00", + "language": "en", + "text": "Good evening.", + "is_eou": true, + "metadata": { + "start_time": 0.84, + "end_time": 1.56 + } + } + ], + "metadata": { + "start_time": 0.84, + "end_time": 1.56, + "processing_time": 0.25 + } +} +``` + +Segment fields: + +- `speaker_id` — speaker label, for example `S1` or `S2`, or a custom label if using [speaker identification](/speech-to-text/agent-stt/speaker-focus#recognize-a-speaker-across-sessions) +- `is_active` — `true` if this speaker is in your current focus list, `false` if they are a background speaker +- `is_eou` — `true` on final segments, `false` on partials +- `text` — clean, punctuated transcript text +- `metadata.start_time` and `metadata.end_time` — time range of this segment, in seconds from session start + +Message-level fields: + +- `metadata.processing_time` — transcription latency in seconds for this message + +### SpeakerStarted / SpeakerEnded + +Emitted when a specific speaker starts or stops being heard. These are voice activity events: they fire based on detected speech, independently of turn boundaries. + +```json +{ + "message": "SpeakerStarted", + "speaker_id": "S1", + "is_active": true, + "time": 0.84, + "metadata": { "start_time": 0.84, "end_time": 0.84 } +} +``` + +```json +{ + "message": "SpeakerEnded", + "speaker_id": "S1", + "is_active": true, + "time": 3.24, + "metadata": { "start_time": 0.84, "end_time": 3.24 } +} +``` + +- `speaker_id` — the speaker whose activity changed +- `is_active` — whether this speaker is in your current focus list +- `time` — seconds from session start when the activity was detected +- `metadata.start_time` — when this speaker started their current speaking interval +- `metadata.end_time` — when this speaker stopped speaking, on `SpeakerEnded` only + +### EndOfTurnPrediction + +Emitted by the `adaptive` and `smart` profiles when the model predicts the current turn is about to end. Use it to begin preparing a response before `EndOfTurn` arrives, reducing perceived latency. + +```json +{ + "message": "EndOfTurnPrediction", + "turn_id": 2, + "predicted_wait": 0.73, + "metadata": { + "ttl": 0.73, + "reasons": ["not__ends_with_eos"] + } +} +``` + +- `turn_id` — the turn this prediction applies to +- `predicted_wait` — estimated seconds until the turn ends +- `metadata.ttl` — time to live; how long this prediction remains valid +- `metadata.reasons` — internal signals that contributed to the prediction + +### SmartTurnResult + +:::warning +This message is emitted as `SmartTurnResult` during Preview. It will be renamed to `SmartTurnPrediction` at general availability. +::: + +Emitted by the `smart` profile only. A higher-confidence acoustic prediction of turn completion, based on the ML model that analyzes vocal cues. + +```json +{ + "message": "SmartTurnResult", + "prediction": { + "prediction": true, + "probability": 0.979, + "processing_time": 0.128 + }, + "metadata": { + "start_time": 0.0, + "end_time": 2.2, + "language": "en", + "speaker_id": "S1", + "total_time": 2.2 + } +} +``` + +- `prediction.prediction` — `true` if the model predicts the turn is complete +- `prediction.probability` — confidence score from 0 to 1 +- `prediction.processing_time` — time taken by the ML model, in seconds +- `metadata.start_time` and `metadata.end_time` — audio window analyzed +- `metadata.total_time` — total session time at the point of prediction +- `metadata.speaker_id` — speaker being analyzed, or `null` if not yet identified + +### SpeechStarted / SpeechEnded + +Voice activity detection events, emitted when speech is first detected in the audio stream or stops. These fire independently of speaker identity and turn boundaries. + +```json +{ + "message": "SpeechStarted", + "probability": 0.508, + "transition_duration_ms": 192.0, + "metadata": { + "start_time": 2.1, + "end_time": 2.1 + } +} +``` + +```json +{ + "message": "SpeechEnded", + "probability": 0.307, + "transition_duration_ms": 192.0, + "metadata": { + "start_time": 0.4, + "end_time": 2.5 + } +} +``` + +- `probability` — VAD confidence score from 0 to 1 +- `transition_duration_ms` — duration of the speech or silence transition, in milliseconds +- `metadata.start_time` — when speech began. On `SpeechStarted` this equals `end_time`; on `SpeechEnded` it is when the speaking interval started +- `metadata.end_time` — when the event was detected + +### SpeakersResult + +Emitted in response to `GetSpeakers`. Contains voice identifiers for all speakers diarized so far. See [Speaker focus and identification](/speech-to-text/agent-stt/speaker-focus#recognize-a-speaker-across-sessions) for how to store and use these. + +```json +{ + "message": "SpeakersResult", + "speakers": [ + { "label": "S1", "speaker_identifiers": [""] }, + { "label": "S2", "speaker_identifiers": [""] } + ] +} +``` + +### SessionMetrics + +Emitted every 5 seconds and once at the end of the session. + +```json +{ + "message": "SessionMetrics", + "total_time": 4.6, + "total_time_str": "00:00:04", + "total_bytes": 148480, + "processing_time": 0.295 +} +``` + +### SpeakerMetrics + +Emitted each time a speaker produces a recognized word. + +```json +{ + "message": "SpeakerMetrics", + "speakers": [ + { + "speaker_id": "S1", + "word_count": 6, + "last_heard": 2.36, + "volume": 5.2 + } + ] +} +``` diff --git a/docs/speech-to-text/agent-stt/profiles.mdx b/docs/speech-to-text/agent-stt/profiles.mdx new file mode 100644 index 00000000..f60adbe3 --- /dev/null +++ b/docs/speech-to-text/agent-stt/profiles.mdx @@ -0,0 +1,78 @@ +--- +title: Turn detection profiles +sidebar_label: Profiles +description: Compare the four agent STT turn detection profiles and choose an endpoint. +toc_max_heading_level: 3 +--- + +A profile is a pre-configured turn detection mode. Each profile sets the defaults for a use case: you choose one when connecting by including it in the endpoint path, and the server handles the rest. + +Agent STT is in [Preview](/speech-to-text/agent-stt/#preview-limitations). + +## Compare the profiles + +| Profile | Turn detection | Best for | +|---|---|---| +| `adaptive` | Adapts to speaker pace and hesitation | General conversational agents | +| `agile` | VAD-based silence detection | Speed-first use cases | +| `smart` | `adaptive` plus ML acoustic turn prediction | High-stakes conversations | +| `external` | Manual — you trigger turn end | Push-to-talk, custom VAD, LLM-driven | + +### adaptive + +**Endpoint:** `/v2/agent/adaptive` + +Adapts to each speaker's pace over the course of a conversation. It adjusts the turn-end threshold based on speech rate and disfluencies such as hesitations and filler words, waiting longer for speakers who tend to pause mid-thought. + +**Best for:** general conversational applications. + +**Languages:** all supported languages. Disfluency detection is available for the [supported languages](/speech-to-text/features/disfluencies-and-profanities#supported-languages-for-disfluencies); other languages fall back to speech-rate adaptation. + +**Trade-off:** latency varies by speaker. + +### agile + +**Endpoint:** `/v2/agent/agile` + +Uses voice activity detection (VAD) to detect silence and finalize turns as quickly as possible. This is the lowest-latency profile. + +**Best for:** use cases where response speed is the top priority and occasional mid-speech finalizations are acceptable. + +**Languages:** all supported languages. + +**Trade-off:** because it relies on silence, it may finalize a turn while the speaker is still mid-sentence, for example during a natural pause. This can result in additional downstream language model calls. + +### smart + +**Endpoint:** `/v2/agent/smart` + +Builds on `adaptive` with an additional ML model that analyzes acoustic cues to predict whether a speaker has genuinely finished their turn. This is the most conservative profile and the least likely to interrupt. + +**Best for:** high-stakes conversations where cutting off the user is costly, such as finance, healthcare, and legal. + +**Languages:** Arabic, Bengali, Chinese, Danish, Dutch, English, Finnish, French, German, Hindi, Indonesian, Italian, Japanese, Korean, Marathi, Norwegian, Polish, Portuguese, Russian, Spanish, Turkish, Ukrainian, Vietnamese. + +**Trade-off:** higher latency than `adaptive`. + +### external + +**Endpoint:** `/v2/agent/external` + +Turn detection is fully manual. The server accumulates audio and transcript until you send a [`ForceEndOfUtterance`](/speech-to-text/agent-stt/messages#forceendofutterance) message, at which point it finalizes everything spoken up to that point and emits an `AddSegment`. + +**Best for:** push-to-talk interfaces, custom VAD pipelines, or setups where a language model decides when to respond. + +**Languages:** all supported languages. + +**Trade-off:** you are responsible for all turn detection logic. + +## Early turn signals + +Two profiles emit predictions before the turn actually ends, which you can use to start preparing a response and reduce perceived latency: + +- [`EndOfTurnPrediction`](/speech-to-text/agent-stt/messages#endofturnprediction) — emitted by `adaptive` and `smart` +- [`SmartTurnResult`](/speech-to-text/agent-stt/messages#smartturnresult) — emitted by `smart` only, and higher-confidence + +## Turn detection in streaming + +Streaming transcription has its own turn detection, configured with a silence threshold rather than a profile. The two are separate mechanisms: see [Turn detection](/speech-to-text/features/turn-detection) for the streaming equivalent. diff --git a/docs/speech-to-text/agent-stt/quickstart.mdx b/docs/speech-to-text/agent-stt/quickstart.mdx new file mode 100644 index 00000000..b38e599e --- /dev/null +++ b/docs/speech-to-text/agent-stt/quickstart.mdx @@ -0,0 +1,89 @@ +--- +title: Quickstart +description: Open an agent STT session, stream audio, and act on each finalized turn. +pagination_prev: null +--- + +:::info Agent STT is in Preview +Agent STT is available on SaaS on Cloud for evaluation and feedback. It is not production-ready and not ready to scale. See [Preview limitations](/speech-to-text/agent-stt/#preview-limitations). +::: + +## Transcribe a conversation with agent STT + +### 1. Create an API key + +Create an API key in the [Speechmatics portal](https://portal.speechmatics.com/settings/api-keys), under **Settings > API Keys**. Store it as a managed secret rather than in source control. + +### 2. Choose a profile + +Agent STT decides when a turn has ended using a profile, which you select in the endpoint path. Start with `adaptive` for a general conversational application. See [Turn detection profiles](/speech-to-text/agent-stt/profiles) to compare all four. + +### 3. Connect + +Open a WebSocket connection to the preview endpoint, naming your chosen profile: + +```text +wss://preview.rt.speechmatics.com/v2/agent/ +``` + +### 4. Authenticate {#authenticate} + +Authenticate every connection using one of the following: + +| Method | Format | +|---|---| +| Header (API key) | `Authorization: Bearer ` | +| Header (JWT) | `Authorization: Bearer ` | +| Query parameter (API key) | `?api_key=` | +| Query parameter (JWT) | `?jwt=` | + +For temporary keys, see [Authentication](/get-started/authentication#temporary-keys). + +### 5. Start the session + +Send `StartRecognition` as your first message. This config transcribes English audio: + +```json +{ + "message": "StartRecognition", + "audio_format": { + "type": "raw", + "encoding": "pcm_s16le", + "sample_rate": 16000 + }, + "transcription_config": { + "language": "en" + } +} +``` + +The server responds with `RecognitionStarted` when the session is ready. Wait for that message before sending audio. + +For every option a session accepts, see [Agent STT configuration](/speech-to-text/agent-stt/configuration). + +### 6. Stream audio and handle turns + +Send audio as binary WebSocket frames. Turn events arrive as the API processes speech. Two messages carry the transcript: + +- `AddPartialSegment` — an interim update while the speaker is still talking. Each one replaces the previous; do not concatenate them. +- `AddSegment` — the final, stable transcript for the turn, emitted just before `EndOfTurn`. **This is the message to pass to your language model.** + +`EndOfTurn` is your cue to respond. For the full sequence and every payload, see [Agent STT messages](/speech-to-text/agent-stt/messages). + +## Code examples + +For working examples in Python and JavaScript, see the [Speechmatics Academy](https://github.com/speechmatics/speechmatics-academy/tree/main/basics/11-voice-api-explorer). + +## Troubleshooting + +**No transcript arrives.** Check your audio format. Only `pcm_s16le` at 8000 Hz or 16000 Hz produces correct output, and other encodings may be accepted silently without working. See [audio format](/speech-to-text/agent-stt/configuration#audio-format). + +**Nothing happens after connecting.** Send `StartRecognition` first, and wait for `RecognitionStarted` before sending audio. + +**A config field is ignored or the session is rejected.** `translation_config` and `audio_events_config` are rejected on agent endpoints. See [unsupported fields](/speech-to-text/agent-stt/configuration#unsupported-fields). + +## Next steps + +- [Turn detection profiles](/speech-to-text/agent-stt/profiles) — pick the profile that matches your latency and interruption trade-offs +- [Agent STT messages](/speech-to-text/agent-stt/messages) — every client and server message, with payloads +- [Speaker focus and identification](/speech-to-text/agent-stt/speaker-focus) — choose which speakers to act on, and recognize them across sessions diff --git a/docs/speech-to-text/agent-stt/sidebar.ts b/docs/speech-to-text/agent-stt/sidebar.ts new file mode 100644 index 00000000..5b261c0e --- /dev/null +++ b/docs/speech-to-text/agent-stt/sidebar.ts @@ -0,0 +1,33 @@ +// Declared explicitly rather than autogenerated: sidebar-generator.ts +// title-cases autogenerated category labels word by word, which would +// render "agent-stt" as "Agent Stt". +export default { + type: "category", + label: "Agent STT", + items: [ + { + type: "doc", + id: "speech-to-text/agent-stt/index", + }, + { + type: "doc", + id: "speech-to-text/agent-stt/quickstart", + }, + { + type: "doc", + id: "speech-to-text/agent-stt/profiles", + }, + { + type: "doc", + id: "speech-to-text/agent-stt/configuration", + }, + { + type: "doc", + id: "speech-to-text/agent-stt/messages", + }, + { + type: "doc", + id: "speech-to-text/agent-stt/speaker-focus", + }, + ], +} as const; diff --git a/docs/speech-to-text/agent-stt/speaker-focus.mdx b/docs/speech-to-text/agent-stt/speaker-focus.mdx new file mode 100644 index 00000000..3ac3fc67 --- /dev/null +++ b/docs/speech-to-text/agent-stt/speaker-focus.mdx @@ -0,0 +1,70 @@ +--- +title: Speaker focus and identification +sidebar_label: Speakers +description: Choose which speakers your application acts on, and recognize the same speaker across sessions. +--- + +Agent STT assigns a speaker label to every segment when diarization is enabled. Two controls build on those labels: speaker focus decides whose speech your application acts on within a session, and speaker identification recognizes the same person across separate sessions. + +Agent STT is in [Preview](/speech-to-text/agent-stt/#preview-limitations). + +## Control which speakers your application acts on + +Speaker focus lets you control which speakers' output your application acts on. By default, all detected speakers are active and their transcripts are included in `AddSegment` output. + +Speaker IDs such as `S1` and `S2` are assigned automatically when diarization is enabled, and persist for the lifetime of the session. Send `UpdateSpeakerFocus` at any point during the session to change who is in focus. The new config takes effect immediately and replaces the previous one. + +```json +{ + "message": "UpdateSpeakerFocus", + "speaker_focus": { + "focus_speakers": ["S1"], + "ignore_speakers": ["S3"], + "focus_mode": "retain" + } +} +``` + +- `focus_speakers` — speaker IDs to treat as active. Their segments appear with `is_active: true`. +- `ignore_speakers` — speaker IDs to exclude entirely. Their speech is dropped and does not affect turn detection. +- `focus_mode` — what happens to speakers who are in neither `focus_speakers` nor `ignore_speakers`: + - `retain` — they remain in the output as passive speakers, with `is_active: false` + - `ignore` — they are excluded from the output entirely + +## Recognize a speaker across sessions + +Speaker identification lets you recognize the same person across separate sessions. At the end of a session you retrieve voice identifiers for each speaker and store them. In later sessions you pass those identifiers into `StartRecognition`, and the system tags matching speakers with a consistent label rather than a generic `S1` or `S2`. + +### Get identifiers + +Send [`GetSpeakers`](/speech-to-text/agent-stt/messages#getspeakers) at any point during a session to retrieve identifiers for all speakers diarized so far. The server responds with [`SpeakersResult`](/speech-to-text/agent-stt/messages#speakersresult). + +Store the `speaker_identifiers` values from the response. These are opaque tokens tied to a speaker's voice profile. + +:::warning +Treat speaker identifiers as credentials and store them securely. +::: + +### Use identifiers in a later session + +Pass stored identifiers into `StartRecognition` using `transcription_config.known_speakers`. You can assign any label: + +```json +{ + "message": "StartRecognition", + "transcription_config": { + "language": "en", + "known_speakers": [ + { "label": "Alice", "speaker_identifiers": [""] }, + { "label": "Bob", "speaker_identifiers": [""] } + ] + } +} +``` + +When those speakers are detected, their segments carry `"Alice"` or `"Bob"` as the `speaker_id` instead of a generic label. Any unrecognized speakers are still assigned generic labels such as `S1` and `S2`. + +## Next steps + +- [Speaker diarization](/speech-to-text/features/speaker-diarization) — how diarization separates speakers +- [Speaker identification](/speech-to-text/features/speaker-identification) — the equivalent feature for pre-recorded and streaming transcription diff --git a/docs/speech-to-text/app-analytics.mdx b/docs/speech-to-text/app-analytics.mdx deleted file mode 100644 index e09f9772..00000000 --- a/docs/speech-to-text/app-analytics.mdx +++ /dev/null @@ -1,38 +0,0 @@ ---- -description: Track usage by adding an application ID to your requests. ---- - -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; - -# App analytics - -If your application allows users to enter their own Speechmatics API key, Speechmatics can offer you aggregated analytics about your users' usage levels. This includes the number of unique users, hours processed, languages used, and more. - -Please reach out to [Support](https://support.speechmatics.com) to get set up. - - - -Once set up, simply use the `sm-app` query parameter when starting a job. For example: - -```bash -APP_ID="YourAppID" -API_KEY="YOUR_API_KEY" -PATH_TO_FILE="example.wav" - -curl -L -X POST "https://eu1.asr.api.speechmatics.com/v2/jobs/?sm-app=${APP_ID}" \ - -H "Authorization: Bearer ${API_KEY}" \ - -F data_file=@${PATH_TO_FILE} \ - -F config='{"type": "transcription","transcription_config": { "model": "enhanced","language": "en" }}' -``` - - - -Once set up, simply use the `sm-app` query parameter in the connection string when starting a session. For example: - -```bash -wss://eu.rt.speechmatics.com/v2?sm-app=YourAppID -``` - - - diff --git a/docs/speech-to-text/availability/index.mdx b/docs/speech-to-text/availability/index.mdx new file mode 100644 index 00000000..7f41744e --- /dev/null +++ b/docs/speech-to-text/availability/index.mdx @@ -0,0 +1,233 @@ +--- +title: Feature availability +description: Check which features and add-ons each model supports on SaaS on Cloud, by interaction pattern. +sidebar_label: SaaS on Cloud +toc_max_heading_level: 3 +--- + +Availability depends on the combination of interaction pattern, model, and deployment, so check the combination rather than the product name. This page covers SaaS on Cloud. For container and virtual appliance deployments, see [On-prem availability](/speech-to-text/availability/on-prem). + +A check mark marks an available combination. An em dash means the combination is either unavailable or not established, and carries no roadmap position either way. + +{/* Derived from .CLAUDE/context/feature-availability.md. Regenerate when the STT product + catalogue changes; do not hand-correct a value here. Rows where no model is available + are omitted entirely, per the authoring rules in that file. */} + +## Readiness by combination + +Readiness is a separate question from availability: availability says whether something exists for a combination, readiness says whether that combination is usable and documentable. Readiness is a property of the combination, so it is stated here once and not repeated on individual feature pages. + +| Interaction pattern | Model | SaaS on Cloud | On-prem | +|---|---|---|---| +| pre-recorded | Standard | GA | Released | +| pre-recorded | Enhanced | GA | Released | +| pre-recorded | Melia 1 | GA | Released | +| streaming | Standard | GA | Released | +| streaming | Enhanced | GA | Released | +| streaming | Melia 1 | Preview | — | +| agent STT | Linden 1 | Preview | — | + +Preview applies to SaaS on Cloud only, because on-prem ships as versioned containers: the question there is which release you need. + +:::info Preview combinations +Streaming with Melia 1, and agent STT with Linden 1, are available for evaluation and feedback. They are not production-ready and not ready to scale. +::: + +## Pre-recorded transcription features + +Pre-recorded transcription runs on the Standard, Enhanced, and Melia 1 models, in the EU, US, and AUS regions. + +### Language coverage and selection + +| Item | Standard | Enhanced | Melia 1 | +|---|---|---|---| +| 56 languages | ✓ | ✓ | ✓ | +| Mixed-language transcription | — | — | ✓ | +| Language hints | — | — | ✓ | +| Language labeling | — | — | ✓ | +| Transcription language packs (including bilingual) | ✓ | ✓ | — | +| Automatic language identification | ✓ | ✓ | — | + +### Output and formatting + +| Item | Standard | Enhanced | Melia 1 | +|---|---|---|---| +| Output locale | ✓ | ✓ | ✓ | +| Smart formatting | ✓ | ✓ | ✓ | +| Punctuation and casing | ✓ | ✓ | ✓ | +| Word-level timings | ✓ | ✓ | ✓ | +| Segment-level timings | ✓ | ✓ | ✓ | +| Confidence scores | ✓ | ✓ | — | + +### Transcript content and tagging + +| Item | Standard | Enhanced | Melia 1 | +|---|---|---|---| +| Custom dictionary | ✓ | ✓ | — | +| Medical domain | — | ✓ | — | +| Entity detection (basic, legacy) | ✓ | ✓ | — | +| Disfluency tagging | ✓ | ✓ | — | +| Profanity tagging | ✓ | ✓ | — | +| Text replacement (find and replace) | ✓ | ✓ | — | + +### Speakers and channels + +| Item | Standard | Enhanced | Melia 1 | +|---|---|---|---| +| Speaker diarization | ✓ | ✓ | ✓ | +| Channel diarization | ✓ | ✓ | ✓ | +| Speaker identification | ✓ | ✓ | — | + +### Audio input + +| Item | Standard | Enhanced | Melia 1 | +|---|---|---|---| +| Audio events | ✓ | ✓ | — | +| Audio filtering (volume filtering) | ✓ | ✓ | — | +| Fetch URL | ✓ | ✓ | ✓ | + +### Operational + +| Item | Standard | Enhanced | Melia 1 | +|---|---|---|---| +| App usage tracking | ✓ | ✓ | ✓ | +| Notifications | ✓ | ✓ | ✓ | + +### Add-ons + +Add-ons are separate products that produce an output derived from a completed transcript. They are selected in addition to transcription. + +| Item | Standard | Enhanced | Melia 1 | +|---|---|---|---| +| Translation | ✓ | ✓ | — | +| Chapters | ✓ | ✓ | — | +| Topics | ✓ | ✓ | — | +| Summaries | ✓ | ✓ | — | +| Sentiment | ✓ | ✓ | — | +| Audio alignment | ✓ | ✓ | — | + +Audio alignment is available to Enterprise customers only. + +### Pre-recorded regions + +| Region | Standard | Enhanced | Melia 1 | +|---|---|---|---| +| EU | ✓ | ✓ | ✓ | +| US | ✓ | ✓ | ✓ | +| AUS | ✓ | ✓ | — | + +## Streaming transcription features + +Streaming transcription runs on the Standard, Enhanced, and Melia 1 models. Streaming is available in the EU and US regions, and is not available in the AUS region. Melia 1 for streaming is in Preview. + +### Streaming language coverage and selection + +| Item | Standard | Enhanced | Melia 1 | +|---|---|---|---| +| 56 languages | ✓ | ✓ | ✓ | +| Mixed-language transcription | — | — | ✓ | +| Language labeling | — | — | ✓ | +| Transcription language packs (including bilingual) | ✓ | ✓ | — | + +### Streaming output and formatting + +| Item | Standard | Enhanced | Melia 1 | +|---|---|---|---| +| Output locale | ✓ | ✓ | ✓ | +| Smart formatting | ✓ | ✓ | ✓ | +| Punctuation and casing | ✓ | ✓ | ✓ | +| Word-level timings | ✓ | ✓ | ✓ | +| Confidence scores | ✓ | ✓ | — | + +### Streaming transcript content and tagging + +| Item | Standard | Enhanced | Melia 1 | +|---|---|---|---| +| Custom dictionary | ✓ | ✓ | — | +| Medical domain | — | ✓ | — | +| Entity detection (basic, legacy) | ✓ | ✓ | — | +| Disfluency tagging | ✓ | ✓ | — | +| Profanity tagging | ✓ | ✓ | — | +| Text replacement (find and replace) | ✓ | ✓ | — | + +### Streaming speakers and channels + +| Item | Standard | Enhanced | Melia 1 | +|---|---|---|---| +| Speaker diarization | ✓ | ✓ | — | +| Channel diarization | ✓ | ✓ | ✓ | +| Speaker identification | ✓ | ✓ | — | + +### Streaming audio input + +| Item | Standard | Enhanced | Melia 1 | +|---|---|---|---| +| Audio events | ✓ | ✓ | — | +| Audio filtering (volume filtering) | ✓ | ✓ | — | + +### Responding + +These features control when the Realtime API returns a result, and how much of the transcript each result contains. + +| Item | Standard | Enhanced | Melia 1 | +|---|---|---|---| +| Force end of utterance | ✓ | ✓ | — | +| Turn detection | ✓ | ✓ | — | +| Word-level partials | ✓ | ✓ | ✓ | + +{/* App usage tracking is absent from the streaming table in feature-availability.md, + which reads as n/a. It is documented and working via the sm-app query parameter on the + WebSocket URL, so it is published here. TODO: correct the upstream STT product + catalogue and regenerate feature-availability.md. Not yet raised. */} + +### Streaming operational and add-ons + +| Item | Standard | Enhanced | Melia 1 | +|---|---|---|---| +| App usage tracking | ✓ | ✓ | ✓ | +| Translation | ✓ | ✓ | — | + +## Agent STT features + +Agent STT runs on the Linden 1 model in the EU and US regions, and is in Preview. Because agent STT offers a single model, the following lists name what Linden 1 supports rather than comparing columns. + +{/* Entity detection (basic, legacy), audio filtering, and force end of utterance are marked + No or Not yet for Linden 1 in feature-availability.md, but all three are documented, + shipped configuration on the /v2/agent endpoints. Published here. TODO: correct the + upstream STT product catalogue and regenerate feature-availability.md. Not yet raised. */} + +**Language and output** + +- 56 languages +- Transcription language packs (including bilingual) +- Output locale +- Smart formatting +- Punctuation and casing +- Segment-level timings + +**Transcript content and tagging** + +- Custom dictionary +- Medical domain +- Entity detection (basic, legacy) +- Text replacement (find and replace) + +**Speakers** + +- Speaker diarization +- Speaker identification + +**Responding** + +- Turn detection +- Voice activity detection (VAD) +- Force end of utterance +- Segment-level partials + +**Audio input** + +- Audio filtering (volume filtering) + +### Agent STT regions + +Agent STT processes audio in the EU and US regions. diff --git a/docs/speech-to-text/availability/on-prem.mdx b/docs/speech-to-text/availability/on-prem.mdx new file mode 100644 index 00000000..0e3aa553 --- /dev/null +++ b/docs/speech-to-text/availability/on-prem.mdx @@ -0,0 +1,163 @@ +--- +title: On-prem availability +description: Check which features each model supports on-prem for pre-recorded and streaming transcription. +sidebar_label: On-prem +toc_max_heading_level: 3 +--- + +Pre-recorded and streaming transcription run on-prem, as a container deployment or a virtual appliance. This page covers on-prem availability. For Speechmatics-hosted deployments, see [Feature availability](/speech-to-text/availability/). + +A check mark marks an available combination. An em dash means the combination is either unavailable or not established, and carries no roadmap position either way. + +Region does not apply on-prem. Regions are an attribute of SaaS on Cloud, because on-prem deployments are customer-hosted. + +{/* Derived from .CLAUDE/context/feature-availability.md. Regenerate when the STT product + catalogue changes; do not hand-correct a value here. Rows where no model is available + are omitted entirely, per the authoring rules in that file. */} + +## Readiness on-prem + +On-prem ships as versioned containers, so the question is which release you need rather than whether a combination is production-ready. + +| Interaction pattern | Model | On-prem | +|---|---|---| +| pre-recorded | Standard | Released | +| pre-recorded | Enhanced | Released | +| pre-recorded | Melia 1 | Released | +| streaming | Standard | Released | +| streaming | Enhanced | Released | + +:::note +Feature availability on-prem depends on the container release you are running. Check [Accessing images](/deployments/container/accessing-images) for the current releases, and the release notes for the change that introduced a given feature. +::: + +## Pre-recorded transcription features on-prem + +Pre-recorded transcription runs on the Standard, Enhanced, and Melia 1 models on-prem. + +### Language coverage and selection + +| Item | Standard | Enhanced | Melia 1 | +|---|---|---|---| +| 56 languages | ✓ | ✓ | ✓ | +| Mixed-language transcription | — | — | ✓ | +| Language hints | — | — | ✓ | +| Language labeling | — | — | ✓ | +| Transcription language packs (including bilingual) | ✓ | ✓ | — | +| Automatic language identification | ✓ | ✓ | — | + +### Output and formatting + +| Item | Standard | Enhanced | Melia 1 | +|---|---|---|---| +| Output locale | ✓ | ✓ | ✓ | +| Smart formatting | ✓ | ✓ | ✓ | +| Punctuation and casing | ✓ | ✓ | ✓ | +| Word-level timings | ✓ | ✓ | ✓ | +| Segment-level timings | ✓ | ✓ | ✓ | +| Confidence scores | ✓ | ✓ | — | + +### Transcript content and tagging + +| Item | Standard | Enhanced | Melia 1 | +|---|---|---|---| +| Custom dictionary | ✓ | ✓ | — | +| Medical domain | — | ✓ | — | +| Entity detection (basic, legacy) | ✓ | ✓ | — | +| Disfluency tagging | ✓ | ✓ | — | +| Profanity tagging | ✓ | ✓ | — | +| Text replacement (find and replace) | ✓ | ✓ | — | + +### Speakers and channels + +| Item | Standard | Enhanced | Melia 1 | +|---|---|---|---| +| Speaker diarization | ✓ | ✓ | ✓ | +| Channel diarization | ✓ | ✓ | ✓ | +| Speaker identification | ✓ | ✓ | — | + +### Audio input + +| Item | Standard | Enhanced | Melia 1 | +|---|---|---|---| +| Audio events | ✓ | ✓ | — | +| Audio filtering (volume filtering) | ✓ | ✓ | — | +| Fetch URL | ✓ | ✓ | ✓ | + +Audio events on-prem requires a GPU container. + +### Operational + +| Item | Standard | Enhanced | Melia 1 | +|---|---|---|---| +| Notifications | ✓ | ✓ | ✓ | + +### Add-ons + +| Item | Standard | Enhanced | Melia 1 | +|---|---|---|---| +| Translation | ✓ | ✓ | — | +| Sentiment | ✓ | ✓ | — | + +Translation on-prem runs in a separate inference container. See [Translation GPU inference container](/deployments/container/gpu-translation). + +## Streaming transcription features on-prem + +Streaming transcription runs on the Standard and Enhanced models on-prem. + +### Streaming language coverage and selection + +| Item | Standard | Enhanced | +|---|---|---| +| 56 languages | ✓ | ✓ | +| Transcription language packs (including bilingual) | ✓ | ✓ | + +### Streaming output and formatting + +| Item | Standard | Enhanced | +|---|---|---| +| Output locale | ✓ | ✓ | +| Smart formatting | ✓ | ✓ | +| Punctuation and casing | ✓ | ✓ | +| Word-level timings | ✓ | ✓ | +| Confidence scores | ✓ | ✓ | + +### Streaming transcript content and tagging + +| Item | Standard | Enhanced | +|---|---|---| +| Custom dictionary | ✓ | ✓ | +| Medical domain | — | ✓ | +| Entity detection (basic, legacy) | ✓ | ✓ | +| Disfluency tagging | ✓ | ✓ | +| Profanity tagging | ✓ | ✓ | +| Text replacement (find and replace) | ✓ | ✓ | + +### Streaming speakers and channels + +| Item | Standard | Enhanced | +|---|---|---| +| Speaker diarization | ✓ | ✓ | +| Channel diarization | ✓ | ✓ | +| Speaker identification | ✓ | ✓ | + +### Streaming audio input + +| Item | Standard | Enhanced | +|---|---|---| +| Audio events | ✓ | ✓ | +| Audio filtering (volume filtering) | ✓ | ✓ | + +### Responding + +| Item | Standard | Enhanced | +|---|---|---| +| Force end of utterance | ✓ | ✓ | +| Turn detection | ✓ | ✓ | +| Word-level partials | ✓ | ✓ | + +### Streaming add-ons + +| Item | Standard | Enhanced | +|---|---|---| +| Translation | ✓ | ✓ | diff --git a/docs/speech-to-text/batch/_category_.json b/docs/speech-to-text/batch/_category_.json deleted file mode 100644 index 09baf6c5..00000000 --- a/docs/speech-to-text/batch/_category_.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "label": "Batch Transcription", - "position": 1, - "link": null -} diff --git a/docs/speech-to-text/batch/batch-diarization.mdx b/docs/speech-to-text/batch/batch-diarization.mdx deleted file mode 100644 index fd93f9ae..00000000 --- a/docs/speech-to-text/batch/batch-diarization.mdx +++ /dev/null @@ -1,198 +0,0 @@ ---- -sidebar_label: 'Batch diarization' -description: "Learn how to use the Speechmatics API to separate speakers in Batch" -keywords: - [ - speechmatics, - batch, - diarization, - transcription, - speech recognition, - automatic speech recognition, - asr, - ] -sidebar_position: 2 ---- - -import DocCardList from '@theme/DocCardList'; -import { Card, DataList, Text } from '@radix-ui/themes'; - -# Batch diarization - -:::tip -To learn more about diarization as a feature, check out the [diarization](../features/diarization.mdx) page. -::: - -Batch diarization offers the following ways to separate speakers in audio: - -- [**Speaker diarization**](#speaker-diarization) — Identifies each speaker by their voice. - Useful when there are multiple speakers in the same audio stream. - -- [**Channel diarization**](#channel-diarization) — Transcribes each audio channel separately. - Useful when each speaker is recorded on their own channel. - -## Speaker diarization - -Speaker diarization aggregates all audio channels into a single stream for processing, and picks out different speakers based on acoustic matching. - -The feature is disabled by default. To enable speaker diarization, `diarization` must be set to `speaker` in the transcription config: - -```json -{ - "type": "transcription", - "transcription_config": { - "model": "enhanced", - "language": "en", - // highlight-start - "diarization": "speaker" - // highlight-end - } -} -``` - -When diarization is enabled, each `word` and `punctuation` object in the transcript includes a `speaker` property that identifies who spoke it. There are two types of labels: - -- `S#` – S stands for speaker, and `#` is a sequential number identifying each speaker. S1 appears first in the results, followed by S2, S3, and so on. -- `UU` – Used when the speaker cannot be identified or diarization is not applied, for example, if background noise is transcribed as speech but no speaker can be determined. - -```json - "results": [ - { - "alternatives": [ - { - "confidence": 0.93, - "content": "hello", - "language": "en", - // highlight-start - "speaker": "S1" - // highlight-end - } - ], - ... - }, - { - "alternatives": [ - { - "confidence": 1.0, - "content": "hi", - "language": "en", - // highlight-start - "speaker": "S2" - // highlight-end - } - ], - }] -``` - -## Channel diarization - -With channel diarization, each channel in your audio is transcribed on its own and then merged into a single transcript. This gives you perfect separation at the channel level and cleaner results when speakers overlap. - -Batch channel diarization supports up to 100 separate input files. - -To enable it, set the `diarization` property to `channel`. You can also add custom names for each channel with the `channel_diarization_labels` setting: - -```json -{ - "type": "transcription", - "transcription_config": { - "model": "enhanced", - "language": "en", - // highlight-start - "diarization": "channel", - "channel_diarization_labels": ["Agent", "Caller"] - // highlight-end - } -} -``` - -If no labels are provided, default names like `Channel 1` are used. The number of labels should match the number of channels in your audio; any extra labels are ignored. - -In the transcript, each word includes a `channel` property that indicates the speaker: - -```json -"results": [ - { - "type": "word", - "end_time": 1.8, - "start_time": 1.45, - "channel": "Agent", - "alternatives": [ - { - "language": "en", - "content": "Hello", - "confidence": 0.76 - } - ] - } -] -``` -## Configuration - -You can customize diarization to match your use case by adjusting settings for sensitivity, preferring the current speaker to reduce false switches, and controlling how punctuation influences accuracy. - -### Speaker sensitivity - -You can configure the sensitivity of speaker detection by using the `speaker_sensitivity` setting in the `speaker_diarization_config` section of the job config object as shown below: - -```json -{ - "type": "transcription", - "transcription_config": { - "model": "enhanced", - "language": "en", - // highlight-start - "diarization": "speaker", - "speaker_diarization_config": { - "speaker_sensitivity": 0.6 - } - // highlight-end - } -} -``` - -This takes a value between 0 and 1 (the default is 0.5). A higher sensitivity will -increase the likelihood of more unique speakers returning. - - -### Prefer current speaker - -You can reduce the likelihood of incorrectly switching between similar sounding speakers by setting the `prefer_current_speaker` flag in the `speaker_diarization_config`: - -```json -{ - "type": "transcription", - "transcription_config": { - "model": "enhanced", - "language": "en", - // highlight-start - "diarization": "speaker", - "speaker_diarization_config": { - "prefer_current_speaker": true - } - // highlight-end - } -} -``` -By default this flag is `false`. When this flag is set to `true`, the system will stay with the speaker of the previous word, if they closely match the speaker of the new word. - -This may result in some shorter speaker turn changes between similar speakers being missed. - -### Speaker diarization and punctuation - -Speaker diarization uses punctuation to improve accuracy. Small corrections are applied to speaker labels based on sentence boundaries. - -For example, if the system initially assigns 9 words in a sentence to S1 and 1 word to S2, the lone S2 word may be corrected to S1. - -This adjustment only works when punctuation is enabled. Disabling punctuation via the `permitted_marks` setting in `punctuation_overrides` can reduce diarization accuracy. - -Adjusting punctuation sensitivity can also affect how accurately speakers are identified. - -### Speaker change (legacy) - -The speaker change detection feature was removed in July 2024. The `speaker_change` and `channel_and_speaker_change` parameters are no longer supported. Use the [speaker diarization](#speaker-diarization) feature for speaker labeling. - -For API-related questions, contact [Support](https://support.speechmatics.com). - -## Considerations -- Enabling diarization for a file increases the amount of time taken to transcribe an audio file. In general, we expect the use of diarization to increase the overall processing time by 10-50%. diff --git a/docs/speech-to-text/batch/sidebar.ts b/docs/speech-to-text/batch/sidebar.ts deleted file mode 100644 index e07e0bf1..00000000 --- a/docs/speech-to-text/batch/sidebar.ts +++ /dev/null @@ -1,70 +0,0 @@ -export default { - type: "category", - label: "Batch transcription", - items: [ - { - type: "doc", - id: "speech-to-text/batch/quickstart", - }, - { - type: "doc", - id: "speech-to-text/batch/input", - }, - { - type: "doc", - id: "speech-to-text/batch/output", - }, - { - type: "doc", - id: "speech-to-text/batch/synchronous", - }, - { - type: "doc", - id: "speech-to-text/batch/limits", - }, - { - type: "doc", - id: "speech-to-text/batch/batch-diarization", - }, - { - type: "doc", - id: "speech-to-text/batch/speaker-identification", - }, - { - type: "category", - label: "Speech intelligence", - items: [ - { - type: "autogenerated", - dirName: "speech-to-text/batch/speech-intelligence", - }, - ], - }, - { - type: "doc", - id: "speech-to-text/batch/notifications", - }, - { - type: "doc", - id: "speech-to-text/batch/language-identification", - }, - { - type: "doc", - id: "speech-to-text/batch/srt-format", - }, - // TODO see about usage requests - // { - // type: "doc", - // id: "speech-to-text/batch/usage-requests", - // }, - { - type: "doc", - id: "speech-to-text/batch/troubleshooting", - }, - { - type: "link", - href: "/api-ref/batch/create-a-new-job", - label: "API reference", - }, - ], -} as const; diff --git a/docs/speech-to-text/batch/speaker-identification.mdx b/docs/speech-to-text/batch/speaker-identification.mdx deleted file mode 100644 index 544e22a0..00000000 --- a/docs/speech-to-text/batch/speaker-identification.mdx +++ /dev/null @@ -1,155 +0,0 @@ ---- -sidebar_label: 'Speaker identification' -description: "Learn how to use the Speechmatics API to identify speakers in Batch" -keywords: - [ - speechmatics, - batch, - diarization, - transcription, - speech recognition, - automatic speech recognition, - asr, - ] -sidebar_position: 2 ---- - -import DocCardList from '@theme/DocCardList'; -import { Card, DataList, Text } from '@radix-ui/themes'; - -# Batch speaker identification - -:::tip -For an overview of the feature, see the [speaker identification](/speech-to-text/features/speaker-identification) page. -::: - -## Enrollment - -To generate identifiers for a desired speaker, run a [speaker diarization](/speech-to-text/features/diarization#diarization-modes) enabled transcription on an audio sample where the speaker is ideally speaking alone. -You can request the identifiers back from the engine by setting the `get_speakers` flag in the transcription config: - -```json -{ - "type": "transcription", - "transcription_config": { - "model": "enhanced", - "language": "en", - "diarization": "speaker", - "speaker_diarization_config": { - // highlight-start - "get_speakers": true - // highlight-end - } - } -} -``` - -When the transcription is done, the speakers identifiers will be attached to the returned transcript: - -```json -{ - "results": [ - { - "alternatives": [ - { - "confidence": 0.93, - "content": "Hello", - "language": "en", - "speaker": "S1" - } - ], - ... - }, - { - "alternatives": [ - { - "confidence": 1.0, - "content": "Hi", - "language": "en", - "speaker": "S2" - } - ], - ... - }], - // highlight-start - "speakers": [ - { - "label": "S1", - "speaker_identifiers": [""] - }, - { - "label": "S2", - "speaker_identifiers": [""] - }] - // highlight-end -} -``` - -## Identification - -Once you have generated speaker identifiers, you can provide them in your next transcription job to identify and tag known speakers. This is done through the `speakers` option in the speaker diarization configuration. All other [speaker diarization options](/speech-to-text/batch/batch-diarization#configuration) remain supported. Notably, the `speakers_sensitivity` parameter can be used to adjust how strongly the system prefers enrolled speakers over detecting new generic ones, where lower values make it more likely to match existing enrolled speakers. - -An example configuration is shown below: - -```json -{ - "type": "transcription", - "transcription_config": { - "model": "enhanced", - "language": "en", - "diarization": "speaker", - "speaker_diarization_config": { - // highlight-start - "speakers": [ - {"label": "Alice", "speaker_identifiers": ["", ""]}, - {"label": "Bob", "speaker_identifiers": [""]} - ] - // highlight-end - } - } -} -``` - -With the config above, transcript segments should be tagged with `"Alice"` and `"Bob"` whenever these speakers are detected, whereas any other speakers should be tagged with the internal labels: - -```json -{ - "results": [ - { - "alternatives": [ - { - "confidence": 0.93, - "content": "Morning", - "language": "en", - // highlight-start - "speaker": "Alice" - // highlight-end - } - ], - ... - }, - { - "alternatives": [ - { - "confidence": 0.93, - "content": "Hi", - "language": "en", - "speaker": "S1" - } - ], - ... - }, - { - "alternatives": [ - { - "confidence": 1.0, - "content": "Morning", - "language": "en", - // highlight-start - "speaker": "Bob" - // highlight-end - } - ], - }] -} -``` diff --git a/docs/speech-to-text/batch/usage.mdx b/docs/speech-to-text/batch/usage.mdx deleted file mode 100644 index f2f451f8..00000000 --- a/docs/speech-to-text/batch/usage.mdx +++ /dev/null @@ -1,49 +0,0 @@ ---- -description: 'Learn how to get information about your API usage' -keywords: [speechmatics, usage, data, tracking, transcription, speech recognition, asr] ---- - -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; - -# Usage reporting - -This section describes how to make requests to Speechmatics Batch SaaS to understand your usage of the service. Usage is reported as the number of jobs successfully processed in a time period, along with the total duration of the corresponding audio files. - -As for other requests to Speechmatics Batch SaaS, usage is queried using an API key. Usage is returned for the account associated with the API key. When using the examples below, you should replace the API key shown with your own. - -All usage is reported on a Coordinated Universal Time (UTC) calendar day basis. Note that usage for the current day (in UTC) is **not** included in the results. - -Note that deleting a completed job will have no effect on reported usage. - -## Requesting usage - -The following request retrieves usage for all jobs from the beginning of the 1st January 2023 until the end of the 31st December 2023. More precisely, this includes all jobs completed successfully at or after 2023-01-01T00:00Z and before 2023-01-01T00:00Z. - -The `since` and `until` parameters are both optional: - -- If `since` is omitted, then usage is reported from the day on which the first job was successfully processed for the account. -- If `until` is omitted, then usage is reported until the end of the previous UTC day. - - - - -```bash -curl -L -X GET "https://eu1.asr.api.speechmatics.com/v2/usage?since=2023-01-01&until=2023-12-31" \ - -H "Authorization: Bearer $API_KEY" -``` - - - - -```powershell -curl -L -X GET "https://eu1.asr.api.speechmatics.com/v2/usage?since=2023-01-01&until=2023-12-31" ` - -H "Authorization: Bearer $API_KEY" -``` - - - - -## Response - -The response is a JSON object containing usage information. Please refer to the [API Reference](/api-ref/batch/get-usage-statistics) for full details. diff --git a/docs/speech-to-text/features/app-usage-tracking.mdx b/docs/speech-to-text/features/app-usage-tracking.mdx new file mode 100644 index 00000000..d8cbe250 --- /dev/null +++ b/docs/speech-to-text/features/app-usage-tracking.mdx @@ -0,0 +1,52 @@ +--- +title: App usage tracking +description: Attribute usage across applications by adding an app ID to your requests. +keywords: [app analytics, app usage, sm-app, usage tracking, attribution] +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +If your application lets users enter their own Speechmatics API key, app usage tracking gives you aggregated analytics about your users' usage levels, including the number of unique users, hours processed, and languages used. + +You add an app ID to each request with the `sm-app` query parameter, and Speechmatics aggregates usage against it. + +:::tip +Contact [Support](https://support.speechmatics.com) to enable this feature before using it. +::: + +## Availability + +App usage tracking is available for pre-recorded and streaming transcription with the Standard, Enhanced, and Melia 1 models. See [Feature availability](/speech-to-text/availability/). + +## Add an app ID to a request + + + + +Add the `sm-app` query parameter when starting a job: + +```bash +APP_ID="YourAppID" +API_KEY="YOUR_API_KEY" +PATH_TO_FILE="example.wav" + +# sm-app attributes this job's usage to your application +curl -L -X POST "https://eu1.asr.api.speechmatics.com/v2/jobs/?sm-app=${APP_ID}" \ + -H "Authorization: Bearer ${API_KEY}" \ + -F data_file=@${PATH_TO_FILE} \ + -F config='{"type": "transcription","transcription_config": { "model": "enhanced","language": "en" }}' +``` + + + + +Add the `sm-app` query parameter to the connection string when starting a session: + +```bash +# sm-app attributes this session's usage to your application +wss://eu.rt.speechmatics.com/v2?sm-app=YourAppID +``` + + + diff --git a/docs/speech-to-text/features/assets/melia-1-streaming-example.mjs b/docs/speech-to-text/features/assets/melia-1-streaming-example.mjs new file mode 100644 index 00000000..557273c9 --- /dev/null +++ b/docs/speech-to-text/features/assets/melia-1-streaming-example.mjs @@ -0,0 +1,59 @@ +import { spawn } from "node:child_process"; +import { createSpeechmaticsJWT } from "@speechmatics/auth"; +import { RealtimeClient } from "@speechmatics/real-time-client"; + +const apiKey = "YOUR_API_KEY_HERE"; // Set your Speechmatics API key here +const client = new RealtimeClient({ url: "wss://preview.rt.speechmatics.com/v2" }); + +const audio_format = { + type: "raw", + encoding: "pcm_s16le", + sample_rate: 44100, +}; + +async function transcribe() { + client.addEventListener("receiveMessage", ({ data }) => { + if (data.message === "AddTranscript") { + const transcript = data.metadata?.transcript; + if (transcript) console.log(`[Final]: ${transcript}`); + } else if (data.message === "Error") { + console.error(`Error [${data.type}]: ${data.reason}`); + process.exit(1); + } + }); + + const jwt = await createSpeechmaticsJWT({ type: "rt", apiKey, ttl: 60 }); + + await client.start(jwt, { + transcription_config: { + language: "multi", + // @ts-ignore: `melia-1` has not been added to the `Model` enum in the SDK + model: "melia-1", + }, + audio_format, + }); + + const recorder = spawn("sox", [ + "-d", // default audio device (mic) + "-q", // quiet + "-r", String(audio_format.sample_rate), // sample rate + "-e", "signed-integer", // match pcm_s16le + "-b", "16", // match pcm_s16le + "-c", "1", // mono + "-t", "raw", // raw PCM output + "-", // pipe to stdout + ]); + + recorder.stdout.on("data", (chunk) => client.sendAudio(chunk)); + recorder.stderr.on("data", (d) => console.error(`sox: ${d}`)); + + process.on("SIGINT", () => { + recorder.kill(); + client.stopRecognition({ noTimeout: true }); + }); +} + +transcribe().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/docs/speech-to-text/features/assets/melia-1-streaming-example.py b/docs/speech-to-text/features/assets/melia-1-streaming-example.py new file mode 100644 index 00000000..e935a55f --- /dev/null +++ b/docs/speech-to-text/features/assets/melia-1-streaming-example.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +import asyncio +from speechmatics.rt import ( + AudioEncoding, AudioFormat, AuthenticationError, + Microphone, ServerMessageType, TranscriptResult, + TranscriptionConfig, AsyncClient, +) + +API_KEY = "YOUR_API_KEY_HERE" # Replace with your actual API key + +# Set up config and format for transcription +audio_format = AudioFormat( + encoding=AudioEncoding.PCM_S16LE, + sample_rate=16000, + chunk_size=4096, +) +config = TranscriptionConfig( + model="melia-1", + language="multi", +) + +async def main(): + + # Set up microphone + mic = Microphone( + sample_rate=audio_format.sample_rate, + chunk_size=audio_format.chunk_size + ) + if not mic.start(): + print("Mic not started — please install PyAudio") + return + + try: + async with AsyncClient(api_key=API_KEY, url="wss://preview.rt.speechmatics.com/v2") as client: + # Handle ADD_TRANSCRIPT message + @client.on(ServerMessageType.ADD_TRANSCRIPT) + def handle_finals(msg): + if final := TranscriptResult.from_message(msg).metadata.transcript: + print(f"[Final]: {final}") + + try: + # Begin transcribing + await client.start_session( + transcription_config=config, + audio_format=audio_format + ) + while True: + await client.send_audio( + await mic.read( + chunk_size=audio_format.chunk_size + ) + ) + except KeyboardInterrupt: + pass + finally: + mic.stop() + + except AuthenticationError as e: + print(f"Auth error: {e}") + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/speech-to-text/features/audio-events.mdx b/docs/speech-to-text/features/audio-events.mdx index d7f1a3f5..3ae6b800 100644 --- a/docs/speech-to-text/features/audio-events.mdx +++ b/docs/speech-to-text/features/audio-events.mdx @@ -27,7 +27,7 @@ The Audio Events feature, available through our Automatic Speech Recognition (AS Enable Audio Events in your application for file processing scenarios by leveraging Speechmatics SaaS or On-Prem solutions. -If you're new to Speechmatics, start by exploring our guides on [Processing a File](/speech-to-text/batch/quickstart) or [Analyzing in Realtime](/speech-to-text/realtime/quickstart). To activate Audio Events, include the following configuration: +If you're new to Speechmatics, start by exploring our guides on [Processing a File](/speech-to-text/pre-recorded/quickstart) or [Analyzing in Realtime](/speech-to-text/streaming/quickstart). To activate Audio Events, include the following configuration: ```json { @@ -42,6 +42,10 @@ If you're new to Speechmatics, start by exploring our guides on [Processing a Fi } ``` +## Availability + +Audio events are available with the Standard and Enhanced models for pre-recorded and streaming transcription. They are not available for agent STT. On-prem requires a GPU container. See [Feature availability](/speech-to-text/availability/). + ## Example @@ -79,7 +83,7 @@ with BatchClient(settings) as client: -Python client example for detecting Audio Events in real-time, see [here](/speech-to-text/realtime/quickstart) for more examples of Realtime Transcription +Python client example for detecting Audio Events in real-time, see [here](/speech-to-text/streaming/quickstart) for more examples of Realtime Transcription ```python showLineNumbers import speechmatics.client @@ -142,7 +146,7 @@ The JSON output for batch processing will include the following information abou - `confidence`: A number indicating the confidence value of the event detected by the model. - `channel`: Only returned if Channel Diarization is enabled, this would indicate the channel in which the event was detected -The JSON output will also contain an `audio_event_summary` which will summarise all detected Audio Events highlighting the number of times and the total duration for which each category of Audio Event occurred. +The JSON output will also contain an `audio_event_summary` which will summarize all detected Audio Events highlighting the number of times and the total duration for which each category of Audio Event occurred. The Audio Event summary will also contain the summary of silence & speech events, with the total duration being calculated by adding duration of all the words spoken. @@ -248,7 +252,7 @@ Ending message example for the music event: ``` :::info -The `channel` fields are only returned when [channel](../../speech-to-text/realtime/realtime-diarization#channel-diarization) or [`channel and speaker`](../../speech-to-text/realtime/realtime-diarization#channel-and-speaker-diarization) diarization is enabled. +The `channel` fields are only returned when [channel](../../speech-to-text/features/channel-diarization) or [`channel and speaker`](../../speech-to-text/features/channel-diarization#channel-and-speaker-diarization) diarization is enabled. These fields indicate which channel the audio event comes from. ::: diff --git a/docs/speech-to-text/features/audio-filtering.mdx b/docs/speech-to-text/features/audio-filtering.mdx index 3374bd00..eb0c1dca 100644 --- a/docs/speech-to-text/features/audio-filtering.mdx +++ b/docs/speech-to-text/features/audio-filtering.mdx @@ -23,7 +23,11 @@ Audio Filtering pre-processes input audio to remove low-volume background speech This can be useful, for example, in a call center to avoid transcribing other agents' speech from the background. ::: -If you're new to Speechmatics, start by exploring our guides on [Transcribing a File](/speech-to-text/batch/quickstart) or [Transcribing in Realtime](/speech-to-text/realtime/quickstart). +If you're new to Speechmatics, start by exploring our guides on [Transcribing a File](/speech-to-text/pre-recorded/quickstart) or [Transcribing in Realtime](/speech-to-text/streaming/quickstart). + +## Availability + +Audio filtering is available with the Standard and Enhanced models for pre-recorded and streaming transcription, and with Linden 1 for agent STT. See [Feature availability](/speech-to-text/availability/). ## Example @@ -47,7 +51,7 @@ This will avoid processing any audio which is below the `3.4` volume threshold. ## Volume labelling -If Audio Filtering is configured, words will be labelled with their volume like this (range for `volume_threshold` is `0-100`): +If Audio Filtering is configured, words will be labeled with their volume like this (range for `volume_threshold` is `0-100`): ```json { diff --git a/docs/speech-to-text/features/channel-diarization.mdx b/docs/speech-to-text/features/channel-diarization.mdx new file mode 100644 index 00000000..260074a6 --- /dev/null +++ b/docs/speech-to-text/features/channel-diarization.mdx @@ -0,0 +1,176 @@ +--- +title: Channel diarization +description: Transcribe each audio channel separately, with optional per-channel labels. +keywords: + [ + speechmatics, + diarization, + channel diarization, + channels, + transcription, + speech recognition, + asr, + ] +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +Channel diarization transcribes each channel in your audio separately and merges the results into a single transcript. This gives perfect speaker separation at the channel level and cleaner results when speakers overlap. + +Use it when each speaker is recorded on their own channel, such as a call recording with the agent and caller on separate channels. To separate speakers who share a channel, use [speaker diarization](/speech-to-text/features/speaker-diarization) instead. + +## Availability + +Channel diarization is available for pre-recorded and streaming transcription with the Standard, Enhanced, and Melia 1 models. It is not available for agent STT. See [Feature availability](/speech-to-text/availability/). + +## Enable channel diarization + +Set `diarization` to `channel`, and optionally name each channel with `channel_diarization_labels`. + + + + +```json +{ + "type": "transcription", + "transcription_config": { + "model": "enhanced", + "language": "en", + // highlight-start + "diarization": "channel", + "channel_diarization_labels": ["Agent", "Caller"] + // highlight-end + } +} +``` + +If no labels are provided, default names such as `Channel 1` are used. The number of labels should match the number of channels in your audio; any extra labels are ignored. + +Pre-recorded channel diarization supports up to 100 separate input files. + + + + +Provide labels for each channel in the transcription config of the `StartRecognition` message: + +```json +{ + "type": "transcription", + "transcription_config": { + "model": "enhanced", + "language": "en", + // highlight-start + "diarization": "channel", + "channel_diarization_labels": ["New_York", "Shanghai"] + // highlight-end + } +} +``` + +The `RecognitionStarted` message lists all the channels you requested: + +```json +{ + "message": "RecognitionStarted", + ... + // highlight-start + "channels": ["New_York", "Shanghai"] + // highlight-end +} +``` + +Streaming requires you to send audio per channel and to close channels explicitly. See [Audio channels](/speech-to-text/streaming/channels). + + + + +## Transcript output + +For pre-recorded transcription, each word includes a `channel` property identifying which channel it came from: + +```json +"results": [ + { + "type": "word", + "end_time": 1.8, + "start_time": 1.45, + "channel": "Agent", + "alternatives": [ + { + "language": "en", + "content": "Hello", + "confidence": 0.76 + } + ] + } +] +``` + +For streaming, transcripts are returned independently for each channel, with the `channel` property on the message: + +```json +{ + "message": "AddTranscript", + // highlight-start + "channel": "New_York", + // highlight-end + ... + "results": [ + { + "type": "word", + "start_time": 1.45, + "end_time": 1.8, + "alternatives": [{ + "language": "en", + "content": "Hello,", + "confidence": 0.98, + }] + }, + ] +} +``` + +:::warning +In streaming, the `channel` property is returned for `AddTranscript` and `AddPartialTranscript` messages only. The [translation](/speech-to-text/add-ons/translation) add-on does not currently include this property. To request it, contact [Support](https://support.speechmatics.com). +::: + +## Channel and speaker diarization {#channel-and-speaker-diarization} + +Channel and speaker diarization combines both modes, splitting transcripts per channel while also separating individual speakers within each channel. Use it when multiple speakers are present across multiple channels. + +This mode is available for streaming transcription only. Set `diarization` to `channel_and_speaker`, and send audio per channel as described in [Audio channels](/speech-to-text/streaming/channels). + +Transcripts are returned in the same way as channel diarization, but with individual speakers identified: + +```json +{ + "message": "AddTranscript", + // highlight-start + "channel": "New_York", + // highlight-end + "results": [ + { + "alternatives": [{ + "content": "Hello", + "confidence": 0.98, + // highlight-start + "speaker": "S1" + // highlight-end + }] + }, + ... + { + "alternatives": [{ + "content": "Hi", + "confidence": 0.98, + // highlight-start + "speaker": "S2" + // highlight-end + }] + }, + ] +} +``` + +With `channel_and_speaker` diarization, speaker labeling is specific to each channel even when the labels are the same. S1 on channel 1 is not necessarily the same person as S1 on channel 2. diff --git a/docs/speech-to-text/features/custom-dictionary.mdx b/docs/speech-to-text/features/custom-dictionary.mdx index da8b24d8..5f0a602c 100644 --- a/docs/speech-to-text/features/custom-dictionary.mdx +++ b/docs/speech-to-text/features/custom-dictionary.mdx @@ -6,7 +6,7 @@ sidebar_position: 3 # Custom dictionary -The Custom dictionary feature allows a list of custom words to be added for each transcription job. This helps when a specific word is not recognised during transcription. It could be that it's not in the vocabulary for that language, for example a company or person's name. Adding custom words can improve the likelihood they will be output. +The Custom dictionary feature allows a list of custom words to be added for each transcription job. This helps when a specific word is not recognized during transcription. It could be that it's not in the vocabulary for that language, for example a company or person's name. Adding custom words can improve the likelihood they will be output. The `sounds_like` feature is an extension to this to allow alternative pronunciations to be specified to aid recognition when the pronunciation is not obvious. @@ -47,8 +47,12 @@ Prior to using this feature, consider the following: In the above example, the words _gnocchi_ and _CEO_ have pronunciations applied to them; the phrase _financial crisis_ does not require a pronunciation. The `content` property represents how you want the word to be output in the transcript. +## Availability + +The custom dictionary is available with the Standard and Enhanced models for pre-recorded and streaming transcription, and with Linden 1 for agent STT. See [Feature availability](/speech-to-text/availability/). + ## Custom dictionary caching -The Speechmatics Realtime SaaS caches custom dictionaries to reduce session initialisation times. +The Speechmatics Realtime SaaS caches custom dictionaries to reduce session initialization times. You will see improvements when reusing an identical custom dictionary from the second time onwards. Cache entries expire when they are not used for 24 hours. diff --git a/docs/speech-to-text/features/diarization.mdx b/docs/speech-to-text/features/diarization.mdx deleted file mode 100644 index 5b876b7d..00000000 --- a/docs/speech-to-text/features/diarization.mdx +++ /dev/null @@ -1,52 +0,0 @@ ---- -description: "Learn how Speechmatics diarization separates speakers in audio" -keywords: - [ - speechmatics, - features, - diarization, - transcription, - speech recognition, - automatic speech recognition, - asr, - ] -sidebar_position: 2 ---- - -import DocCardList from '@theme/DocCardList'; -import { Card, DataList, Text } from '@radix-ui/themes'; - -# Diarization - -Use Speechmatics' **Diarization** to separate a transcript into distinct speakers or channels, so you can clearly see who said what. - -It’s especially useful in conversations, meetings, interviews, or multi-channel recordings where keeping track of each voice matters. By labeling speakers or channels, diarization makes transcripts easier to read, analyze, and share. - -## Use cases - -- **Call centers** – Identify agents and customers for training, compliance, and quality assurance. -- **Video conferences** – Track who said what in multi-participant discussions. -- **Medical consultations** – Capture conversations between doctors and patients with clear speaker labels. -- **Media production** – Make multi-speaker audio easier to edit, search, and annotate. - -## Diarization modes - -Speechmatics offer the following diarization modes: - -- **Speaker diarization** — Identifies each speaker by their voice. - Useful when there are multiple speakers in the same audio recording. - -- **Channel diarization** — Transcribes each audio channel separately. - Useful when each speaker is recorded on a separate audio channel. - -- **Channel & speaker diarization** — Transcribes each channel separately and also identifies individual speakers within each channel. - Useful when multiple speakers are recorded across multiple channels. Available with Realtime diarization only. - - -These modes can be used with our Realtime or Batch APIs: - -**[Realtime diarization](../realtime/realtime-diarization.mdx)**
- Used for live, streaming audio. Ideal for scenarios like video conferencing, real-time captions, and conversational AI. - -**[Batch diarization](../batch/batch-diarization.mdx)**
- Used for pre-recorded audio files. Great for scenarios like call recordings, podcasts, and long-form interviews. \ No newline at end of file diff --git a/docs/speech-to-text/features/disfluencies-and-profanities.mdx b/docs/speech-to-text/features/disfluencies-and-profanities.mdx new file mode 100644 index 00000000..d02d8a77 --- /dev/null +++ b/docs/speech-to-text/features/disfluencies-and-profanities.mdx @@ -0,0 +1,167 @@ +--- +title: Disfluency and profanity tagging +sidebar_label: Disfluencies and profanities +description: Tag hesitations and profanity in the transcript, or remove disfluencies automatically. +keywords: [disfluency, disfluencies, profanity, profanities, hesitation, tags, transcript filtering] +--- + +import CodeBlock from '@theme/CodeBlock' + +Speechmatics tags hesitation sounds and profanity in the transcript so you can display, filter, or remove them. Both appear in the `tags` array on the word in the transcript output. + +## Availability + +Disfluency tagging and profanity tagging are available with the Standard and Enhanced models, for pre-recorded and streaming transcription. See [Feature availability](/speech-to-text/availability/). + +Language coverage differs between the two features and is listed in each section below. + +## Profanities {#profanities} + +You can tag profanities to identify or censor offensive language in your workflow. Profanity tagging is available for: + +- English (`en`) +- Italian (`it`) +- Spanish (`es`) + +Tagged profanities appear in the transcript with the `profanity` tag: + +```json +"results": [ + { + "alternatives": [ + { + "confidence": 1.0, + "content": "$PROFANITY", + "language": "en", + "tags": [ + "profanity" + ] + } + ], + "end_time": 18.03, + "start_time": 17.61, + "type": "word" + } +] +``` + +For other languages, consider using [text replacement](/speech-to-text/features/text-replacement) to identify profanities. + +## Disfluencies {#disfluencies} + +Disfluencies are hesitation sounds like "um", "uh", and "hmm". Speechmatics automatically tags them with `disfluency` in the transcript output: + +```json +"results": [ + { + "alternatives": [ + { + "confidence": 1.0, + "content": "hmm", + "language": "en", + "tags": [ + "disfluency" + ] + } + ], + "end_time": 18.03, + "start_time": 17.61, + "type": "word" + } +] +``` + +
+ Full list of tagged English disfluencies + +{`huh +aha +ah +aw +eh +err +hmm +mm +um +uh +uh-oh +uh-huh +uh-uh +mhm +a-ha +aah +aahh +aaw +ah-ha +ahaa +ahh +ahha +aww +eeh +erm +hhm +hhmm +hm +huh-uh +m-hm +uggh +ugh +ughh +uhh +uhhm +uhm +uhmm +umm +uuh +uuhh +uum`} + +
+ +### Supported languages for disfluencies {#supported-languages-for-disfluencies} + +Disfluency tagging and removal are available for the following languages. Each language has its own set of hesitation sounds; the expandable list above covers English. + +- Arabic (`ar`) +- Danish (`da`) +- Dutch (`nl`) +- English (`en`) +- French (`fr`) +- German (`de`) +- Greek (`el`) +- Hebrew (`he`) +- Hindi (`hi`) +- Hungarian (`hu`) +- Italian (`it`) +- Japanese (`ja`) +- Mandarin (`cmn`) +- Polish (`pl`) +- Portuguese (`pt`) +- Russian (`ru`) +- Spanish (`es`) + +Coverage of hesitation sounds varies by language. If you rely on disfluency removal for a specific language, test it with representative audio rather than assuming full coverage. + +### Removing disfluencies {#removing-disfluencies} + +You can automatically remove disfluencies from your transcript: + +```json +"transcription_config": { + "model": "enhanced", + "language": "en", + "transcript_filtering_config": { + "remove_disfluencies": true + } +} +``` + +This simplifies client-side processing by removing hesitation sounds and properly adjusting capitalization and spacing. For example: + +Without disfluency removal: +> Um, what would you like, hmm? + +With disfluency removal: +> What would you like? + +This feature is available for the [supported languages](#supported-languages-for-disfluencies). The default setting is `"remove_disfluencies": false`. diff --git a/docs/speech-to-text/features/entity-detection.mdx b/docs/speech-to-text/features/entity-detection.mdx new file mode 100644 index 00000000..93fe4dc4 --- /dev/null +++ b/docs/speech-to-text/features/entity-detection.mdx @@ -0,0 +1,127 @@ +--- +title: Entity detection +description: Expose the class, spoken form, and written form of each detected entity in JSON output. +keywords: [entity formatting, entity detection, entity classes, spoken form, written form] +--- + +[Smart formatting](/speech-to-text/features/formatting#smart-formatting) converts spoken values into their written form by default. Entity detection additionally exposes the structure behind each conversion: which class of entity was detected, and the individual words it was made from as spoken and as written. + +## Availability + +Entity detection is available with the Standard and Enhanced models for pre-recorded and streaming transcription, and with Linden 1 for agent STT. See [Feature availability](/speech-to-text/availability/). + +Entity metadata appears in JSON output only. SRT and TXT output are unchanged. + +## Enable entity detection + +Add `enable_entities` to your configuration: + +```json +{ + "type": "transcription", + "transcription_config": { + "model": "enhanced", + "language": "en", + "enable_entities": true + } +} +``` + +By default, `enable_entities` is `false`. When it is `false`, the words appear individually in the output. + +## Entity output + +With `enable_entities` set to `true`, the JSON output includes: + +- A `type` field with the value `entity` for formatted numeric entities +- The full written form in the `content` section, including any spaces or symbols +- An `entity_class` field describing how the entity was formatted +- Start and end times spanning all words in the entity +- Two additional representations: `spoken_form`, the original words as spoken with individual timing and confidence, and `written_form`, the formatted words separated individually + +This is a transcript with `enable_entities` set to `true`: + +```json +{ + "results": [ + { + "alternatives": [ + { + "confidence": 0.99, + "content": "17th of January 2022", + "language": "en", + "speaker": "UU" + } + ], + "end_time": 3.14, + "entity_class": "date", + "spoken_form": [ + { + "alternatives": [ + { + "confidence": 1.0, + "content": "seventeenth", + "language": "en", + "speaker": "UU" + } + ], + "end_time": 1.41, + "start_time": 0.72, + "type": "word" + } + // Additional spoken words omitted for brevity + ], + "start_time": 0.72, + "type": "entity", + "written_form": [ + { + "alternatives": [ + { + "confidence": 0.99, + "content": "17th", + "language": "en", + "speaker": "UU" + } + ], + "end_time": 1.33, + "start_time": 0.72, + "type": "word" + } + // Additional written words omitted for brevity + ] + } + ] +} +``` + +## Entity classes + +Formatting rules are applied based on the class of entity detected. + +Numbers and quantities: + +| Entity class | Description | Spoken example | Written example | +|---|---|---|---| +| cardinal | Whole numbers (in English, numbers ≤10 remain as words) | "nineteen" | 19 | +| decimal | Numbers with decimal point | "eighteen point one two" | 18.12 | +| fraction | Fractions (complex ones use n/d format) | "three sixteenths" | 3/16 | +| ordinal | Position numbers with suffix | "forty second" | 42nd | +| money | Currency values with symbol | "twenty dollars" | $20 | +| percentage | Percentages with % symbol | "two hundred percent" | 200% | +| measurement | Units with abbreviations | "ten kilometers per second" | 10 km/s | + +Dates, identifiers, and contact details: + +| Entity class | Description | Spoken example | Written example | +|---|---|---|---| +| date | Calendar dates and years | "fifteenth of January twenty twenty two" | 15th of January 2022 | +| time | Clock times with separators | "eleven forty a m" | 11:40 a.m. | +| span | Ranges (x to y format) | "one hundred to two hundred million pounds" | 100 to £200 million | +| alphanum | Alphanumeric sequences (3+ characters) | "a z triple seven five four" | AZ77754 | +| credit card | Payment card number sequences | "one one one one..." | 1111 2222 3333 4444 | +| telephone | Phone number formatting | "five five five..." | (555) 429-2228 | +| electronic | Email and web addresses | "bob at speechmatics dot com" | bob@speechmatics.com | + +:::warning +Entity classes are chosen based on context, so occasionally a value might be classified differently than expected. For example, "2001" could be a `cardinal` number or a `date`. +::: diff --git a/docs/speech-to-text/features/feature-discovery.mdx b/docs/speech-to-text/features/feature-discovery.mdx index 152d8159..12b0835e 100644 --- a/docs/speech-to-text/features/feature-discovery.mdx +++ b/docs/speech-to-text/features/feature-discovery.mdx @@ -18,11 +18,11 @@ curl "https://eu1.asr.api.speechmatics.com/v1/discovery/features" The feature discovery endpoint will include an object with the following properties: - `metadata` - - `language_pack_info` - For each of our [supported languages](/speech-to-text/languages), give the full name of the language, as well as any [Domain Language Optimizations](/speech-to-text/languages#bilingual-and-multi-language-packs) or [Output Locales](/speech-to-text/formatting#output-locale) + - `language_pack_info` - For each of our [supported languages](/speech-to-text/languages), give the full name of the language, as well as any [Domain Language Optimizations](/speech-to-text/languages#bilingual-and-multi-language-packs) or [Output Locales](/speech-to-text/features/formatting#output-locale) - `batch` - Capabilities relating to our Batch API - `transcription` - Capabilities relating to transcription - `languages` - Includes a list of supported ISO language codes - - `locales` - Includes any languages with a supported [Output Locale](/speech-to-text/formatting#output-locale) + - `locales` - Includes any languages with a supported [Output Locale](/speech-to-text/features/formatting#output-locale) - `domains` - Includes any languages with a supported [Domain Language Optimizations](/speech-to-text/languages#bilingual-and-multi-language-packs) - - `translation` - Includes all [supported translation pairs](/speech-to-text/features/translation#languages) - - `languageid` - List of languages supported by [Language Identification](/speech-to-text/batch/language-identification) + - `translation` - Includes all [supported translation pairs](/speech-to-text/add-ons/translation#languages) + - `languageid` - List of languages supported by [Language Identification](/speech-to-text/pre-recorded/language-identification) diff --git a/docs/speech-to-text/features/formatting.mdx b/docs/speech-to-text/features/formatting.mdx new file mode 100644 index 00000000..ae88cad2 --- /dev/null +++ b/docs/speech-to-text/features/formatting.mdx @@ -0,0 +1,133 @@ +--- +title: Formatting and punctuation +sidebar_label: Formatting +description: Control output spelling, number and date formatting, and which punctuation marks appear. +keywords: + [ + numeral formatting, + output formatting, + punctuation, + smart formatting, + transcript format, + transcription + ] +--- + +Three features shape how the transcript text is written: the output locale standardizes regional spelling, smart formatting converts spoken values into their written form, and punctuation controls which marks appear. + +## Availability + +Output locale, smart formatting, and punctuation and casing are available for every interaction pattern and every model. See [Feature availability](/speech-to-text/availability/). + +## Output locale {#output-locale} + +Some languages have multiple spelling conventions that vary by region. To ensure consistent spelling throughout your transcript, specify an output locale: + +```json +{ + "type": "transcription", + "transcription_config": { + "model": "enhanced", + "language": "en", + "output_locale": "en-GB" + } +} +``` + +Available English locales: +- British English (`en-GB`) +- US English (`en-US`) +- Australian English (`en-AU`) + +Available Mandarin locales: +- Simplified Mandarin (`cmn-Hans`, default) +- Traditional Mandarin (`cmn-Hant`) + +:::tip +Recommended for English transcription. Without a specified locale, spelling may be inconsistent within the same transcript. +::: + +## Smart formatting {#smart-formatting} + +Smart formatting converts spoken numbers, dates, currencies, and other entities into properly formatted text. This makes transcripts more readable without losing timing information. + +An *entity* is a spoken value that has a conventional written form, such as a number, date, currency, time, or measurement. Speechmatics detects each entity and converts it from the words as spoken into its written form. For example, the spoken words "nineteen ninety nine" become "1999" in the output. + +Smart formatting is applied by default. To also expose the structure of each entity in the JSON output — its class, and the individual spoken and written words it is made from — see [Entity detection](/speech-to-text/features/entity-detection). + +### Languages for smart formatting + +Each language follows its own conventions for thousand separators, decimal separators, and currency symbol position. For example: + +- **English**: commas for thousands (20,000), decimal points (10.5), currency symbols before values ($10) +- **German**: periods for thousands (20.000), commas for decimals (10,5), currency symbols after values with a non-breaking space (10 $) +- **French**: non-breaking spaces for thousands (20 000), commas for decimals (10,5), currency symbols after values with a non-breaking space (10 $) + +Smart formatting has had dedicated work for consistent results in these languages: + +- Cantonese +- Dutch +- English +- French +- German +- Hindi +- Italian +- Japanese +- Mandarin (Simplified and Traditional) +- Mandarin & English (bilingual) +- Mandarin Malay Tamil & English (multilingual) +- Norwegian +- Portuguese +- Russian +- Spanish +- Swedish +- Tamil & English (bilingual) + +Other languages still format numbers and entities on a best-effort basis through the model, with variable results. If you rely on formatting for a language that is not listed, test it with representative audio rather than assuming full coverage. + +Formatting coverage is not reported by [feature discovery](/speech-to-text/features/feature-discovery), which covers transcription, translation, and language identification. This page is the reference for formatting language support. + +## Punctuation {#punctuation} + +All Speechmatics language packs support punctuation to improve transcript readability. Each language supports specific punctuation marks: + +| Language | Supported marks | End-of-sentence marks | Notes | +|---|---|---|---| +| Cantonese, Mandarin | , 。 ? ! 、 | 。 ? ! | Full-width punctuation | +| Japanese | 。 、 | 。 | Full-width punctuation | +| Hindi | । ? , ! | । ? ! | | +| All other languages | . , ! ? | . ! ? | | + +### Configure punctuation marks + +Control which punctuation marks appear in your transcripts using the `punctuation_overrides` setting: + +```json +"transcription_config": { + "model": "enhanced", + "language": "en", + // highlight-start + "punctuation_overrides": { + "permitted_marks": [".", ","], + "sensitivity": 0.4 + } + // highlight-end +} +``` + +This configuration allows only periods and commas, with no question or exclamation marks, and sets punctuation sensitivity to 0.4, lower than the default 0.5. + +To select all marks, set `"permitted_marks": ["all"]`. If you use an empty list, no punctuation marks appear in the output. + +The `sensitivity` parameter accepts values from 0 to 1. Higher values produce more punctuation in the output. + +:::warning +Disabling punctuation may slightly reduce speaker diarization accuracy. See [speaker diarization](/speech-to-text/features/speaker-diarization) for details. +::: + +## Next steps + +- [Entity detection](/speech-to-text/features/entity-detection): expose the class, spoken form, and written form of each entity. +- [Disfluency and profanity tagging](/speech-to-text/features/disfluencies-and-profanities): tag or remove hesitations and profanity. +- [Text replacement](/speech-to-text/features/text-replacement): substitute words or patterns in the transcript. +- [Custom dictionary](/speech-to-text/features/custom-dictionary): improve recognition of specific words and phrases. diff --git a/docs/speech-to-text/features/medical-domain.mdx b/docs/speech-to-text/features/medical-domain.mdx new file mode 100644 index 00000000..9cdb5fdb --- /dev/null +++ b/docs/speech-to-text/features/medical-domain.mdx @@ -0,0 +1,47 @@ +--- +title: Medical domain +description: Tune recognition for healthcare audio by setting the medical domain. +keywords: [medical, healthcare, domain, clinical, dictation, scribe] +--- + +The medical domain tunes recognition for healthcare audio such as ambient scribes and dictation tools. It improves recognition of medical terminology including procedures, medications, conditions, and anatomy, and is kept up to date using officially maintained data sources. + +The medical domain is a feature you configure on a request, not a separate model. + +## Availability + +The medical domain is available with the Enhanced model for pre-recorded and streaming transcription, and with Linden 1 for agent STT. It is not offered with Standard. For the full matrix, see [Feature availability](/speech-to-text/availability/). + +## Set the medical domain + +Set the `domain` property to `medical`: + +```json +{ + "type": "transcription", + "transcription_config": { + "model": "enhanced", + "language": "en", + // highlight-start + "domain": "medical" + // highlight-end + } +} +``` + +## Languages with medical tuning + +The medical domain is tuned for the following languages, for both pre-recorded and streaming transcription: + +- Arabic English +- Danish +- Dutch +- English +- Finnish +- French +- German +- Norwegian +- Spanish +- Swedish + +For any other language, Enhanced still delivers high accuracy on healthcare audio without the medical domain. To ask about a language that is not listed, [contact us](https://www.speechmatics.com/speak-to-sales). diff --git a/docs/speech-to-text/features/mixed-language-transcription.mdx b/docs/speech-to-text/features/mixed-language-transcription.mdx new file mode 100644 index 00000000..d3801e30 --- /dev/null +++ b/docs/speech-to-text/features/mixed-language-transcription.mdx @@ -0,0 +1,216 @@ +--- +title: Mixed-language transcription +description: Transcribe audio that switches language mid-conversation, and read the per-word language labels. +keywords: [multilingual, mixed language, code switching, melia, language hints, language labeling] +--- + +import CodeBlock from '@theme/CodeBlock'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +import meliaStreamingPython from "./assets/melia-1-streaming-example.py?raw"; +import meliaStreamingJs from "./assets/melia-1-streaming-example.mjs?raw"; + +Mixed-language transcription uses one unified multilingual model to identify each language in the audio and switch between languages mid-sentence, returning a single continuous transcript. You do not select a language. + +This is a different capability from a [bilingual language pack](/speech-to-text/languages/#bilingual-and-multi-language-packs), where you choose a fixed set of languages in advance. + +## Availability + +Mixed-language transcription requires the Melia 1 model. It is available for pre-recorded transcription, and for streaming transcription in Preview. See [Feature availability](/speech-to-text/availability/). + +:::info Melia 1 for streaming is in Preview +Streaming with Melia 1 is available on SaaS on Cloud for evaluation and feedback. It is not production-ready and not ready to scale. +::: + +## Enable mixed-language transcription + +Set `model` to `melia-1` and `language` to `multi`. + + + + +```json +{ + "type": "transcription", + "transcription_config": { + // highlight-start + "model": "melia-1", + "language": "multi" + // highlight-end + } +} +``` + + + + +Send the model and language in your `StartRecognition` message: + +```json +{ + "message": "StartRecognition", + "transcription_config": { + // highlight-start + "language": "multi", + "model": "melia-1" + // highlight-end + } +} +``` + +Streaming with Melia 1 runs on a dedicated Preview endpoint, separate from the production Realtime endpoints: + +| Region | Endpoint | +|---|---| +| Preview (EU and US) | `wss://preview.rt.speechmatics.com/v2` | + +The Preview is served from France for the EU and Oregon for the US, on SaaS on Cloud. Your existing API keys work against it, and so do `rt` [temporary keys](/get-started/authentication#temporary-keys). The message schema matches the [Realtime WebSocket API](/api-ref/realtime-transcription-websocket), with the additions on this page. + +:::warning +Send only the configuration that the Preview supports. Including other Realtime settings, such as `max_delay`, causes the session to fail. Final transcripts arrive in about four seconds on average, and the interval varies. +::: + + + + +:::warning +Melia 1 does not support the `auto` language value, which returns an error. Set `language` to `multi`. +::: + +Melia 1 has no language pack selection. For the model itself, see [Models](/speech-to-text/models#melia-1). + +## Language hints {#language-hints} + +Melia 1 detects every language it hears automatically, so language hints are optional. Hints tell the model which languages to expect in the audio, biasing detection toward them. They are most useful for short clips, audio with heavy accents, or recordings where two languages sound similar, where they make language labeling more reliable. + +Language hints are available for pre-recorded transcription. + +Provide hints as a list of [supported languages](/speech-to-text/languages/transcription) to guide detection without restricting it. This config hints that the audio contains English and Arabic: + +```json +{ + "type": "transcription", + "transcription_config": { + "model": "melia-1", + "language": "multi", + // highlight-start + "language_hints": ["en", "ar"] + // highlight-end + } +} +``` + +The model can still detect and label a language you did not hint, and it labels only the languages it actually hears. + +## Per-word language labels {#multilingual-transcript-output} + +For a Melia 1 job, the `language` property on each word reflects the language detected for that word, so it can change across the transcript. For Enhanced and Standard jobs, which transcribe one selected language, the same language is reported for every word. + +This example shows two words in different languages within one transcript: + +```json +{ + "results": [ + { + "alternatives": [ + { "content": "Hello", "confidence": 0.98, "language": "en" } + ], + "start_time": 0.20, + "end_time": 0.52, + "type": "word" + }, + { + "alternatives": [ + { "content": "مرحبا", "confidence": 0.95, "language": "ar" } + ], + "start_time": 0.60, + "end_time": 1.04, + "type": "word" + } + ] +} +``` + +For multilingual transcripts, `language_pack_info` reports the word delimiter and writing direction per language rather than for a single language pack: + +```json +{ + "metadata": { + "language_pack_info": { + "per_language_word_delimiters": { + "en": " ", + "ar": " " + }, + "per_language_writing_direction": { + "en": "left-to-right", + "ar": "right-to-left" + } + } + } +} +``` + +`per_language_word_delimiters` gives the word delimiter for each language in the transcript, and `per_language_writing_direction` gives its writing direction. + +### The LanguageInfo message {#language-info} + +In a streaming session, the server sends a `LanguageInfo` message the first time it detects a new language code in the audio. It arrives once per language code, immediately before that language first appears in an `AddPartialTranscript` or `AddTranscript` message. + +Use it to set up how you process and display the transcript for that language, such as word spacing and text direction for right-to-left scripts. + +```json +{ + "message": "LanguageInfo", + "language": "ar", + "word_delimiter": " ", + "writing_direction": "right-to-left", + "partial": true +} +``` + +| Field | Type | Description | +|---|---|---| +| `language` | String | The detected language code. | +| `word_delimiter` | String | The delimiter to use between words when you reconstruct the transcript for this language. | +| `writing_direction` | String | Either `left-to-right` or `right-to-left`. | +| `partial` | Boolean | Optional. Present and `true` when a partial transcript triggered the message. | + +Handling `LanguageInfo` is not yet available in the SDKs. Read it from the WebSocket connection directly. + +## Transcribe live audio with Melia 1 + +These examples transcribe your microphone against the Preview endpoint and print each final transcript. Replace `YOUR_API_KEY_HERE` with your API key. + + + + +Install the SDK and an audio input library: + +```bash +pip install speechmatics-rt pyaudio +``` + +{meliaStreamingPython} + +Press `Ctrl+C` to stop. + + + + +Install the SDK, and `sox` for microphone input: + +```bash +npm install @speechmatics/real-time-client @speechmatics/auth +``` + +Install `sox` with `brew install sox` on macOS, or `apt install sox` on Linux. + +{meliaStreamingJs} + +Press `Ctrl+C` to stop. + + + + +The JavaScript SDK does not list `melia-1` in its `Model` type, so the example suppresses the type error on that line. diff --git a/docs/speech-to-text/features/speaker-diarization.mdx b/docs/speech-to-text/features/speaker-diarization.mdx new file mode 100644 index 00000000..412da5f7 --- /dev/null +++ b/docs/speech-to-text/features/speaker-diarization.mdx @@ -0,0 +1,215 @@ +--- +title: Speaker diarization +description: Separate and label speakers by voice, and tune sensitivity, speaker limits, and label stability. +keywords: + [ + speechmatics, + diarization, + speaker diarization, + speakers, + transcription, + speech recognition, + asr, + ] +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +Speaker diarization separates a transcript into distinct speakers, so you can see who said what. It picks out different speakers based on acoustic matching, aggregating all audio channels into a single stream for processing. + +It is especially useful in conversations, meetings, interviews, and recordings where keeping track of each voice matters. By labeling speakers, diarization makes transcripts easier to read, analyze, and share. + +## Use cases + +- **Call centers** — identify agents and customers for training, compliance, and quality assurance. +- **Video conferences** — track who said what in multi-participant discussions. +- **Medical consultations** — capture conversations between doctors and patients with clear speaker labels. +- **Media production** — make multi-speaker audio easier to edit, search, and annotate. + +## Availability + +Speaker diarization is available for pre-recorded transcription with Standard, Enhanced, and Melia 1; for streaming with Standard and Enhanced; and for agent STT with Linden 1. See [Feature availability](/speech-to-text/availability/). + +To separate speakers by audio channel instead of by voice, see [Channel diarization](/speech-to-text/features/channel-diarization). + +## Enable speaker diarization + +Speaker diarization is disabled by default for pre-recorded and streaming transcription. Set `diarization` to `speaker` in the transcription config. + + + + +```json +{ + "type": "transcription", + "transcription_config": { + "model": "enhanced", + "language": "en", + // highlight-start + "diarization": "speaker" + // highlight-end + } +} +``` + + + + +```json +{ + "type": "transcription", + "transcription_config": { + "model": "enhanced", + "language": "en", + // highlight-start + "diarization": "speaker" + // highlight-end + } +} +``` + + + + +Agent STT enables speaker diarization by default. Set `diarization` to `none` to disable it. + +```json +{ + "message": "StartRecognition", + "transcription_config": { + "language": "en", + // highlight-start + "diarization": "speaker" + // highlight-end + } +} +``` + + + + +## Speaker labels + +For pre-recorded and streaming transcription, each `word` and `punctuation` object in the transcript includes a `speaker` property identifying who spoke it. There are two types of label: + +- `S#` — S stands for speaker, and `#` is a sequential number identifying each speaker. S1 appears first in the results, followed by S2, S3, and so on. +- `UU` — used when the speaker cannot be identified or diarization is not applied, for example if background noise is transcribed as speech but no speaker can be determined. + +```json + "results": [ + { + "alternatives": [ + { + "confidence": 0.93, + "content": "hello", + "language": "en", + // highlight-start + "speaker": "S1" + // highlight-end + } + ], + ... + }, + { + "alternatives": [ + { + "confidence": 1.0, + "content": "hi", + "language": "en", + // highlight-start + "speaker": "S2" + // highlight-end + } + ], + }] +``` + +Agent STT returns the label as `speaker_id` on each segment rather than on each word. See [Agent STT messages](/speech-to-text/agent-stt/messages#addsegment). + +## Speaker sensitivity {#speaker-sensitivity} + +Configure the sensitivity of speaker detection with the `speaker_sensitivity` setting in `speaker_diarization_config`: + +```json +{ + "type": "transcription", + "transcription_config": { + "model": "enhanced", + "language": "en", + // highlight-start + "diarization": "speaker", + "speaker_diarization_config": { + "speaker_sensitivity": 0.6 + } + // highlight-end + } +} +``` + +This takes a value between 0 and 1, with a default of 0.5. A higher sensitivity increases the likelihood of more unique speakers being returned. + +## Prefer current speaker {#prefer-current-speaker} + +Reduce the likelihood of incorrectly switching between similar-sounding speakers by setting the `prefer_current_speaker` flag in `speaker_diarization_config`: + +```json +{ + "type": "transcription", + "transcription_config": { + "model": "enhanced", + "language": "en", + // highlight-start + "diarization": "speaker", + "speaker_diarization_config": { + "prefer_current_speaker": true + } + // highlight-end + } +} +``` + +By default this flag is `false`. When set to `true`, the system stays with the speaker of the previous word if they closely match the speaker of the new word. This may result in some shorter speaker turn changes between similar speakers being missed. + +## Max speakers {#max-speakers} + +Prevent too many speakers from being detected with the `max_speakers` setting. This applies to streaming transcription and agent STT. + +```json +{ + "message": "StartRecognition", + "audio_format": { + "type": "raw", + "encoding": "pcm_f32le", + "sample_rate": 48000 + }, + "transcription_config": { + "language": "en", + "model": "enhanced", + // highlight-start + "diarization": "speaker", + "speaker_diarization_config": { + "max_speakers": 10 + } + // highlight-end + } +} +``` + +By default there is no limit on the number of speakers. When set explicitly, `max_speakers` accepts any integer greater than or equal to 2. + +## Speaker diarization and punctuation + +Speaker diarization uses punctuation to improve accuracy. Small corrections are applied to speaker labels based on sentence boundaries. For example, if the system initially assigns 9 words in a sentence to S1 and 1 word to S2, the lone S2 word may be corrected to S1. + +This adjustment only works when punctuation is enabled. Disabling punctuation with the `permitted_marks` setting in `punctuation_overrides` can reduce diarization accuracy, and adjusting punctuation sensitivity can also affect how accurately speakers are identified. See [Formatting and punctuation](/speech-to-text/features/formatting#punctuation). + +## Considerations + +Enabling diarization increases the time taken to transcribe a pre-recorded file. Expect diarization to increase overall processing time by 10 to 50 percent. + +## Deprecated: speaker change + +Speaker change detection was removed in July 2024. The `speaker_change` and `channel_and_speaker_change` parameters are no longer supported. Use speaker diarization for speaker labeling instead. + +For API-related questions, contact [Support](https://support.speechmatics.com). diff --git a/docs/speech-to-text/features/speaker-identification.mdx b/docs/speech-to-text/features/speaker-identification.mdx index 513ff2c8..cf7ca459 100644 --- a/docs/speech-to-text/features/speaker-identification.mdx +++ b/docs/speech-to-text/features/speaker-identification.mdx @@ -1,5 +1,6 @@ --- -description: "Learn how Speechmatics identifies speakers in audio" +title: Speaker identification +description: Enroll speaker identifiers and label known speakers consistently across recordings. keywords: [ speechmatics, @@ -8,51 +9,322 @@ keywords: diarization, transcription, speech recognition, - automatic speech recognition, asr, ] -sidebar_position: 2 --- -# Speaker identification +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; +import speakerIdEnrollmentPythonExample from '../streaming/assets/speaker-id-enrollment-file-example.py'; +import speakerIdIdentificationPythonExample from '../streaming/assets/speaker-id-identification-file-example.py'; -Speaker identification lets you tag speakers consistently across recordings with the help of speaker identifiers. You can generate these string-encoded voice representations using short audio samples of the target speakers. +Speaker identification lets you tag speakers consistently across recordings using speaker identifiers, which are string-encoded voice representations generated from short audio samples of the target speakers. -By tagging known speakers with consistent labels, speaker identification makes transcripts more accurate, searchable, and easier to analyze over time. Providing speaker identifiers can also increase the accuracy of our diarization system. +By tagging known speakers with consistent labels, speaker identification makes transcripts more accurate, searchable, and easier to analyze over time. Providing speaker identifiers can also increase the accuracy of diarization. ## Use cases -- **Contact centers** – Recognize and tag individual agents and returning customers by name for personalized service, training, and compliance tracking. -- **Video conferences** – Automatically label participants across multiple meetings to know who said what and maintain consistent speaker records and analytics. -- **Medical consultations** – Identify doctors and patients across sessions for accurate records and follow-up care. -- **Media production** – Consistently label recurring speakers or public figures across episodes or segments, which is valuable in subtitling, media search and archiving. +- **Contact centers** — recognize and tag individual agents and returning customers by name for personalized service, training, and compliance tracking. +- **Video conferences** — automatically label participants across multiple meetings to know who said what and maintain consistent speaker records and analytics. +- **Medical consultations** — identify doctors and patients across sessions for accurate records and follow-up care. +- **Media production** — consistently label recurring speakers or public figures across episodes or segments, which is valuable in subtitling, media search, and archiving. -## How it works +## Availability + +Speaker identification is available with the Standard and Enhanced models for pre-recorded and streaming transcription, and with Linden 1 for agent STT. See [Feature availability](/speech-to-text/availability/). + +It requires [speaker diarization](/speech-to-text/features/speaker-diarization) to be enabled. -To use speaker identification you must enable diarization in the [speaker mode](/speech-to-text/features/diarization#diarization-modes) and then follow the two steps below: +## How it works -- **Enrollment** - For each speaker you want to recognize, generate identifiers from short audio clips (5–30s) where they ideally speak alone. - To improve robustness, you can enroll the same speaker with multiple clips recorded under different acoustic conditions, selected to represent the degree of variety and quality that could be expected in the target audio. +Speaker identification is a two-step process. -- **Identification** - Use the enrolled identifiers in transcription jobs to label known speakers with meaningful names (for example, `Alice` or `John`). The system matches voices to identifiers and tags the output with the desired labels. +- **Enrollment** — for each speaker you want to recognize, generate identifiers from short audio clips of 5 to 30 seconds where they ideally speak alone. To improve robustness, you can enroll the same speaker with multiple clips recorded under different acoustic conditions, chosen to represent the variety and quality expected in the target audio. +- **Identification** — use the enrolled identifiers in later requests to label known speakers with meaningful names such as `Alice` or `John`. The system matches voices to identifiers and tags the output with your labels. :::info -It is recommended to minimize the number of speaker IDs to achieve optimal accuracy. A maximum of 50 speaker identifiers across all speakers can be configured per session. Additionally, labels for identified speakers must not use reserved internal labels (e.g., UU, S1, S2) and should not contain leading or trailing spaces. +Minimize the number of speaker identifiers for optimal accuracy. A maximum of 50 speaker identifiers across all speakers can be configured per session. Labels for identified speakers must not use reserved internal labels such as `UU`, `S1`, or `S2`, and must not contain leading or trailing spaces. ::: +## Enroll speakers + +Run a transcription with speaker diarization enabled on an audio sample where the speaker ideally speaks alone, then request the identifiers back. + + + + +Set the `get_speakers` flag in the transcription config: + +```json +{ + "type": "transcription", + "transcription_config": { + "model": "enhanced", + "language": "en", + "diarization": "speaker", + "speaker_diarization_config": { + // highlight-start + "get_speakers": true + // highlight-end + } + } +} +``` + +When the transcription is done, the speaker identifiers are attached to the returned transcript: + +```json +{ + "results": [ + { + "alternatives": [ + { + "confidence": 0.93, + "content": "Hello", + "language": "en", + "speaker": "S1" + } + ], + ... + }, + { + "alternatives": [ + { + "confidence": 1.0, + "content": "Hi", + "language": "en", + "speaker": "S2" + } + ], + ... + }], + // highlight-start + "speakers": [ + { + "label": "S1", + "speaker_identifiers": [""] + }, + { + "label": "S2", + "speaker_identifiers": [""] + }] + // highlight-end +} +``` + + + + +Send a `GetSpeakers` request. By default the engine returns identifiers created up to the time of the request; set the optional `final` flag to wait until the end of the stream: + +```json +{ + "message": "GetSpeakers", + "final": true +} +``` + +- `final: false` (default) — returns identifiers generated up to the point of the request. To avoid empty results, wait until the server has issued at least one `AddTranscript` message before sending the request. +- `final: true` — waits until the end of the stream and returns identifiers based on all audio. + +Alternatively, enable automatic speaker retrieval by setting `get_speakers` to `true` in the diarization config. This is recommended for enrollment: it guarantees the engine provides identifiers once transcription is complete, equivalent to calling `GetSpeakers(final=true)`. If `get_speakers` is absent or `false`, you can still request speakers explicitly by sending `GetSpeakers(final=true)`; the request takes precedence and the engine returns identifiers at the end of transcription. + +```json +{ + "type": "transcription", + "transcription_config": { + "model": "enhanced", + "language": "en", + "diarization": "speaker", + "speaker_diarization_config": { + // highlight-start + "get_speakers": true + // highlight-end + } + } +} +``` + +The server replies with a `SpeakersResult` message containing the identifiers for each diarized speaker: + +```json +{ + "message": "SpeakersResult", + "speakers": [ + {"label": "S1", "speaker_identifiers": [""]}, + {"label": "S2", "speaker_identifiers": [""]} + ] +} +``` + + + + +Send a [`GetSpeakers`](/speech-to-text/agent-stt/messages#getspeakers) message at any point during the session. The server replies with [`SpeakersResult`](/speech-to-text/agent-stt/messages#speakersresult). + +```json +{ + "message": "GetSpeakers" +} +``` + +See [Speaker focus and identification](/speech-to-text/agent-stt/speaker-focus) for the agent STT workflow in full. + + + + +## Identify known speakers + +Provide your stored identifiers in a later request to tag known speakers. + +Note the field differs by interaction pattern: pre-recorded and streaming use `speaker_diarization_config.speakers`, while agent STT uses `transcription_config.known_speakers`. Both express the same concept. + + + + +All other [speaker diarization options](/speech-to-text/features/speaker-diarization) remain supported. The `speakers_sensitivity` parameter adjusts how strongly the system prefers enrolled speakers over detecting new generic ones; lower values make it more likely to match existing enrolled speakers. + +```json +{ + "type": "transcription", + "transcription_config": { + "model": "enhanced", + "language": "en", + "diarization": "speaker", + "speaker_diarization_config": { + // highlight-start + "speakers": [ + {"label": "Alice", "speaker_identifiers": ["", ""]}, + {"label": "Bob", "speaker_identifiers": [""]} + ] + // highlight-end + } + } +} +``` + + + + +All [speaker diarization options](/speech-to-text/features/speaker-diarization) work with speaker identification. The [`max_speakers`](/speech-to-text/features/speaker-diarization#max-speakers) parameter applies only to generic, non-enrolled speakers: if it is set to 10 and 10 speakers are enrolled, the system can still add up to 10 additional generic speakers. The `speakers_sensitivity` parameter adjusts how strongly the system prefers enrolled speakers over detecting new generic ones; lower values make it more likely to match existing enrolled speakers. + +```json +{ + "type": "transcription", + "transcription_config": { + "model": "enhanced", + "language": "en", + "diarization": "speaker", + "speaker_diarization_config": { + // highlight-start + "speakers": [ + {"label": "Alice", "speaker_identifiers": ["", ""]}, + {"label": "Bob", "speaker_identifiers": [""]} + ] + // highlight-end + } + } +} +``` + + + + +Pass identifiers in `transcription_config.known_speakers`: + +```json +{ + "message": "StartRecognition", + "transcription_config": { + "language": "en", + // highlight-start + "known_speakers": [ + { "label": "Alice", "speaker_identifiers": [""] }, + { "label": "Bob", "speaker_identifiers": [""] } + ] + // highlight-end + } +} +``` + + + + +With the config above, transcript segments are tagged with `"Alice"` and `"Bob"` whenever those speakers are detected. Any other speakers are tagged with the internal labels: + +```json +{ + "results": [ + { + "alternatives": [ + { + "confidence": 0.93, + "content": "Morning", + "language": "en", + // highlight-start + "speaker": "Alice" + // highlight-end + } + ], + ... + }, + { + "alternatives": [ + { + "confidence": 0.93, + "content": "Hi", + "language": "en", + "speaker": "S1" + } + ], + ... + }, + { + "alternatives": [ + { + "confidence": 1.0, + "content": "Morning", + "language": "en", + // highlight-start + "speaker": "Bob" + // highlight-end + } + ], + }] +} +``` + ## Known caveats Speaker identifiers have the following limitations and scoping rules: -- **Model-specific** — Identifiers are tied to the model used to generate them. Using identifiers across different models is not supported and any such identifiers will be ignored. - Whenever a model is updated, existing identifiers must always be regenerated. +- **Model-specific** — identifiers are tied to the model used to generate them. Using identifiers across different models is not supported, and any such identifiers are ignored. Whenever a model is updated, existing identifiers must be regenerated. +- **Encrypted and scoped** — identifiers are securely encrypted and scoped to your account context: + - **Per customer** — identifiers are unique to each customer and cannot be shared or reused across customers. + - **Per project** — if you use multiple [projects](/administration/projects) under the same customer, identifiers are isolated per project and cannot be used across them. + +In all of these cases, including model mismatches and attempts to use identifiers across customers or projects, a warning is issued to alert you that the affected identifiers have been ignored. + +## Code examples + + + + +Streaming speaker enrollment example. + + +{speakerIdEnrollmentPythonExample} + -- **Encrypted and scoped** — Identifiers are securely encrypted and scoped to your account context: - - **Per customer** — Identifiers are unique to each customer and cannot be shared or reused across customers. - - **Per project** — If you use multiple [projects](/administration/projects) under the same customer, identifiers are isolated per project and cannot be used across them. + + -In all of the above cases — including model mismatches or attempts to use identifiers across customers or projects — a **warning will be issued** to alert you that the affected identifiers have been ignored. +Streaming speaker identification example. -## Supported modes + +{speakerIdIdentificationPythonExample} + -Speaker identification is supported in both **[Realtime speaker identification](/speech-to-text/realtime/speaker-identification)** and **[Batch speaker identification](/speech-to-text/batch/speaker-identification)** modes. + + diff --git a/docs/speech-to-text/features/text-replacement.mdx b/docs/speech-to-text/features/text-replacement.mdx new file mode 100644 index 00000000..8b44c4cc --- /dev/null +++ b/docs/speech-to-text/features/text-replacement.mdx @@ -0,0 +1,61 @@ +--- +title: Text replacement +description: Replace matched words or phrases in the transcript with your own wording. +keywords: [text replacement, find and replace, replacements, regex, transcript filtering] +--- + +Text replacement substitutes specific words or patterns in the transcript after transcription is complete. + +## Availability + +Text replacement is available with the Standard and Enhanced models for pre-recorded and streaming transcription, and with Linden 1 for agent STT. See [Feature availability](/speech-to-text/availability/). + +## Replace words and phrases + +Add `replacements` to `transcript_filtering_config`: + +```json +"transcription_config": { + "model": "enhanced", + "language": "en", + "transcript_filtering_config": { + "replacements": [ + {"from": "foo", "to": "bar"}, + {"from": "heavy", "to": "light"} + ] + } +} +``` + +Common uses for text replacement: + +- Censoring profanities in languages without built-in support +- Masking sensitive information such as card numbers and personal data +- Standardizing terminology or brand names +- Fixing known issues with particular words + +Text replacement is case-sensitive and applied after transcription is complete. In the example above, "Foo" would not be replaced by "bar". + +:::tip +To add new vocabulary rather than replace it, use the [custom dictionary](/speech-to-text/features/custom-dictionary) instead. +::: + +## Match with a regular expression {#regex} + +You can use regular expressions in ECMAScript format in the `from` field by adding forward-slash delimiters: + +```javascript +// Replace both "Hello" and "hello" with "goodbye" +{"from": "/^[hH]ello$/", "to": "goodbye"} + +// Add brackets around "cheese" while preserving the original word +{"from": "/(cheese)/", "to": "[$1]"} +``` + +Replacement rules are applied in this order: + +1. Plain word replacements are processed first. +2. If no match is found, regex replacements are tried in the order listed. +3. Once a word matches a replacement, no further replacements are applied to it. +4. Regex replacements are global, so all matches are replaced. +5. Malformed regex patterns cause the transcription to fail with an error. diff --git a/docs/speech-to-text/realtime/turn-detection.mdx b/docs/speech-to-text/features/turn-detection.mdx similarity index 86% rename from docs/speech-to-text/realtime/turn-detection.mdx rename to docs/speech-to-text/features/turn-detection.mdx index ce4abdd2..35f75013 100644 --- a/docs/speech-to-text/realtime/turn-detection.mdx +++ b/docs/speech-to-text/features/turn-detection.mdx @@ -14,8 +14,8 @@ toc_max_heading_level: 3 import CodeBlock from "@theme/CodeBlock"; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import eouStreamingPythonExample from "./assets/end-of-utterance-streaming-example.py" -import eouFilePythonExample from "./assets/end-of-utterance-file-example.py" +import eouStreamingPythonExample from "../streaming/assets/end-of-utterance-streaming-example.py" +import eouFilePythonExample from "../streaming/assets/end-of-utterance-file-example.py" import SchemaNode from '@theme/Schema' import realtimeSchema from "!asyncapi-schema-loader!@site/spec/realtime.yaml" @@ -23,6 +23,10 @@ import realtimeSchema from "!asyncapi-schema-loader!@site/spec/realtime.yaml" Use the end of utterance feature to help with turn detection in real-time conversational scenarios. +## Availability + +Turn detection is available with the Standard and Enhanced models for streaming transcription, and with Linden 1 for agent STT. Agent STT selects it with a [profile](/speech-to-text/agent-stt/profiles) rather than a silence trigger. See [Feature availability](/speech-to-text/availability/). + ## Use cases **Voice AI and conversational systems**: Enable voice assistants and chatbots to detect when the user has finished speaking, allowing the system to respond promptly without awkward delays. @@ -39,7 +43,7 @@ End of utterance detection is a feature that allows you to detect when a person End of utterance uses server-side word timings to detect periods without speech. The moment a word is detected on the server, a countdown begins. If a new word is detected, the countdown restarts. If the configured interval passes without another word being detected, an end of utterance is triggered. -When this happens, the server sends a [final transcript](/speech-to-text/realtime/quickstart#receiving-finals-and-partials) message to the client, followed by an extra `EndOfUtterance` message. +When this happens, the server sends a [final transcript](/speech-to-text/streaming/quickstart#receiving-finals-and-partials) message to the client, followed by an extra `EndOfUtterance` message. ### Configuration @@ -72,8 +76,8 @@ On detecting an end of utterance, the server sends an `EndOfUtterance` message, :::tip - We recommend 0.5-0.8 seconds for most voice AI applications. Longer values (0.8-1.2s) may be better for dictation applications. - Keep the `end_of_utterance_silence_trigger` lower than the `max_delay` value. -- `EndOfUtterance` messages are only sent after some speech is recognised and duplicate `EndOfUtterance` messages will never be sent for the same period of silence. -- The `EndOfUtterance` message is not related to any specific individual identified by [diarization](/speech-to-text/features/diarization) and will not contain speaker information. +- `EndOfUtterance` messages are only sent after some speech is recognized and duplicate `EndOfUtterance` messages will never be sent for the same period of silence. +- The `EndOfUtterance` message is not related to any specific individual identified by [diarization](/speech-to-text/features/speaker-diarization) and will not contain speaker information. ::: ### Code examples diff --git a/docs/speech-to-text/formatting.mdx b/docs/speech-to-text/formatting.mdx deleted file mode 100644 index c27fa6e8..00000000 --- a/docs/speech-to-text/formatting.mdx +++ /dev/null @@ -1,448 +0,0 @@ ---- -keywords: - [ - entity formatting, - numeral formatting, - output formatting, - punctuation, - smart formatting, - transcript format, - transcription - ] -toc_max_heading_level: 4 -description: 'Control how numbers, punctuation, and special text appear in your transcripts.' ---- - -import CodeBlock from '@theme/CodeBlock' - -# Formatting - -## Output locale - -Some languages have multiple spelling conventions that vary by region. To ensure consistent spelling throughout your transcript, specify an output locale: - -```json -{ - "type": "transcription", - "transcription_config": { - "model": "enhanced", - "language": "en", - "output_locale": "en-GB" - } -} -``` - -Available English locales: -- British English (`en-GB`) -- US English (`en-US`) -- Australian English (`en-AU`) - -Available Mandarin locales: -- Simplified Mandarin (`cmn-Hans`, default) -- Traditional Mandarin (`cmn-Hant`) - -:::tip -**Recommended for English transcription**. Without a specified locale, spelling may be inconsistent within the same transcript. -::: - -## Profanities - -You can tag profanities to identify or censor offensive language in your workflow. Profanity tagging is available for: -- English (`en`) -- Italian (`it`) -- Spanish (`es`) - -Tagged profanities appear in the transcript with the `profanity` tag: - -```json -"results": [ - { - "alternatives": [ - { - "confidence": 1.0, - "content": "$PROFANITY", - "language": "en", - "tags": [ - "profanity" - ] - } - ], - "end_time": 18.03, - "start_time": 17.61, - "type": "word" - } -] -``` - -For other languages, consider using [word replacement](#word-replacement) to identify profanities. - -## Disfluencies - -Disfluencies are hesitation sounds like "um", "uh", and "hmm". Speechmatics automatically tags them with `disfluency` in the transcript output: - -```json -"results": [ - { - "alternatives": [ - { - "confidence": 1.0, - "content": "hmm", - "language": "en", - "tags": [ - "disfluency" - ] - } - ], - "end_time": 18.03, - "start_time": 17.61, - "type": "word" - } -] -``` - -
- Full list of tagged English disfluencies - -{`huh -aha -ah -aw -eh -err -hmm -mm -um -uh -uh-oh -uh-huh -uh-uh -mhm -a-ha -aah -aahh -aaw -ah-ha -ahaa -ahh -ahha -aww -eeh -erm -hhm -hhmm -hm -huh-uh -m-hm -uggh -ugh -ughh -uhh -uhhm -uhm -uhmm -umm -uuh -uuhh -uum`} - -
- -### Supported languages for disfluencies - -Disfluency tagging and removal are available for the following languages. Each language has its own set of hesitation sounds; the expandable list at the start of this section covers English. - -- Arabic (`ar`) -- Danish (`da`) -- Dutch (`nl`) -- English (`en`) -- French (`fr`) -- German (`de`) -- Greek (`el`) -- Hebrew (`he`) -- Hindi (`hi`) -- Hungarian (`hu`) -- Italian (`it`) -- Japanese (`ja`) -- Mandarin (`cmn`) -- Polish (`pl`) -- Portuguese (`pt`) -- Russian (`ru`) -- Spanish (`es`) - -Coverage of hesitation sounds varies by language. If you rely on disfluency removal for a specific language, test it with representative audio rather than assuming full coverage. - -### Removing disfluencies - -You can automatically remove disfluencies from your transcript: - -```json -"transcription_config": { - "model": "enhanced", - "language": "en", - "transcript_filtering_config": { - "remove_disfluencies": true - } -} -``` - -This simplifies client-side processing by removing hesitation sounds and properly adjusting capitalization and spacing. For example: - -Without disfluency removal: -> Um, what would you like, hmm? - -With disfluency removal: -> What would you like? - -This feature is available for the [supported languages](#supported-languages-for-disfluencies). The default setting is `"remove_disfluencies": false`. - -## Word replacement - -Word replacement lets you substitute specific words or patterns in the transcript after processing: - -```json -"transcription_config": { - "model": "enhanced", - "language": "en", - "transcript_filtering_config": { - "replacements": [ - {"from": "foo", "to": "bar"}, - {"from": "heavy", "to": "light"} - ] - } -} -``` - -Common uses for word replacement: -- Censoring profanities in languages without built-in support -- Masking sensitive information (card numbers, personal data) -- Standardizing terminology or brand names -- Fixing known issues with particular words - -Word replacement is case-sensitive and applied after transcription is complete. For example, "Foo" would not be replaced by "bar" in the example above. - -:::tip -For adding new vocabulary, use the [custom dictionary](/speech-to-text/features/custom-dictionary) feature instead. -::: - -### Regex - -You can use regular expressions (ECMAScript format) in the `from` field by adding forward-slash delimiters: - -```javascript -// Replace both "Hello" and "hello" with "goodbye" -{"from": "/^[hH]ello$/", "to": "goodbye"} - -// Add brackets around "cheese" while preserving the original word -{"from": "/(cheese)/", "to": "[$1]"} -``` - -Word replacement rules: -1. Plain word replacements are processed first -2. If no match is found, regex replacements are tried in the order listed -3. Once a word matches a replacement, no further replacements are applied to it -4. Regex replacements are global (all matches are replaced) -5. Malformed regex patterns will cause the transcription to fail with an error - - -## Smart formatting - -Smart formatting converts spoken numbers, dates, currencies, and other entities into properly formatted text. This makes transcripts more readable without losing timing information. - -An *entity* is a spoken value that has a conventional written form, such as a number, date, currency, time, or measurement. Speechmatics detects each entity and converts it from the words as spoken into its written form. For example, the spoken words "nineteen ninety nine" become "1999" in the output. - -Smart formatting is applied by default. Set `enable_entities` to `true` to also expose the structure of each entity in the JSON output: the class of entity, and the individual spoken and written words it is made from. - -### Configuration - -To include detailed entity information in your JSON output, add `enable_entities` to your configuration: - -```json -{ - "type": "transcription", - "transcription_config": { - "model": "enhanced", - "language": "en", - "enable_entities": true - } -} -``` - -By default, `enable_entities` is `false`. When enabled, entity metadata appears only in JSON output (SRT and TXT formats remain unchanged). - -### Output - -The JSON output will include: - -- A new `type` field with value `entity` for formatted numeric entities -- Full written form in the `content` section, including any spaces or symbols -- An `entity_class` field describing how the entity was formatted -- Start and end times spanning all words in the entity -- Two additional representations: - - `spoken_form`: Original words as spoken, with individual timing and confidence - - `written_form`: Formatted words separated individually - -Here's an example of a transcript with `enable_entities` set to `true`: - -```json -{ - "results": [ - { - "alternatives": [ - { - "confidence": 0.99, - "content": "17th of January 2022", - "language": "en", - "speaker": "UU" - } - ], - "end_time": 3.14, - "entity_class": "date", - "spoken_form": [ - { - "alternatives": [ - { - "confidence": 1.0, - "content": "seventeenth", - "language": "en", - "speaker": "UU" - } - ], - "end_time": 1.41, - "start_time": 0.72, - "type": "word" - }, - // Additional spoken words omitted for brevity - ], - "start_time": 0.72, - "type": "entity", - "written_form": [ - { - "alternatives": [ - { - "confidence": 0.99, - "content": "17th", - "language": "en", - "speaker": "UU" - } - ], - "end_time": 1.33, - "start_time": 0.72, - "type": "word" - }, - // Additional written words omitted for brevity - ] - } - ] -} -``` - -When `enable_entities` is `false`, the words appear individually in the output. - - - -### Entity classes - -The system applies different formatting rules based on the type of entity detected. The following classes are available: - -| Entity class | Description | Spoken example | Written example | -|-------------|-------------|---------------|-----------------| -| alphanum | Alphanumeric sequences (3+ characters) | "a z triple seven five four" | AZ77754 | -| cardinal | Whole numbers (in English, numbers ≤10 remain as words) | "nineteen" | 19 | -| decimal | Numbers with decimal point | "eighteen point one two" | 18.12 | -| fraction | Fractions (complex ones use n/d format) | "three sixteenths" | 3/16 | -| ordinal | Position numbers with suffix | "forty second" | 42nd | -| money | Currency values with symbol | "twenty dollars" | $20 | -| percentage | Percentages with % symbol | "two hundred percent" | 200% | -| date | Calendar dates and years | "fifteenth of January twenty twenty two" | 15th of January 2022 | -| time | Clock times with separators | "eleven forty a m" | 11:40 a.m. | -| span | Ranges (x to y format) | "one hundred to two hundred million pounds" | 100 to £200 million | -| credit card | Payment card number sequences | "one one one one..." | 1111 2222 3333 4444 | -| telephone | Phone number formatting | "five five five..." | (555) 429-2228 | -| electronic | Email and web addresses | "bob at speechmatics dot com" | bob@speechmatics.com | -| measurement | Units with abbreviations | "ten kilometers per second" | 10 km/s | - -:::warning -The system chooses entity classes based on context, so occasionally a value might be classified differently than expected. For example, "2001" could be a "cardinal" number or a "date". -::: - -### Languages for smart formatting - -Each language follows its own conventions for: - -- Thousand separators -- Decimal separators -- Currency symbol position - -Examples: -- **English**: Uses commas for thousands (20,000), decimal points (10.5), and places currency symbols before values ($10) -- **German**: Uses periods for thousands (20.000), commas for decimals (10,5), and places currency symbols after values with a non-breaking space (10 $) -- **French**: Uses non-breaking spaces for thousands (20 000), commas for decimals (10,5), and places currency symbols after values with a non-breaking space (10 $) - -Smart formatting has had dedicated work for consistent results in these languages: - -- Cantonese -- Dutch -- English -- French -- German -- Hindi -- Italian -- Japanese -- Mandarin (Simplified and Traditional) -- Mandarin & English (bilingual) -- Mandarin Malay Tamil & English (multilingual) -- Norwegian -- Portuguese -- Russian -- Spanish -- Swedish -- Tamil & English (bilingual) - -Other languages still format numbers and entities on a best-effort basis through the model, with variable results. If you rely on formatting for a language that isn't listed, test it with representative audio rather than assuming full coverage. - -Formatting coverage isn't reported by [feature discovery](/speech-to-text/features/feature-discovery), which covers transcription, translation, and language identification. This page is the reference for formatting language support. - -## Punctuation - -All Speechmatics language packs support punctuation to improve transcript readability. Each language supports specific punctuation marks: - -| Language | Supported marks | End-of-sentence marks | Notes | -|----------|----------------|---------------------|-------| -| Cantonese, Mandarin | , 。 ? ! 、 | 。 ? ! | Full-width punctuation | -| Japanese | 。 、 | 。 | Full-width punctuation | -| Hindi | । ? , ! | । ? ! | | -| All other languages | . , ! ? | . ! ? | | - -### Configuration - -You can control which punctuation marks appear in your transcripts using the `punctuation_overrides` setting: - -```json -"transcription_config": { - "model": "enhanced", - "language": "en", - // highlight-start - "punctuation_overrides": { - "permitted_marks": [".", ","], - "sensitivity": 0.4 - } - // highlight-end -} -``` - -This configuration: -- Allows only periods and commas (no question or exclamation marks) -- Sets punctuation sensitivity to 0.4 (lower than the default 0.5) -- You can select all marks by selecting `"permitted_marks": ["all"]` -- If you use an empty list for permitted_marks, no punctuation marks will be present in the output - -The `sensitivity` parameter accepts values from 0 to 1. Higher values produce more punctuation in the output. - -:::warning -Disabling punctuation may slightly reduce speaker diarization accuracy. See [speaker diarization](/speech-to-text/features/diarization) for details. -::: - -## Next steps - -- [Custom Dictionary](/speech-to-text/features/custom-dictionary): Improve recognition of specific words and phrases by adding them to a custom dictionary. -- [Diarization](/speech-to-text/features/diarization): Enhance your transcripts with speaker and channel information. diff --git a/docs/speech-to-text/index.mdx b/docs/speech-to-text/index.mdx index 30995b10..614c585b 100644 --- a/docs/speech-to-text/index.mdx +++ b/docs/speech-to-text/index.mdx @@ -1,11 +1,11 @@ --- -title: Speech to Text — Overview +title: Speech to Text pagination_prev: null pagination_next: null -description: Learn how to turn audio into text. +description: Choose an interaction pattern, a model, and a deployment, then start transcribing audio. --- import { LinkCard } from "@site/src/theme/LinkCard"; -import { Flex, Grid } from "@radix-ui/themes"; +import { Grid } from "@radix-ui/themes"; import { ChevronsRightIcon, FileAudio, @@ -17,86 +17,49 @@ import { ChartLine, } from "lucide-react"; -# Speech to Text overview +Speech to Text transcribes audio in three interaction patterns, which differ in how you present audio for processing: +- **pre-recorded**: submit a file and receive a complete transcript +- **streaming**: send live audio and receive transcripts as you speak +- **agent STT**: send live audio and receive complete speaker turns -Use Speech to Text to transcribe using one of the modes: -- [Realtime processing](#realtime-processing): Stream audio from an input device or file and receive instant updates of the transcription as it happens -- [Batch processing](#batch-processing): Submit an audio file and receive a complete text transcription once the processing is finished +Your choice of pattern determines which models you can use, and which features those models support. To understand how the three patterns relate to the Batch and Realtime APIs, see [Understanding interaction patterns](/speech-to-text/interaction-patterns). ## Developer quickstart } + icon={} direction="column" - title="Transcribe in real time" - description="Instantly convert streaming audio to text with Realtime processing" - href="/speech-to-text/realtime/quickstart" + title="Transcribe a file" + description="Submit an audio file and get a complete transcript back." + href="/speech-to-text/pre-recorded/quickstart" /> } + icon={} direction="column" - title="Transcribe a file" - description="Submit an audio file and receive a complete text transcription once the processing is finished" - href="/speech-to-text/batch/quickstart" + title="Transcribe live audio" + description="Stream live audio and receive transcripts as you speak." + href="/speech-to-text/streaming/quickstart" /> - :::tip -The quickest way to transcribe voice from audio is in our [web portal](https://portal.speechmatics.com/jobs/create/batch). +The quickest way to transcribe audio is in the [web portal](https://portal.speechmatics.com/jobs/create/batch). ::: -## Deployments - -Speechmatics provides flexible deployment options tailored to your requirements. You can host the platform in your own environment, use our managed service, or choose a hybrid approach. +## Choose a model and check availability -For deploying our API in your environment, [contact sales](https://www.speechmatics.com/speak-to-sales), or see our [on-prem documentation](/deployments). +Pre-recorded and streaming transcription offer the Standard, Enhanced, and Melia 1 models. Agent STT offers Linden 1. See [Models](/speech-to-text/models) to compare them. -## Realtime processing +Availability depends on the combination of pattern, model, and deployment, so check the combination rather than the product name. See [Feature availability](/speech-to-text/availability/). -Turn live audio into accurate transcripts — instantly. - -The [Speechmatics Realtime Speech to Text API](/api-ref/realtime-transcription-websocket) converts spoken audio into text with low latency and high accuracy. - -:::info -- Use when speed matters -- Transcribe live broadcasts or events -- Caption webinars, meetings, or podcasts in real time -- Power voice assistants or AI agents with live input -- Monitor contact center calls as they happen -- Build accessibility features like live captions -::: - -## Batch processing - -Create transcripts from pre-recorded audio or video. - -The [Speechmatics Batch Speech to Text API](/api-ref/batch/create-a-new-job) processes pre-recorded files asynchronously by default, returning highly accurate transcripts in a range of formats. To block for the result in a single request instead, use [synchronous transcription](/speech-to-text/batch/synchronous). - -:::info -- Transcribe recorded meetings or interviews -- Caption on-demand videos and podcasts -- Generate searchable transcripts for media archives -- Process customer service recordings for compliance or insights -- Automate subtitles across large video libraries -::: - - -### What is a job? - -Each transcription request is handled as a job — a self-contained unit representing a single transcription task. - -A job includes: -- The audio or video file to be transcribed -- Configuration settings (e.g. language, formatting, diarization) -- Metadata and status tracking -- The resulting transcript(s) +## Deployments -You submit a job to the API, monitor its progress, and retrieve results once it's complete. Jobs can be created via direct upload or by referencing a URL. +Speechmatics provides flexible deployment options tailored to your requirements. You can host the platform in your own environment, use our managed service, or choose a hybrid approach. +For deploying our API in your environment, [contact sales](https://www.speechmatics.com/speak-to-sales), or see our [on-prem documentation](/deployments). ## Quicklinks @@ -104,7 +67,7 @@ You submit a job to the API, monitor its progress, and retrieve results once it' } - href="/speech-to-text/languages" + href="/speech-to-text/languages/" description="See the languages Speechmatics supports" /> } description="Translate your transcript to another language" - href="/speech-to-text/features/translation" + href="/speech-to-text/add-ons/translation" /> } description="Identify speakers in your transcript" - href="/speech-to-text/features/diarization" + href="/speech-to-text/features/speaker-diarization" /> - \ No newline at end of file + diff --git a/docs/speech-to-text/interaction-patterns.mdx b/docs/speech-to-text/interaction-patterns.mdx new file mode 100644 index 00000000..2a0b30e3 --- /dev/null +++ b/docs/speech-to-text/interaction-patterns.mdx @@ -0,0 +1,82 @@ +--- +title: Understanding interaction patterns +sidebar_label: Interaction patterns +description: Understand how pre-recorded, streaming, and agent STT map to the Batch and Realtime APIs. +--- + +Speech to Text is described in two ways, for two different readers, and both are correct. + +An **interaction pattern** describes how you present audio for processing. There are three: pre-recorded, streaming, and agent STT. This is the vocabulary used for onboarding and explanation. + +A **processing mode** describes the transport you use to reach the service. There are two: Batch and Realtime. This is the vocabulary used in the API reference and the SDKs, including client names such as `BatchClient`. + +The two systems are not a subset of one another. Two interaction patterns share the Realtime processing mode, which is why "Realtime" alone does not tell you which pattern you are using. + +## Pre-recorded transcription + +You submit a complete audio file and receive a complete transcript. Processing is asynchronous by default: you submit a job, monitor its progress, and retrieve the result when it finishes. To block for the result in a single request instead, use [synchronous transcription](/speech-to-text/pre-recorded/synchronous). + +Pre-recorded transcription suits work where the audio already exists and accuracy matters more than latency: + +- Transcribe recorded meetings or interviews +- Caption on-demand videos and podcasts +- Generate searchable transcripts for media archives +- Process customer service recordings for compliance or insights +- Automate subtitles across large video libraries + +Available models: Standard, Enhanced, and Melia 1. See [models for pre-recorded transcription](/speech-to-text/models#pre-recorded-models). + +### What is a job? + +Each pre-recorded transcription request is handled as a job, a self-contained unit representing a single transcription task. + +A job includes: + +- The audio or video file to be transcribed +- Configuration settings such as language, formatting, and diarization +- Metadata and status tracking +- The resulting transcript or transcripts + +You submit a job to the API, monitor its progress, and retrieve results once it is complete. You can create a job by uploading a file directly or by referencing a URL. + +## Streaming transcription + +You open a session, send audio as it is captured, and receive transcripts while the audio is still arriving. Results arrive word by word: interim partial transcripts are refined until a final transcript is emitted. + +Streaming transcription suits work where the audio is live and latency matters: + +- Transcribe live broadcasts or events +- Caption webinars, meetings, or podcasts as they happen +- Monitor contact center calls in real time +- Build accessibility features such as live captions + +Available models: Standard, Enhanced, and Melia 1, the last in Preview. See [models for streaming transcription](/speech-to-text/models#streaming-models). + +## Agent STT + +You open a session and send live audio, as with streaming, but results arrive as complete speaker turns rather than as a word-by-word stream. Turn detection runs on the server, so the transcript you receive is already punctuated, speaker-labeled, and segmented at turn boundaries. + +Agent STT suits conversational applications that hand each turn to a language model and need to know when a speaker has finished talking. + +Agent STT provides the speech-to-text layer only. It is not a conversational pipeline and does not include a language model or speech synthesis. + +Available model: Linden 1, in Preview. See [models for agent STT](/speech-to-text/models#agent-stt-models). + +## How patterns map to the APIs + +This crosswalk is the authoritative mapping between the two naming systems. + +| Interaction pattern | Processing mode | Transport | `session_type` | +|---|---|---|---| +| pre-recorded | Batch | REST API | `batch` | +| streaming | Realtime | WebSocket API | `realtime` | +| agent STT | Realtime | WebSocket API | `agent` | + +Processing mode, transport, and `session_type` are fixed by your choice of interaction pattern rather than selected separately. + +`session_type` is a field on the usage endpoint. Its values name interaction patterns, not processing modes: `realtime` means streaming only, and agent STT reports as `agent`. Do not read `realtime` as covering all Realtime traffic. + +## Next steps + +- [Models](/speech-to-text/models#compare-the-models) — compare the models available for your pattern and set one in your request +- [Feature availability](/speech-to-text/availability/) — check what each combination supports diff --git a/docs/speech-to-text/languages.mdx b/docs/speech-to-text/languages.mdx deleted file mode 100644 index 2a77f070..00000000 --- a/docs/speech-to-text/languages.mdx +++ /dev/null @@ -1,169 +0,0 @@ ---- -title: Languages -description: See which languages Speechmatics supports for transcription and translation, including bilingual packs. -keywords: - [ - speechmatics, - languages, - transcription, - speech recognition, - asr, - en-us, - en-gb, - en-nz, - en-au, - fr-ca, - fr-be, - de-at, - de-ch, - es-mx, - es-cl, - es-ve, - es-pr, - es-ar, - pt-br, - model, - standard, - enhanced, - ] ---- - -# Languages - -To choose a transcription model, refer to [Models](/speech-to-text/models). - -The languages, packs, and options on this page apply to the Enhanced and Standard models. The Melia 1 model is multilingual: it transcribes the individual languages listed here and switches between them automatically, without language selection. You can use their language codes as [language hints](/speech-to-text/batch/input#language-hints). Melia 1 does not support the `auto` option, the bilingual and multi-language pack codes, or translation. For Melia 1, refer to [Models](/speech-to-text/models). - -## Transcription languages - -To automatically identify the language in an audio file, use the [Language Identification](/speech-to-text/batch/language-identification) feature. - -To dynamically update your system with the latest languages and features offered by Speechmatics, use the [Feature Discovery](/speech-to-text/features/feature-discovery) endpoint. - -Speechmatics supports the following languages. Your ability to use any or all of them depends on the languages you are contracted to use. - -Speechmatics takes a global-first approach to languages. A single language pack supports many accents and dialects, so you do not need to know which accent is in your audio before selecting a language. This approach achieves high accuracy compared to accent-specific language packs. - -| Language | Language code | Description | -|---|---|---| -| Automatic | `auto` | Automatically detect the language using the [Language Identification](/speech-to-text/batch/language-identification) feature. Currently supported with Batch transcription only. | -| Arabic | `ar` | Global Arabic gives high-accuracy transcription across many accents and dialects, including (but not limited to) Modern Standard Arabic (MSA) and Arabic spoken in the Gulf, Egypt, and the Levant. | -| Arabic & English bilingual | `ar_en` | Ideal when transcribing Arabic and English in the same media file or stream. Supports all accents and dialects listed under Arabic and English. | -| Bashkir | `ba` | | -| Basque | `eu` | | -| Belarusian | `be` | | -| Bengali | `bn` | | -| Bulgarian | `bg` | | -| Cantonese | `yue` | | -| Catalan | `ca` | | -| Croatian | `hr` | | -| Czech | `cs` | | -| Danish | `da` | | -| Dutch | `nl` | | -| English | `en` | Global English gives high-accuracy transcription across many accents, including (but not limited to) English spoken in the United Kingdom, United States, Australia, New Zealand, and by non-native speakers. To standardize spelling, specify the [output locale](/speech-to-text/formatting#output-locale). | -| Esperanto | `eo` | | -| Estonian | `et` | | -| Finnish | `fi` | | -| French | `fr` | Global French gives high-accuracy transcription across many accents, including (but not limited to) French spoken in France, Canada, and Belgium. | -| Galician | `gl` | | -| German | `de` | Global German gives high-accuracy transcription across many accents, including (but not limited to) German spoken in Germany, Austria, and Switzerland. | -| Greek | `el` | | -| Hebrew | `he` | | -| Hindi | `hi` | | -| Hungarian | `hu` | | -| Indonesian | `id` | | -| Interlingua | `ia` | | -| Irish | `ga` | | -| Italian | `it` | | -| Japanese | `ja` | | -| Korean | `ko` | | -| Latvian | `lv` | | -| Lithuanian | `lt` | | -| Malay | `ms` | | -| Malay & English bilingual | `en_ms` | Ideal when transcribing Malay and English in the same media file or stream. Supports all accents and dialects listed under Malay and English. | -| Maltese | `mt` | | -| Mandarin | `cmn` | Global Mandarin can output [Traditional or Simplified characters](/speech-to-text/formatting#output-locale) and gives high-accuracy transcription across many accents, including (but not limited to) China, Taiwan, Singapore, and Malaysia. | -| Mandarin & English bilingual | `cmn_en` | Ideal when transcribing Mandarin and English in the same media file or stream. Supports all accents and dialects listed under Mandarin and English. | -| Mandarin Malay Tamil & English | `cmn_en_ms_ta` | Ideal when transcribing Mandarin, Malay, Tamil, and English in the same media file or stream. Supports all accents and dialects listed under Mandarin, Malay, Tamil, and English. | -| Marathi | `mr` | | -| Mongolian | `mn` | | -| Norwegian | `no` | | -| Persian | `fa` | | -| Polish | `pl` | | -| Portuguese | `pt` | Global Portuguese gives high-accuracy transcription across many accents, including (but not limited to) Portuguese spoken in Portugal and Brazil. | -| Romanian | `ro` | | -| Russian | `ru` | | -| Slovakian | `sk` | | -| Slovenian | `sl` | | -| Spanish | `es` | Global Spanish gives high-accuracy transcription across many accents, including (but not limited to) Spanish spoken in Spain, the US, Mexico, Colombia, Argentina, Venezuela, Chile, and Peru. | -| Spanish & English bilingual | `es` (with domain=`bilingual-en`) | Ideal when transcribing Spanish and English in the same media file or stream. Supports all accents and dialects listed under English and Spanish. [Requires the domain config to be set](/speech-to-text/languages#bilingual-and-multi-language-packs). | -| Swahili | `sw` | | -| Swedish | `sv` | | -| Tagalog (Filipino) & English bilingual | `tl` | Ideal when transcribing Tagalog (Filipino) and English in the same media file or stream. Supports all accents and dialects listed under English. | -| Tamil | `ta` | | -| Tamil & English bilingual | `en_ta` | Ideal when transcribing Tamil and English in the same media file or stream. Supports all accents and dialects listed under Tamil and English. | -| Thai | `th` | | -| Turkish | `tr` | | -| Ukrainian | `uk` | | -| Urdu | `ur` | | -| Uyghur | `ug` | | -| Vietnamese | `vi` | | -| Welsh | `cy` | Welsh must be explicitly added to the [expected languages](/speech-to-text/batch/language-identification#expected-languages) list when using the Language Identification feature. Otherwise a [language not supported for transcription error](/speech-to-text/batch/language-identification#language-not-supported-for-transcription) is returned. | - -Each language is uniquely identified by a two-letter code (ISO 639-1) or three-letter code (ISO 639-3) in API requests and responses. - -## Translation languages - -Translation is available with the Enhanced and Standard models. It is supported for most Speechmatics languages, with the supported translation pairs listed below. For more details, see [Translation](/speech-to-text/features/translation). - -| Audio language | Translation target language | -|---|---| -| English (en) | Bulgarian (bg), Catalan (ca), Mandarin (cmn), Czech (cs), Danish (da), German (de), Greek (el), Spanish (es), Estonian (et), Finnish (fi), French (fr), Galician (gl), Hindi (hi), Croatian (hr), Hungarian (hu), Indonesian (id), Italian (it), Japanese (ja), Korean (ko), Lithuanian (lt), Latvian (lv), Malay (ms), Dutch (nl), Norwegian (no), Polish (pl), Portuguese (pt), Romanian (ro), Russian (ru), Slovakian (sk), Slovenian (sl), Swedish (sv), Turkish (tr), Ukrainian (uk), Vietnamese (vi) | -| Bulgarian (bg), Catalan (ca), Mandarin (cmn), Czech (cs), Danish (da), German (de), Greek (el), Spanish (es), Estonian (et), Finnish (fi), French (fr), Galician (gl), Hindi (hi), Croatian (hr), Hungarian (hu), Indonesian (id), Italian (it), Japanese (ja), Korean (ko), Lithuanian (lt), Latvian (lv), Malay (ms), Dutch (nl), Norwegian (no), Polish (pl), Portuguese (pt), Romanian (ro), Russian (ru), Slovakian (sk), Slovenian (sl), Swedish (sv), Turkish (tr), Ukrainian (uk), Vietnamese (vi) | English (en) | -| Norwegian Bokmål (no) | Norwegian Nynorsk (nn) | - -## Bilingual and multi-language packs - -:::note -These packs handle a fixed set of languages that you select in advance. To transcribe audio without selecting languages, including spontaneous switching across all supported languages, use the Melia 1 multilingual model. Refer to [Models](/speech-to-text/models). -::: - -The Enhanced and Standard models can transcribe a selected combination of languages in one media file or stream, including speakers who switch between the languages in that pack. Each pack covers a fixed set of languages that you select with the `language` property. - -Supported packs are: - -| Language pack | Transcription config | -|---|---| -| Arabic and English | `{"language": "ar_en"}` | -| Malay and English | `{"language": "en_ms"}` | -| Mandarin and English | `{"language": "cmn_en"}` | -| Mandarin Malay Tamil and English | `{"language": "cmn_en_ms_ta"}` | -| Spanish and English | `{"language": "es", "domain": "bilingual-en"}` | -| Tamil and English | `{"language": "en_ta"}` | -| Tagalog (Filipino) and English | `{"language": "tl"}` | - -This config selects the Mandarin and English pack: - -```json -{ - "type": "transcription", - "transcription_config": { - "model": "enhanced", - "language": "cmn_en" - } -} -``` - -This config selects the Spanish and English pack, which requires the `domain` property: - -```json -{ - "type": "transcription", - "transcription_config": { - "model": "enhanced", - "language": "es", - "domain": "bilingual-en" - } -} -``` - diff --git a/docs/speech-to-text/languages/index.mdx b/docs/speech-to-text/languages/index.mdx new file mode 100644 index 00000000..985b7da8 --- /dev/null +++ b/docs/speech-to-text/languages/index.mdx @@ -0,0 +1,103 @@ +--- +title: Languages +description: Select a transcription language or a bilingual pack, or let the model handle more than one. +sidebar_label: Languages +keywords: + [ + speechmatics, + languages, + transcription, + speech recognition, + asr, + en-us, + en-gb, + en-nz, + en-au, + fr-ca, + fr-be, + de-at, + de-ch, + es-mx, + es-cl, + es-ve, + es-pr, + es-ar, + pt-br, + model, + standard, + enhanced, + ] +--- + +How you select a language depends on the model. Standard and Enhanced transcribe one language that you choose, either a single-language pack or a bilingual pack. Melia 1 is multilingual and needs no language selection at all. + +For the code to use for a given language, see [Transcription languages](/speech-to-text/languages/transcription). To check which language features each model supports, see [Feature availability](/speech-to-text/availability/). + +Speechmatics takes a global-first approach to languages. A single language pack supports many accents and dialects, so you do not need to know which accent is in your audio before selecting a language. This approach achieves high accuracy compared to accent-specific language packs. + +## Choose a language with Standard or Enhanced + +Set the `language` property to a language code. To detect the language automatically instead of setting it, use [language identification](/speech-to-text/pre-recorded/language-identification). + +To standardize the spelling of the output for a region, set the [output locale](/speech-to-text/features/formatting#output-locale). + +## Transcribe more than one language with Melia 1 + +Melia 1 transcribes audio containing more than one language, including speakers who switch language mid-conversation, and returns a single continuous transcript. It requires `"language": "multi"` and has no language pack selection. You can list the languages you expect as [language hints](/speech-to-text/features/mixed-language-transcription#language-hints) to improve accuracy, which does not restrict the model to those languages. + +For the configuration, see [Models](/speech-to-text/models#melia-1). + +## Bilingual and multi-language packs + +Standard and Enhanced can transcribe a selected combination of languages in one media file or stream, including speakers who switch between the languages in that pack. Each pack covers a fixed set of languages that you select with the `language` property. + +:::note +These packs handle a fixed set of languages that you choose in advance. To transcribe audio without selecting languages, including spontaneous switching across all supported languages, use Melia 1. +::: + +Supported packs are: + +| Language pack | Transcription config | +|---|---| +| Arabic and English | `{"language": "ar_en"}` | +| Malay and English | `{"language": "en_ms"}` | +| Mandarin and English | `{"language": "cmn_en"}` | +| Mandarin Malay Tamil and English | `{"language": "cmn_en_ms_ta"}` | +| Spanish and English | `{"language": "es", "domain": "bilingual-en"}` | +| Tamil and English | `{"language": "en_ta"}` | +| Tagalog (Filipino) and English | `{"language": "tl"}` | + +This config selects the Mandarin and English pack: + +```json +{ + "type": "transcription", + "transcription_config": { + "model": "enhanced", + "language": "cmn_en" + } +} +``` + +This config selects the Spanish and English pack, which requires the `domain` property: + +```json +{ + "type": "transcription", + "transcription_config": { + "model": "enhanced", + "language": "es", + "domain": "bilingual-en" + } +} +``` + +## Translation languages + +Translation is supported for most Speechmatics languages, with the supported translation pairs listed below. For how to configure it, see [Translation](/speech-to-text/add-ons/translation). + +| Audio language | Translation target language | +|---|---| +| English (en) | Bulgarian (bg), Catalan (ca), Mandarin (cmn), Czech (cs), Danish (da), German (de), Greek (el), Spanish (es), Estonian (et), Finnish (fi), French (fr), Galician (gl), Hindi (hi), Croatian (hr), Hungarian (hu), Indonesian (id), Italian (it), Japanese (ja), Korean (ko), Lithuanian (lt), Latvian (lv), Malay (ms), Dutch (nl), Norwegian (no), Polish (pl), Portuguese (pt), Romanian (ro), Russian (ru), Slovakian (sk), Slovenian (sl), Swedish (sv), Turkish (tr), Ukrainian (uk), Vietnamese (vi) | +| Bulgarian (bg), Catalan (ca), Mandarin (cmn), Czech (cs), Danish (da), German (de), Greek (el), Spanish (es), Estonian (et), Finnish (fi), French (fr), Galician (gl), Hindi (hi), Croatian (hr), Hungarian (hu), Indonesian (id), Italian (it), Japanese (ja), Korean (ko), Lithuanian (lt), Latvian (lv), Malay (ms), Dutch (nl), Norwegian (no), Polish (pl), Portuguese (pt), Romanian (ro), Russian (ru), Slovakian (sk), Slovenian (sl), Swedish (sv), Turkish (tr), Ukrainian (uk), Vietnamese (vi) | English (en) | +| Norwegian Bokmål (no) | Norwegian Nynorsk (nn) | diff --git a/docs/speech-to-text/languages/transcription.mdx b/docs/speech-to-text/languages/transcription.mdx new file mode 100644 index 00000000..ed8545be --- /dev/null +++ b/docs/speech-to-text/languages/transcription.mdx @@ -0,0 +1,85 @@ +--- +title: Transcription languages +description: Look up the language code for every language Speechmatics transcribes. +sidebar_label: Language codes +keywords: + [ + speechmatics, + languages, + language codes, + transcription, + speech recognition, + asr, + iso 639, + ] +--- + +Each language is uniquely identified by a two-letter code (ISO 639-1) or three-letter code (ISO 639-3) in API requests and responses. Set the code with the `language` property in your transcription config. + +Your ability to use any or all of these languages depends on the languages you are contracted to use. To read the current language set at runtime rather than from this page, use [feature discovery](/speech-to-text/features/feature-discovery). + +| Language | Language code | Description | +|---|---|---| +| Automatic | `auto` | Detect the language automatically with [language identification](/speech-to-text/pre-recorded/language-identification). Available for pre-recorded transcription. | +| Arabic | `ar` | Global Arabic gives high-accuracy transcription across many accents and dialects, including (but not limited to) Modern Standard Arabic (MSA) and Arabic spoken in the Gulf, Egypt, and the Levant. | +| Arabic & English bilingual | `ar_en` | Ideal when transcribing Arabic and English in the same media file or stream. Supports all accents and dialects listed under Arabic and English. | +| Bashkir | `ba` | | +| Basque | `eu` | | +| Belarusian | `be` | | +| Bengali | `bn` | | +| Bulgarian | `bg` | | +| Cantonese | `yue` | | +| Catalan | `ca` | | +| Croatian | `hr` | | +| Czech | `cs` | | +| Danish | `da` | | +| Dutch | `nl` | | +| English | `en` | Global English gives high-accuracy transcription across many accents, including (but not limited to) English spoken in the United Kingdom, United States, Australia, New Zealand, and by non-native speakers. To standardize spelling, specify the [output locale](/speech-to-text/features/formatting#output-locale). | +| Esperanto | `eo` | | +| Estonian | `et` | | +| Finnish | `fi` | | +| French | `fr` | Global French gives high-accuracy transcription across many accents, including (but not limited to) French spoken in France, Canada, and Belgium. | +| Galician | `gl` | | +| German | `de` | Global German gives high-accuracy transcription across many accents, including (but not limited to) German spoken in Germany, Austria, and Switzerland. | +| Greek | `el` | | +| Hebrew | `he` | | +| Hindi | `hi` | | +| Hungarian | `hu` | | +| Indonesian | `id` | | +| Interlingua | `ia` | | +| Irish | `ga` | | +| Italian | `it` | | +| Japanese | `ja` | | +| Korean | `ko` | | +| Latvian | `lv` | | +| Lithuanian | `lt` | | +| Malay | `ms` | | +| Malay & English bilingual | `en_ms` | Ideal when transcribing Malay and English in the same media file or stream. Supports all accents and dialects listed under Malay and English. | +| Maltese | `mt` | | +| Mandarin | `cmn` | Global Mandarin can output [Traditional or Simplified characters](/speech-to-text/features/formatting#output-locale) and gives high-accuracy transcription across many accents, including (but not limited to) China, Taiwan, Singapore, and Malaysia. | +| Mandarin & English bilingual | `cmn_en` | Ideal when transcribing Mandarin and English in the same media file or stream. Supports all accents and dialects listed under Mandarin and English. | +| Mandarin Malay Tamil & English | `cmn_en_ms_ta` | Ideal when transcribing Mandarin, Malay, Tamil, and English in the same media file or stream. Supports all accents and dialects listed under Mandarin, Malay, Tamil, and English. | +| Marathi | `mr` | | +| Mongolian | `mn` | | +| Norwegian | `no` | | +| Persian | `fa` | | +| Polish | `pl` | | +| Portuguese | `pt` | Global Portuguese gives high-accuracy transcription across many accents, including (but not limited to) Portuguese spoken in Portugal and Brazil. | +| Romanian | `ro` | | +| Russian | `ru` | | +| Slovakian | `sk` | | +| Slovenian | `sl` | | +| Spanish | `es` | Global Spanish gives high-accuracy transcription across many accents, including (but not limited to) Spanish spoken in Spain, the US, Mexico, Colombia, Argentina, Venezuela, Chile, and Peru. | +| Spanish & English bilingual | `es` (with domain=`bilingual-en`) | Ideal when transcribing Spanish and English in the same media file or stream. Supports all accents and dialects listed under English and Spanish. [Requires the domain config to be set](/speech-to-text/languages/#bilingual-and-multi-language-packs). | +| Swahili | `sw` | | +| Swedish | `sv` | | +| Tagalog (Filipino) & English bilingual | `tl` | Ideal when transcribing Tagalog (Filipino) and English in the same media file or stream. Supports all accents and dialects listed under English. | +| Tamil | `ta` | | +| Tamil & English bilingual | `en_ta` | Ideal when transcribing Tamil and English in the same media file or stream. Supports all accents and dialects listed under Tamil and English. | +| Thai | `th` | | +| Turkish | `tr` | | +| Ukrainian | `uk` | | +| Urdu | `ur` | | +| Uyghur | `ug` | | +| Vietnamese | `vi` | | +| Welsh | `cy` | Welsh must be explicitly added to the [expected languages](/speech-to-text/pre-recorded/language-identification#expected-languages) list when using language identification. Otherwise a [language not supported for transcription error](/speech-to-text/pre-recorded/language-identification#language-not-supported-for-transcription) is returned. | diff --git a/docs/speech-to-text/models.mdx b/docs/speech-to-text/models.mdx index 9841d674..433524ea 100644 --- a/docs/speech-to-text/models.mdx +++ b/docs/speech-to-text/models.mdx @@ -1,50 +1,37 @@ --- title: Models -description: Compare the Enhanced, Standard, and Melia 1 models and choose the right one for your audio. +description: Compare the Standard, Enhanced, Melia 1, and Linden 1 models, then set the model in your request. --- -# Models +Speechmatics offers four models for Speech to Text. All four use the same API. The model you choose determines accuracy, how the model handles more than one language, and which features you can configure. -Speechmatics offers three models for Speech to Text: Enhanced, Standard, and Melia 1. All three use the same API. The model you choose determines accuracy, how multilingual audio is handled, and which processing modes and regions are available. +## Choose a model {#compare-the-models} -## Compare the models +Your [interaction pattern](/speech-to-text/interaction-patterns) determines which models you can use, so start there. -| Capability | Enhanced | Standard | Melia 1 | -|---|---|---|---| -| Model | `enhanced` | `standard` | `melia-1` | -| Specialized variant | Medical | — | — | -| Accuracy | Highest | High | High | -| Turnaround | Fast | Fastest | Fastest | -| Processing modes | Batch and Realtime | Batch and Realtime | Batch | -| Regions | EU, US, AUS | EU, US, AUS | EU, US | -| Language handling | [Selected language or pack](/speech-to-text/languages) ([auto-detect](/speech-to-text/batch/language-identification) available) | [Selected language or pack](/speech-to-text/languages) ([auto-detect](/speech-to-text/batch/language-identification) available) | [Automatic multilingual](#melia-1) | -| Diarization | Speaker and channel | Speaker and channel | Speaker and channel | -| Language labeling | — | — | Per word | -| Custom dictionary | ✅ | ✅ | Not yet | -| Confidence scores | ✅ | ✅ | Not yet | -| Speaker identification | ✅ | ✅ | Not yet | -| Speech intelligence | ✅ | ✅ | Not yet | +### Models for pre-recorded transcription {#pre-recorded-models} -Enhanced and Standard are feature-identical and differ only in accuracy and speed: Enhanced delivers the highest accuracy, and Standard prioritizes throughput. Melia 1 matches Standard for accuracy and adds automatic multilingual transcription, but it is available for Batch only and supports a reduced feature set. Speech intelligence covers translation, summarization, topic detection, chapters, sentiment, and audio events. +Standard, Enhanced, and Melia 1, all generally available. -Melia 1 matches the Enhanced and Standard models for core transcription features, including diarization, word timings, punctuation, notifications, and output locale. It does not yet support the following features, which are available with the Enhanced and Standard models: +- **Enhanced** — the highest accuracy on single-language audio, such as medical, legal, or subtitling work. The only model offering the [medical domain](/speech-to-text/features/medical-domain). +- **Standard** — when throughput or latency matter more than maximum accuracy, such as archival transcription, content indexing, or large-scale captioning. +- **Melia 1** — audio that contains more than one language, including speakers who switch language mid-conversation. -- Custom vocabulary and formatting: custom dictionary, find and replace, spoken form output, profanity tagging -- Output detail: confidence scores, entity detection, audio filtering -- Speaker identification -- Speech intelligence: audio events, translation, summarization, chapters, topics, sentiment +To compare them feature by feature, see [pre-recorded transcription features](/speech-to-text/availability/#pre-recorded-transcription-features). -Melia 1 is an early-access model and its feature support is expanding. Check the [release notes](https://speechmatics.featurebase.app/en/changelog) for the latest. +### Models for streaming transcription {#streaming-models} -## Choose a model +Standard, Enhanced, and Melia 1. Standard and Enhanced trade accuracy against throughput exactly as they do for pre-recorded audio, and both are generally available. -Use Enhanced for the highest accuracy on single-language audio, such as medical, legal, or subtitling work. +Melia 1 for streaming is in Preview, on a separate endpoint. See [Melia 1 for streaming](#melia-1-streaming). -Use Standard when throughput or latency matter more than maximum accuracy, such as archival transcription, content indexing, or large-scale captioning. +To compare them feature by feature, see [streaming transcription features](/speech-to-text/availability/#streaming-transcription-features). -Use Melia 1 for audio that contains more than one language, including speakers who switch language mid-conversation. +### Models for agent STT {#agent-stt-models} -For healthcare use cases, use the [Enhanced Medical model](#healthcare-domain) which is tuned for medical terminology. +Linden 1 only, in Preview. Use agent STT for conversational applications that need a turn-based transcript ready to pass to a language model. + +For what it supports, see [agent STT features](/speech-to-text/availability/#agent-stt-features). ## Specify a model @@ -64,11 +51,21 @@ This config selects the `enhanced` model: } ``` -Enhanced and Standard are available for Realtime and Batch transcription. Melia 1 is currently available for Batch transcription. +Linden 1 is the exception: you select it by connecting to an agent STT endpoint rather than by setting `model`. + +## Standard + +Standard prioritizes throughput. It transcribes a single language, selected with a [transcription language pack](/speech-to-text/languages), and it can detect the language automatically. + +## Enhanced + +Enhanced delivers the highest accuracy of the four models. Like Standard, it transcribes a single selected language and can detect the language automatically. + +Enhanced also offers the medical domain, which tunes recognition for healthcare audio. Standard does not. ## Melia 1 -Melia 1 is a multilingual model. It transcribes audio that contains more than one language, including speakers who switch language mid-conversation, and returns a single continuous transcript. It does not require you to select a language pack, and its accuracy is on par with the Standard model. +Melia 1 is a multilingual model. It transcribes audio that contains more than one language, including speakers who switch language mid-conversation, and returns a single continuous transcript. It does not require you to select a language pack, and its accuracy is on par with Standard. Set `"model": "melia-1"` and `"language": "multi"`: @@ -88,60 +85,39 @@ Set `"model": "melia-1"` and `"language": "multi"`: Melia 1 does not support the `auto` language value, which returns an error. Set `language` to `multi`. ::: -Melia 1 is available for Batch transcription in the EU and US regions only. It is not available in the Australia (AU1) region. +Melia 1 has no language pack selection. For pre-recorded transcription on SaaS on Cloud it is served from the EU1 and US1 endpoints: | Region | Endpoint | |---|---| | EU1 (Europe) | `eu1.asr.api.speechmatics.com` | | US1 (USA) | `us1.asr.api.speechmatics.com` | -For the full list of Batch endpoints, refer to [Authentication](/get-started/authentication#supported-endpoints). +For the full list of endpoints, see [Authentication](/get-started/authentication#supported-endpoints). For the regions and features available with Melia 1, see [Feature availability](/speech-to-text/availability/). -For the features Melia 1 does not yet support, refer to [Compare the models](#compare-the-models). +### Melia 1 for streaming {#melia-1-streaming} -To configure language hints and read the per-language output metadata, refer to [Input](/speech-to-text/batch/input#language-hints) and [Output](/speech-to-text/batch/output#multilingual-transcript-output). +:::info Melia 1 for streaming is in Preview +Streaming with Melia 1 is available on SaaS on Cloud for evaluation and feedback. It is not production-ready and not ready to scale. +::: -## Enhanced Medical model {#healthcare-domain} +Streaming with Melia 1 runs on a dedicated Preview endpoint, `wss://preview.rt.speechmatics.com/v2`, and returns a language label on every word. On this endpoint Melia 1 transcribes multilingual audio more accurately than Enhanced, and monolingual audio more accurately than Standard. -Speechmatics offers the Enhanced Medical model that delivers high accuracy on healthcare audio such as ambient scribes and dictation tools. +For the configuration, the supported settings and a runnable example, see [Mixed-language transcription](/speech-to-text/features/mixed-language-transcription). -The Enhanced Medical model is kept up to date using officially maintained data sources. This improves recognition of medical terminology such as procedures, medications, conditions, and anatomy. +## Linden 1 -For languages not listed below, the Enhanced model still delivers high accuracy on healthcare audio without the medical domain. +Linden 1 serves agent STT only. It returns transcripts as complete speaker turns rather than as a word-by-word stream, with turn detection running server-side, which suits applications that hand each turn to a language model. -To use the Enhanced Medical model, set the `domain` property to `medical`: +Linden 1 returns segment-level timings and segment-level partials rather than the word-level equivalents the other three models return. -```json -{ - "type": "transcription", - "transcription_config": { - "model": "enhanced", - "language": "en", - // highlight-start - "domain": "medical" - // highlight-end - } -} -``` +:::info Linden 1 is in Preview +Linden 1 is available on SaaS on Cloud for evaluation and feedback. It is not production-ready and not ready to scale. +::: -| Language | Realtime | Batch | -|---|---|---| -| Arabic English | Available | Available | -| Danish | Available | Available | -| Dutch | Available | Available | -| English | Available | Available | -| Finnish | Available | Available | -| French | Available | Available | -| German | Available | Available | -| Norwegian | Available | Available | -| Spanish | Available | Available | -| Swedish | Available | Available | -| Additional languages | [Contact us](https://www.speechmatics.com/speak-to-sales) | | - -## Operating points +## Deprecated: operating_point The `model` property replaces the `operating_point` property. Existing configs that use `operating_point` continue to transcribe without changes. :::note -In SaaS (cloud) deployments, `operating_point` is deprecated. It maps to `model` and accepts the same `enhanced` and `standard` values. Use [`model`](#specify-a-model) going forward. +On SaaS on Cloud, `operating_point` is deprecated. It maps to `model` and accepts the same `enhanced` and `standard` values. Use [`model`](#specify-a-model) going forward. ::: diff --git a/docs/speech-to-text/batch/assets/check-job-status.sh b/docs/speech-to-text/pre-recorded/assets/check-job-status.sh similarity index 100% rename from docs/speech-to-text/batch/assets/check-job-status.sh rename to docs/speech-to-text/pre-recorded/assets/check-job-status.sh diff --git a/docs/speech-to-text/batch/assets/check-multiple-jobs-status.sh b/docs/speech-to-text/pre-recorded/assets/check-multiple-jobs-status.sh similarity index 100% rename from docs/speech-to-text/batch/assets/check-multiple-jobs-status.sh rename to docs/speech-to-text/pre-recorded/assets/check-multiple-jobs-status.sh diff --git a/docs/speech-to-text/batch/assets/file-transcription-quickstart.example.js b/docs/speech-to-text/pre-recorded/assets/file-transcription-quickstart.example.js similarity index 100% rename from docs/speech-to-text/batch/assets/file-transcription-quickstart.example.js rename to docs/speech-to-text/pre-recorded/assets/file-transcription-quickstart.example.js diff --git a/docs/speech-to-text/batch/assets/file-transcription-quickstart.py b/docs/speech-to-text/pre-recorded/assets/file-transcription-quickstart.py similarity index 96% rename from docs/speech-to-text/batch/assets/file-transcription-quickstart.py rename to docs/speech-to-text/pre-recorded/assets/file-transcription-quickstart.py index d141aebe..2f3a0f4e 100644 --- a/docs/speech-to-text/batch/assets/file-transcription-quickstart.py +++ b/docs/speech-to-text/pre-recorded/assets/file-transcription-quickstart.py @@ -24,7 +24,7 @@ print(f"job {job_id} submitted successfully, waiting for transcript") # Note that in production, you should set up notifications instead of polling. - # Notifications are described here: https://docs.speechmatics.com/speech-to-text/batch/notifications + # Notifications are described here: https://docs.speechmatics.com/speech-to-text/pre-recorded/notifications transcript = client.wait_for_completion(job_id, transcription_format="txt") # To see the full output, try setting transcription_format='json-v2'. print(transcript) diff --git a/docs/speech-to-text/batch/assets/file-transcription-quickstart.sh b/docs/speech-to-text/pre-recorded/assets/file-transcription-quickstart.sh similarity index 100% rename from docs/speech-to-text/batch/assets/file-transcription-quickstart.sh rename to docs/speech-to-text/pre-recorded/assets/file-transcription-quickstart.sh diff --git a/docs/speech-to-text/batch/assets/speechmatics-batch-quickstart.py b/docs/speech-to-text/pre-recorded/assets/speechmatics-batch-quickstart.py similarity index 100% rename from docs/speech-to-text/batch/assets/speechmatics-batch-quickstart.py rename to docs/speech-to-text/pre-recorded/assets/speechmatics-batch-quickstart.py diff --git a/docs/speech-to-text/batch/assets/transcript-response-example.json b/docs/speech-to-text/pre-recorded/assets/transcript-response-example.json similarity index 100% rename from docs/speech-to-text/batch/assets/transcript-response-example.json rename to docs/speech-to-text/pre-recorded/assets/transcript-response-example.json diff --git a/docs/speech-to-text/batch/input.mdx b/docs/speech-to-text/pre-recorded/input.mdx similarity index 80% rename from docs/speech-to-text/batch/input.mdx rename to docs/speech-to-text/pre-recorded/input.mdx index 3840d3d8..9ab692ee 100644 --- a/docs/speech-to-text/batch/input.mdx +++ b/docs/speech-to-text/pre-recorded/input.mdx @@ -42,24 +42,7 @@ Below are the complete fields of the configuration object: ### Language hints -The Melia 1 model detects every language it hears automatically, so language hints are optional. To select Melia 1, refer to [Models](/speech-to-text/models). - -Hints tell the model which languages to expect in the audio, biasing detection toward them. They are most useful for short clips, audio with heavy accents, or recordings where two languages sound similar, where they make language labeling more reliable. - -Provide hints as a list of [supported languages](/speech-to-text/languages#transcription-languages) to guide detection without restricting it. This config hints that the audio contains English and Arabic: - -```json -{ - "type": "transcription", - "transcription_config": { - "model": "melia-1", - "language": "multi", - "language_hints": ["en", "ar"] - } -} -``` - -The model can still detect and label a language you did not hint, and it labels only the languages it actually hears. +The Melia 1 model detects every language it hears automatically. To bias detection toward languages you expect, see [Mixed-language transcription](/speech-to-text/features/mixed-language-transcription#language-hints). ## Fetch URL diff --git a/docs/speech-to-text/batch/language-identification.mdx b/docs/speech-to-text/pre-recorded/language-identification.mdx similarity index 90% rename from docs/speech-to-text/batch/language-identification.mdx rename to docs/speech-to-text/pre-recorded/language-identification.mdx index bcbb092a..bfbf2713 100644 --- a/docs/speech-to-text/batch/language-identification.mdx +++ b/docs/speech-to-text/pre-recorded/language-identification.mdx @@ -22,7 +22,7 @@ You can also learn about deploying this [On-prem](/deployments/container/languag Automatic Language Identification can be set when calling the Speechmatics transcription API. You can also try it for free in the [Speechmatics On-Demand Portal](https://portal.speechmatics.com/signup) with no code. -If you're new to Speechmatics, please see our guide on [Transcribing a File](/speech-to-text/batch/quickstart) through our API. +If you're new to Speechmatics, please see our guide on [Transcribing a File](/speech-to-text/pre-recorded/quickstart) through our API. Once you are set up, just set `language` to `auto` to use Automatic Language Identification: @@ -106,7 +106,7 @@ To configure a job which would use your predefined Default Language: ``` -When getting [Job Details](//api-ref/batch/get-job-details) or [Transcript](/api-ref/batch/get-the-transcript-for-a-transcription-job), the job will succeed and you will see an error message in the job metadata: +When getting [Job Details](/api-ref/batch/get-job-details) or [Transcript](/api-ref/batch/get-the-transcript-for-a-transcription-job), the job will succeed and you will see an error message in the job metadata: ```json { @@ -144,7 +144,7 @@ To configure a job with Default Language: } ``` -When getting [Job Details](//api-ref/batch/get-job-details) or [Transcript](/api-ref/batch/get-the-transcript-for-a-transcription-job), the job will succeed and you will see an error message in the job metadata: +When getting [Job Details](/api-ref/batch/get-job-details) or [Transcript](/api-ref/batch/get-the-transcript-for-a-transcription-job), the job will succeed and you will see an error message in the job metadata: ```json { @@ -210,11 +210,11 @@ The Custom Dictionary will be used with the identified language. Some language-s ### Output locale -[Output Locale](/speech-to-text/formatting#output-locale) is currently not supported in combination with using Automatic Language Identification. Jobs with this combination of features will be rejected. +[Output Locale](/speech-to-text/features/formatting#output-locale) is currently not supported in combination with using Automatic Language Identification. Jobs with this combination of features will be rejected. ### Translation -[Translation](/speech-to-text/features/translation) can be used with Automatic Language Identification. +[Translation](/speech-to-text/add-ons/translation) can be used with Automatic Language Identification. If the identified transcription language and target translation language match, then the translation will contain the transcription sentences. @@ -287,7 +287,7 @@ This error is available when checking the [job details](/api-ref/batch/get-job-d If there is not enough speech detected in the file, and you have not set a [Default Language](#default-language), then the job will be rejected. -This error is available when checking the [job details](//api-ref/batch/get-job-details): +This error is available when checking the [job details](/api-ref/batch/get-job-details): ```json { @@ -314,7 +314,7 @@ If the confidence on the language prediction is too low, and you have not set a This can occur when there is not enough speech in the file or if the file contains multiple languages in similar proportions. -This error is available when checking the [job details](//api-ref/batch/get-job-details): +This error is available when checking the [job details](/api-ref/batch/get-job-details): ```json { "job": { @@ -337,7 +337,7 @@ This error is available when checking the [job details](//api-ref/batch/get-job- If the predicted language cannot be transcribed then the job will be rejected. -This error is available when checking the [job details](//api-ref/batch/get-job-details): +This error is available when checking the [job details](/api-ref/batch/get-job-details): ```json { @@ -362,7 +362,7 @@ This error is available when checking the [job details](//api-ref/batch/get-job- If in the unlikely event the Language Identification stage fails then the job will be rejected. -This error is available when checking the [job details](//api-ref/batch/get-job-details): +This error is available when checking the [job details](/api-ref/batch/get-job-details): ```json { @@ -385,7 +385,7 @@ This error is available when checking the [job details](//api-ref/batch/get-job- ### Errors when used with translation -It is not possible to translate between all language pairs. When `auto` language is used, this can mean some translation target languages will not be available. See the full list of [Supported Language Pairs](/speech-to-text/features/translation#languages). +It is not possible to translate between all language pairs. When `auto` language is used, this can mean some translation target languages will not be available. See the full list of [Supported Language Pairs](/speech-to-text/add-ons/translation#languages). These errors are available when getting the [job transcript](/api-ref/batch/get-the-transcript-for-a-transcription-job): diff --git a/docs/speech-to-text/batch/limits.mdx b/docs/speech-to-text/pre-recorded/limits.mdx similarity index 93% rename from docs/speech-to-text/batch/limits.mdx rename to docs/speech-to-text/pre-recorded/limits.mdx index 560213b6..1bce37a8 100644 --- a/docs/speech-to-text/batch/limits.mdx +++ b/docs/speech-to-text/pre-recorded/limits.mdx @@ -14,7 +14,7 @@ Speechmatics Batch SaaS applies rate limiting and fair queueing to provide a con If you make a large number of requests in a short period of time, some of these requests may fail with the response `HTTP 429 - Rate Limited`. To minimize the possibility of encountering rate limiting errors, we recommend that you do not exceed the following rates: - 10 new jobs per second ( API calls) -- 50 job status requests per second ( API calls). Note that Speechmatics recommends using [Notifications](/speech-to-text/batch/notifications) for job status updates in production +- 50 job status requests per second ( API calls). Note that Speechmatics recommends using [Notifications](/speech-to-text/pre-recorded/notifications) for job status updates in production If you have a large number of concurrent jobs, the most recently submitted jobs may take longer to complete. @@ -27,7 +27,7 @@ This limit is intended as a safety mechanism and is unlikely to be reached durin ## File size limits If you submit your media file in the body of the `/jobs` request to Speechmatics Batch SaaS, the file must be less than 1 GB in size or the job will be rejected. -To process files larger than 1 GB, you can provide the URL of the audio file in the job config as described in the [Fetch URL documentation](/speech-to-text/batch/input#fetch-url). +To process files larger than 1 GB, you can provide the URL of the audio file in the job config as described in the [Fetch URL documentation](/speech-to-text/pre-recorded/input#fetch-url). ## Data retention limits diff --git a/docs/speech-to-text/batch/notifications.md b/docs/speech-to-text/pre-recorded/notifications.md similarity index 99% rename from docs/speech-to-text/batch/notifications.md rename to docs/speech-to-text/pre-recorded/notifications.md index 8177fce4..482a6985 100644 --- a/docs/speech-to-text/batch/notifications.md +++ b/docs/speech-to-text/pre-recorded/notifications.md @@ -84,7 +84,7 @@ Here is the relevant information you can pass in a Notification request: - Callbacks with a single attachment will send the content item as the HTTP request body, rather than using multipart mode. This allows writing an individual item to an object store like Amazon S3 - `auth_headers` should be specified: - To satisfy authentication/authorization requirements for systems that do not support auth tokens in query parameters - - To control behaviour of an object store or another existing service endpoint + - To control behavior of an object store or another existing service endpoint - Multiple callbacks can be specified per job - This allows sending individual pieces of content to different URLs, e.g. to allow uploading the audio and transcript to an object store as distinct objects for a downstream workflow. - It allows sending arbitrary combinations of the inputs/outputs to multiple destinations, to support a fanout workflow diff --git a/docs/speech-to-text/batch/output.mdx b/docs/speech-to-text/pre-recorded/output.mdx similarity index 72% rename from docs/speech-to-text/batch/output.mdx rename to docs/speech-to-text/pre-recorded/output.mdx index b833e027..22108e64 100644 --- a/docs/speech-to-text/batch/output.mdx +++ b/docs/speech-to-text/pre-recorded/output.mdx @@ -20,15 +20,15 @@ import { LinkCard } from "@site/src/theme/LinkCard"; # Output -Transcription jobs are processed asynchronously by default: you submit a job, then check its status to see whether it has completed. To block for the result in a single request instead, use [Synchronous transcription](/speech-to-text/batch/synchronous). +Transcription jobs are processed asynchronously by default: you submit a job, then check its status to see whether it has completed. To block for the result in a single request instead, use [Synchronous transcription](/speech-to-text/pre-recorded/synchronous). :::tip -You can also configure notifications to be sent to a webhook when a job is completed. See [Notifications](/speech-to-text/batch/notifications) for more details. +You can also configure notifications to be sent to a webhook when a job is completed. See [Notifications](/speech-to-text/pre-recorded/notifications) for more details. ::: ## Check single job status -If you wish to retrieve a particular job, you can do so using the job ID for up to 7 days, after which time it will be automatically deleted in accordance with our [Data Retention Policy](/speech-to-text/batch/limits#data-retention-limits). +If you wish to retrieve a particular job, you can do so using the job ID for up to 7 days, after which time it will be automatically deleted in accordance with our [Data Retention Policy](/speech-to-text/pre-recorded/limits#data-retention-limits). You can make a GET request to check the status of a job as follows: @@ -49,7 +49,7 @@ You can make a GET request to check the status of a job as follows:
-This endpoint applies a short default wait, returning as soon as the job reaches a terminal state. Pass the `wait` query parameter to control the duration, or `wait=0` to return immediately. See [Synchronous transcription](/speech-to-text/batch/synchronous#default-wait-on-the-get-endpoints). +This endpoint applies a short default wait, returning as soon as the job reaches a terminal state. Pass the `wait` query parameter to control the duration, or `wait=0` to return immediately. See [Synchronous transcription](/speech-to-text/pre-recorded/synchronous#default-wait-on-the-get-endpoints). The response is a JSON object containing details of the job, with the `status` field showing whether the job is still processing or not. The possible values are: @@ -116,9 +116,9 @@ A few useful things to know about transcript formats: - The default format is JSON. - Use the `format=txt` query parameter to get the transcript in plain text. Useful for quick access to the transcript. - Use the `format=srt` query parameter to get the transcript in SRT format. Useful for displaying the transcript in a subtitle file. -- To access other data, including word timestamps, translations, and speech intelligence features, use the default JSON format. +- To access other data, including word timestamps, translations, and add-on outputs, use the default JSON format. -Like the job status endpoint, this endpoint applies a short default wait. Pass the `wait` query parameter alongside `format` to block for the transcript instead of polling. See [Synchronous transcription](/speech-to-text/batch/synchronous#wait-for-the-transcript). +Like the job status endpoint, this endpoint applies a short default wait. Pass the `wait` query parameter alongside `format` to block for the transcript instead of polling. See [Synchronous transcription](/speech-to-text/pre-recorded/synchronous#wait-for-the-transcript). ### Transcript response schema @@ -130,7 +130,7 @@ Refer to the [API reference](/api-ref/batch/get-the-transcript-for-a-transcripti ### Example response -The following is an example of a transcript response, which you should see as an output of the provided [example.wav](https://github.com/speechmatics/speechmatics-js-sdk/raw/7d219bfee9166736e6aa21598535a194387b84be/examples/nodejs/example.wav) file used in the [code samples in the quickstart](/speech-to-text/batch/quickstart). +The following is an example of a transcript response, which you should see as an output of the provided [example.wav](https://github.com/speechmatics/speechmatics-js-sdk/raw/7d219bfee9166736e6aa21598535a194387b84be/examples/nodejs/example.wav) file used in the [code samples in the quickstart](/speech-to-text/pre-recorded/quickstart). {JSON.stringify(transcriptResponseExample, null, 2)} @@ -138,53 +138,7 @@ The following is an example of a transcript response, which you should see as an ### Multilingual transcript output -For a Melia 1 job, the `language` property on each word reflects the language detected for that word, so it can change across the transcript. For Enhanced and Standard jobs, which transcribe one selected language, the same language is reported for every word. - -The example below shows two words in different languages within one transcript: - -```json -{ - "results": [ - { - "alternatives": [ - { "content": "Hello", "confidence": 0.98, "language": "en" } - ], - "start_time": 0.20, - "end_time": 0.52, - "type": "word" - }, - { - "alternatives": [ - { "content": "مرحبا", "confidence": 0.95, "language": "ar" } - ], - "start_time": 0.60, - "end_time": 1.04, - "type": "word" - } - ] -} -``` - -For multilingual transcripts, `language_pack_info` reports the word delimiter and writing direction per language rather than for a single language pack: - -```json -{ - "metadata": { - "language_pack_info": { - "per_language_word_delimiters": { - "en": " ", - "ar": " " - }, - "per_language_writing_direction": { - "en": "left-to-right", - "ar": "right-to-left" - } - } - } -} -``` - -`per_language_word_delimiters` gives the word delimiter for each language in the transcript, and `per_language_writing_direction` gives its writing direction. +For a Melia 1 job, the `language` property on each word reflects the language detected for that word. See [Mixed-language transcription](/speech-to-text/features/mixed-language-transcription#multilingual-transcript-output). ## Tracking metadata @@ -231,23 +185,7 @@ For example: ## App usage tracking -:::tip -First, please contact [Support](https://support.speechmatics.com) to enable this feature. -::: - -For integrations where customers use their own Speechmatics API keys, gain insights into application usage by aggregating data on unique users, processing hours, languages used, and more. -Once enabled, use the `sm-app` query parameter when starting a job. For example: - -```bash -APP_ID="YourAppID" -API_KEY="YOUR_API_KEY" -PATH_TO_FILE="example.wav" - -curl -L -X POST "https://eu1.asr.api.speechmatics.com/v2/jobs/?sm-app=${APP_ID}" \ - -H "Authorization: Bearer ${API_KEY}" \ - -F data_file=@${PATH_TO_FILE} \ - -F config='{"type": "transcription","transcription_config": { "model": "enhanced","language": "en" }}' -``` +To attribute usage across applications, add an app ID to your requests. See [App usage tracking](/speech-to-text/features/app-usage-tracking). ## Next steps @@ -255,13 +193,13 @@ curl -L -X POST "https://eu1.asr.api.speechmatics.com/v2/jobs/?sm-app=${APP_ID}" } /> } /> } /> } /> diff --git a/docs/speech-to-text/batch/quickstart.mdx b/docs/speech-to-text/pre-recorded/quickstart.mdx similarity index 93% rename from docs/speech-to-text/batch/quickstart.mdx rename to docs/speech-to-text/pre-recorded/quickstart.mdx index f5563b53..6ae96431 100644 --- a/docs/speech-to-text/batch/quickstart.mdx +++ b/docs/speech-to-text/pre-recorded/quickstart.mdx @@ -97,7 +97,7 @@ Now that you have a basic transcription working, explore these features to get m } /> } /> } /> } /> } /> diff --git a/docs/speech-to-text/pre-recorded/sidebar.ts b/docs/speech-to-text/pre-recorded/sidebar.ts new file mode 100644 index 00000000..6c15aed9 --- /dev/null +++ b/docs/speech-to-text/pre-recorded/sidebar.ts @@ -0,0 +1,59 @@ +export default { + type: "category", + label: "Pre-recorded transcription", + items: [ + { + type: "doc", + id: "speech-to-text/pre-recorded/quickstart", + }, + { + // Anchor link: sidebar hrefs are not checked by onBrokenLinks, so the + // target is verified against the built HTML instead. + type: "link", + href: "/speech-to-text/models#pre-recorded-models", + label: "Choosing a model", + }, + { + type: "doc", + id: "speech-to-text/pre-recorded/input", + }, + { + type: "doc", + id: "speech-to-text/pre-recorded/output", + }, + { + type: "doc", + id: "speech-to-text/pre-recorded/synchronous", + }, + { + type: "doc", + id: "speech-to-text/pre-recorded/limits", + }, + { + type: "doc", + id: "speech-to-text/pre-recorded/notifications", + }, + { + type: "doc", + id: "speech-to-text/pre-recorded/language-identification", + }, + { + type: "doc", + id: "speech-to-text/pre-recorded/srt-format", + }, + // TODO see about usage requests + // { + // type: "doc", + // id: "speech-to-text/pre-recorded/usage-requests", + // }, + { + type: "doc", + id: "speech-to-text/pre-recorded/troubleshooting", + }, + { + type: "link", + href: "/api-ref/batch/create-a-new-job", + label: "API reference", + }, + ], +} as const; diff --git a/docs/speech-to-text/batch/srt-format.mdx b/docs/speech-to-text/pre-recorded/srt-format.mdx similarity index 100% rename from docs/speech-to-text/batch/srt-format.mdx rename to docs/speech-to-text/pre-recorded/srt-format.mdx diff --git a/docs/speech-to-text/batch/synchronous.mdx b/docs/speech-to-text/pre-recorded/synchronous.mdx similarity index 98% rename from docs/speech-to-text/batch/synchronous.mdx rename to docs/speech-to-text/pre-recorded/synchronous.mdx index c6407479..8b6ca20d 100644 --- a/docs/speech-to-text/batch/synchronous.mdx +++ b/docs/speech-to-text/pre-recorded/synchronous.mdx @@ -134,5 +134,5 @@ curl -L -X GET "https://eu1.asr.api.speechmatics.com/v2/jobs/${JOB_ID}?wait=0" \ ## Next steps -- [Output formats](/speech-to-text/batch/output) — load and process the transcript. +- [Output formats](/speech-to-text/pre-recorded/output) — load and process the transcript. - [API reference](/api-ref/batch/create-a-new-job) — full request and response details. diff --git a/docs/speech-to-text/batch/troubleshooting.mdx b/docs/speech-to-text/pre-recorded/troubleshooting.mdx similarity index 100% rename from docs/speech-to-text/batch/troubleshooting.mdx rename to docs/speech-to-text/pre-recorded/troubleshooting.mdx diff --git a/docs/speech-to-text/realtime/assets/microphone-example.py b/docs/speech-to-text/realtime/assets/microphone-example.py deleted file mode 100644 index 58a7107f..00000000 --- a/docs/speech-to-text/realtime/assets/microphone-example.py +++ /dev/null @@ -1,140 +0,0 @@ -import speechmatics -from httpx import HTTPStatusError -import asyncio -import pyaudio - -API_KEY = "YOUR_API_KEY" -LANGUAGE = "en" -CONNECTION_URL = f"wss://eu.rt.speechmatics.com/v2/{LANGUAGE}" -DEVICE_INDEX = -1 -CHUNK_SIZE = 1024 - - -class AudioProcessor: - def __init__(self): - self.wave_data = bytearray() - self.read_offset = 0 - - async def read(self, chunk_size): - while self.read_offset + chunk_size > len(self.wave_data): - await asyncio.sleep(0.001) - new_offset = self.read_offset + chunk_size - data = self.wave_data[self.read_offset : new_offset] - self.read_offset = new_offset - return data - - def write_audio(self, data): - self.wave_data.extend(data) - return - - -audio_processor = AudioProcessor() - - -# PyAudio callback -def stream_callback(in_data, frame_count, time_info, status): - audio_processor.write_audio(in_data) - return in_data, pyaudio.paContinue - - -# Set up PyAudio -p = pyaudio.PyAudio() -if DEVICE_INDEX == -1: - DEVICE_INDEX = p.get_default_input_device_info()["index"] - device_name = p.get_default_input_device_info()["name"] - DEF_SAMPLE_RATE = int(p.get_device_info_by_index(DEVICE_INDEX)["defaultSampleRate"]) - print( - f"***\nIf you want to use a different microphone, update DEVICE_INDEX at the start of the code to one of the following:" - ) - # Filter out duplicates that are reported on some systems - device_seen = set() - for i in range(p.get_device_count()): - if p.get_device_info_by_index(i)["name"] not in device_seen: - device_seen.add(p.get_device_info_by_index(i)["name"]) - try: - supports_input = p.is_format_supported( - DEF_SAMPLE_RATE, - input_device=i, - input_channels=1, - input_format=pyaudio.paFloat32, - ) - except Exception: - supports_input = False - if supports_input: - print( - f"-- To use << {p.get_device_info_by_index(i)['name']} >>, set DEVICE_INDEX to {i}" - ) - print("***\n") - -SAMPLE_RATE = int(p.get_device_info_by_index(DEVICE_INDEX)["defaultSampleRate"]) -device_name = p.get_device_info_by_index(DEVICE_INDEX)["name"] - -print(f"\nUsing << {device_name} >> which is DEVICE_INDEX {DEVICE_INDEX}") -print("Starting transcription (type Ctrl-C to stop):") - -stream = p.open( - format=pyaudio.paFloat32, - channels=1, - rate=SAMPLE_RATE, - input=True, - frames_per_buffer=CHUNK_SIZE, - input_device_index=DEVICE_INDEX, - stream_callback=stream_callback, -) - -# Define connection parameters -conn = speechmatics.models.ConnectionSettings( - url=CONNECTION_URL, - auth_token=API_KEY, -) - -# Create a transcription client -ws = speechmatics.client.WebsocketClient(conn) - -# Define transcription parameters -# Full list of parameters described here: https://speechmatics.github.io/speechmatics-python/models -conf = speechmatics.models.TranscriptionConfig( - language=LANGUAGE, - enable_partials=True, - model="enhanced", - max_delay=1, -) - - -# Define an event handler to print the partial transcript -def print_partial_transcript(msg): - print(f"[partial] {msg['metadata']['transcript']}") - - -# Define an event handler to print the full transcript -def print_transcript(msg): - print(f"[ FINAL] {msg['metadata']['transcript']}") - - -# Register the event handler for partial transcript -ws.add_event_handler( - event_name=speechmatics.models.ServerMessageType.AddPartialTranscript, - event_handler=print_partial_transcript, -) - -# Register the event handler for full transcript -ws.add_event_handler( - event_name=speechmatics.models.ServerMessageType.AddTranscript, - event_handler=print_transcript, -) - -settings = speechmatics.models.AudioSettings() -settings.encoding = "pcm_f32le" -settings.sample_rate = SAMPLE_RATE -settings.chunk_size = CHUNK_SIZE - -print("Starting transcription (type Ctrl-C to stop):") -try: - ws.run_synchronously(audio_processor, conf, settings) -except KeyboardInterrupt: - print("\nTranscription stopped.") -except HTTPStatusError as e: - if e.response.status_code == 401: - print("Invalid API key - Check your API_KEY at the top of the code!") - else: - raise e diff --git a/docs/speech-to-text/realtime/guides/python-using-microphone.mdx b/docs/speech-to-text/realtime/guides/python-using-microphone.mdx deleted file mode 100644 index ca85613c..00000000 --- a/docs/speech-to-text/realtime/guides/python-using-microphone.mdx +++ /dev/null @@ -1,55 +0,0 @@ ---- -description: Use the Speechmatics Python library to transcribe your voice using a microphone. ---- - -import Admonition from '@theme/Admonition'; -import CodeBlock from '@theme/CodeBlock'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import microphonePythonExample from '../assets/microphone-example.py'; - -# Python microphone input - -The example below will help you transcribe your voice by using the official Speechmatics Python library and CLI. - -You can of course integrate with Speechmatics using our [Official JavaScript SDK](https://github.com/speechmatics/speechmatics-js-sdk), or in the programming language of your choice by referring to the [Realtime API Reference](/api-ref/realtime-transcription-websocket.mdx). - -### Setup - -The Speechmatics Python library and CLI can be installed using pip: - -```bash -pip3 install speechmatics-python -``` - -Transcribe your voice in real-time using the Speechmatics Python library. Just copy in your API key to get started!

- -In order to use this script, you may also need to install PyAudio by running: - -```bash -pip3 install pyaudio -``` - -:::info - For Mac M1/M2, use the following instructions to install PyAudio: - ```bash - brew install portaudio - brew link portaudio - BREW_PREFIX=$(brew --prefix) - CFLAGS="-I$BREW_PREFIX/include -L$BREW_PREFIX/lib" python3 -m pip install pyaudio - ``` -::: - - -### Code example - - -{microphonePythonExample} - - - -## Enhanced voice AI features - -For building voice AI applications, translation systems, or dictation software with microphone input, consider enabling [End of Utterance Detection](/speech-to-text/realtime/turn-detection#configuration). This feature detects when users finish speaking by monitoring silence periods, enabling natural turn-taking and responsive voice applications. - -The end-of-utterance feature works perfectly with microphone streaming and can be added to the configuration above by including `conversation_config` with a silence threshold parameter. diff --git a/docs/speech-to-text/realtime/realtime-diarization.mdx b/docs/speech-to-text/realtime/realtime-diarization.mdx deleted file mode 100644 index fa6cf296..00000000 --- a/docs/speech-to-text/realtime/realtime-diarization.mdx +++ /dev/null @@ -1,350 +0,0 @@ ---- -sidebar_label: 'Realtime diarization' -description: "Learn how to use the Speechmatics API to separate speakers in real-time" -keywords: - [ - speechmatics, - realtime, - diarization, - channel diarization, - speaker diarization, - transcription, - speech recognition, - automatic speech recognition, - asr, - ] ---- - -import DocCardList from '@theme/DocCardList'; -import { Card, DataList, Text } from '@radix-ui/themes'; - -# Realtime diarization - -:::tip -To learn more about diarization as a feature, check out the [diarization](../features/diarization.mdx) page. -::: - -## Overview - -Realtime diarization offers the following ways to separate speakers in audio: - -- [**Speaker diarization**](#speaker-diarization) — Identifies each speaker by their voice. - Useful when there are multiple speakers in the same audio stream. - -- [**Channel diarization**](#channel-diarization) — Transcribes each audio channel separately. - Useful when each speaker is recorded on their own channel. - -- [**Channel & speaker diarization**](#channel-and-speaker-diarization) — Combines both methods. - Each channel is transcribed separately, with unique speakers identified within each channel. - Useful when multiple speakers are present across multiple channels. - -## Speaker diarization - - -Speaker diarization picks out different speakers from the audio stream based on acoustic matching. - -To enable Speaker diarization, `diarization` must be set to `speaker` in the transcription config: - -```json -{ - "type": "transcription", - "transcription_config": { - "model": "enhanced", - "language": "en", - // highlight-start - "diarization": "speaker" - // highlight-end - } -} -``` - -When diarization is enabled, each `word` and `punctuation` object in the transcript includes a `speaker` property that identifies who spoke it. There are two types of labels: - -- `S#` – S stands for speaker, and `#` is a sequential number identifying each speaker. S1 appears first in the results, followed by S2, S3, and so on. -- `UU` – Used when the speaker cannot be identified or diarization is not applied, for example, if background noise is transcribed as speech but no speaker can be determined. - -```json - "results": [ - { - "alternatives": [ - { - "confidence": 0.93, - "content": "hello", - "language": "en", - // highlight-start - "speaker": "S1" - // highlight-end - } - ], - }, - { - "alternatives": [ - { - "confidence": 1.0, - "content": "hi", - "language": "en", - // highlight-start - "speaker": "S2" - // highlight-end - } - ], - }] -``` - -## Channel diarization - -Channel diarization processes audio with multiple channels and returns a separate transcript for each one. This gives you perfect speaker separation at the channel level and more accurate handling of cross-talk. - -To enable channel diarization, `diarization` must be set to `channel` and labels for each channel provided in `channel_diarization_labels` in the transcription config of the `StartRecognition` message: - -```json -{ - "type": "transcription", - "transcription_config": { - "model": "enhanced", - "language": "en", - // highlight-start - "diarization": "channel", - "channel_diarization_labels": ["New_York", "Shanghai"] - // highlight-end - } -} -``` - -You should see a `channels` field in the `RecognitionStarted` message which lists all the channels you requested: - -```json -{ - "message": "RecognitionStarted", - ... - // highlight-start - "channels": ["New_York", "Shanghai"] - // highlight-end -} -``` -### Send audio to a channel - -To send audio for a specific channel, you can use the `AddChannelAudio` message. You'll need to encode the data in base64 format: - -```json -{ - "message": "AddChannelAudio", - "channel": "New_York", - "data": -} -``` - -You should get an acknowledgement in the form of a `ChannelAudioAdded` message from the server, with a corresponding sequence number for the channel: - -```json -{ - "message": "ChannelAudioAdded", - "channel": "New_York", - "seq_no": <10> -} -``` - -### Transcript response - -Transcripts are returned independently for each channel, with the `channel` property identifying the channel. - -```json -{ - "message": "AddTranscript", - // highlight-start - "channel": "New_York", - // highlight-end - ... - "results": [ - { - "type": "word", - "start_time": 1.45, - "end_time": 1.8, - "alternatives": [{ - "language": "en", - "content": "Hello,", - "confidence": 0.98, - }] - }, - ] -} -``` - -:::warning -The `channel` property will be returned for `AddTranscript` and `AddPartialTranscript` messages only. -The [translation](/speech-to-text/features/translation) feature does not currently include this property. To request this feature, please contact [support](https://support.speechmatics.com). -::: - -### Channel and speaker diarization - -Channel and speaker diarization combines speaker diarization and channel diarization, splitting transcripts per channel whilst also separating individual speakers in each channel. - -To enable this mode, follow the steps in [speaker diarization](#speaker-diarization) and set the `diarization` mode to `channel_and_speaker`. - -To send audio to a channel, follow the instructions in [send audio to a channel](#send-audio-to-a-channel). - -Transcripts are returned in the same way as channel diarization, but with individual speakers identified: - -```json -{ - "message": "AddTranscript", - // highlight-start - "channel": "New_York", - // highlight-end - "results": [ - { - "alternatives": [{ - "content": "Hello", - "confidence": 0.98, - // highlight-start - "speaker": 'S1', - // highlight-end - }] - }, - ... - { - "alternatives": [{ - "content": "Hi", - "confidence": 0.98, - // highlight-start - "speaker": 'S2', - // highlight-end - }] - }, - ] -} -``` - -When using `channel_and_speaker` diarization, speaker labelling is specific to channels even if the speaker labels are the same. S1 on channel 1 is not necessarily the same as S1 on channel 2. - -## Closing channels - -When you're finished with a channel, you can signal that it's no longer in use by sending an [EndOfChannel](../../api-ref/realtime-transcription-websocket#endofchannel) message. -```json -{ - // highlight-start - "message": "EndOfChannel", - "channel": "New_York", - "last_seq_no": 2564 - // highlight-end -} -``` - -Once this happens, the channel will stop accepting or processing any additional data. When all channels are closed, the session ends. This may be more convenient to use when you don't want to keep track of the number of open channels in a given stream. - -You can also use [EndOfStream](../../api-ref/realtime-transcription-websocket#endofstream) to simultaneously close all channels. This may be more convenient to use when you know you want to close all channels at the same time. - -:::info -At this time, closing an individual channel has no impact on pricing. -::: - -### Limits - -For SaaS customers, the maximum number of channels is 2. - -For On-prem Container customers, the maximum number of channels depends on your [Multi-session container's](../../deployments/container/cpu-speech-to-text.mdx#multi-session-containers) maximum number of connections. - -The Speechmatics Python client CLI is currently limited to transcribing multi-channel audio in via files and not streaming/raw audio. - -## Configuration - -You can customize diarization to match your use case by adjusting settings for sensitivity, limiting the maximum number of speakers, preferring the current speaker to reduce false switches, and controlling how punctuation influences accuracy. - -### Speaker sensitivity - - -You can configure the sensitivity of speaker detection by using the `speaker_sensitivity` setting in the `speaker_diarization_config` section of the job config object as shown below: - -```json -{ - "type": "transcription", - "transcription_config": { - "model": "enhanced", - "language": "en", - // highlight-start - "diarization": "speaker", - "speaker_diarization_config": { - "speaker_sensitivity": 0.6 - } - // highlight-end - } -} -``` - -This takes a value between 0 and 1 (the default is 0.5). A higher sensitivity will -increase the likelihood of more unique speakers returning. - -### Prefer current speaker - -You can reduce the likelihood of incorrectly switching between similar sounding speakers by setting the `prefer_current_speaker` flag in the `speaker_diarization_config`: - -```json -{ - "type": "transcription", - "transcription_config": { - "model": "enhanced", - "language": "en", - // highlight-start - "diarization": "speaker", - "speaker_diarization_config": { - "prefer_current_speaker": true - } - // highlight-end - } -} -``` -By default this is `false`. When this is set to `true`, the system will stay with the speaker of the previous word, if they closely match the speaker of the new word. - -This may result in some shorter speaker turn changes between similar speakers being missed. - -### Max. speakers - -You can prevent too many speakers from being detected by using the `max_speakers` setting in the `StartRecognition` message as shown below: - -```json -{ - "message": "StartRecognition", - "audio_format": { - "type": "raw", - "encoding": "pcm_f32le", - "sample_rate": 48000 - }, - "transcription_config": { - "language": "en", - "model": "enhanced", - // highlight-start - "diarization": "speaker", - "speaker_diarization_config": { - "max_speakers": 10 - } - // highlight-end - } -} -``` - -By default, there is no limit on the number of speakers. When set explicitly, `max_speakers` can be set to any integer greater than or equal to 2. - -### Punctuation - -Speaker diarization uses punctuation to improve accuracy. Small corrections are applied to speaker labels based on sentence boundaries. - -For example, if the system initially assigns 9 words in a sentence to S1 and 1 word to S2, the lone S2 word may be corrected to S1. - -This adjustment only works when punctuation is enabled. Disabling punctuation via the `permitted_marks` setting in `punctuation_overrides` can reduce diarization accuracy. - -Adjusting punctuation sensitivity can also affect how accurately speakers are identified. - -### Speaker change (legacy) - -The Speaker Change Detection feature was removed in July 2024. The `speaker_change` and `channel_and_speaker_change` parameters are no longer supported. Use the [Speaker diarization](#speaker-diarization) feature for speaker labeling. - -For API-related questions, contact [support](https://support.speechmatics.com). - -## On-prem - -To run `channel` or `channel_and_speaker` diarization with an on-prem deployment, configure your environment as follows: - -- Use a [GPU Speech-to-Text container](../../deployments/container/gpu-speech-to-text.mdx). Handling multiple audio streams is computationally intensive and benefits from GPU acceleration. -- Set the `SM_MAX_CONCURRENT_CONNECTIONS` environment variable to match the number of channels you want to process. - -For more details on container setup, see the [on-prem deployment docs](../../deployments/index.md). diff --git a/docs/speech-to-text/realtime/sidebar.ts b/docs/speech-to-text/realtime/sidebar.ts deleted file mode 100644 index a791a8eb..00000000 --- a/docs/speech-to-text/realtime/sidebar.ts +++ /dev/null @@ -1,49 +0,0 @@ -export default { - type: "category", - label: "Realtime transcription", - items: [ - { - type: "doc", - id: "speech-to-text/realtime/quickstart", - }, - { - type: "doc", - id: "speech-to-text/realtime/input", - }, - { - type: "doc", - id: "speech-to-text/realtime/output", - }, - { - type: "doc", - id: "speech-to-text/realtime/limits", - }, - { - type: "doc", - id: "speech-to-text/realtime/realtime-diarization", - }, - { - type: "doc", - id: "speech-to-text/realtime/speaker-identification", - }, - { - type: "doc", - id: "speech-to-text/realtime/turn-detection", - }, - { - type: "category", - label: "Guides", - items: [ - { - type: "autogenerated", - dirName: "speech-to-text/realtime/guides", - }, - ], - }, - { - type: "link", - label: "API reference", - href: "/api-ref/realtime-transcription-websocket", - }, - ], -} as const; diff --git a/docs/speech-to-text/realtime/speaker-identification.mdx b/docs/speech-to-text/realtime/speaker-identification.mdx deleted file mode 100644 index 07fbbd48..00000000 --- a/docs/speech-to-text/realtime/speaker-identification.mdx +++ /dev/null @@ -1,174 +0,0 @@ ---- -sidebar_label: 'Speaker identification' -description: "Learn how to use the Speechmatics API to identify speakers in real-time" -keywords: - [ - speechmatics, - realtime, - speaker identification, - diarization, - speaker diarization, - transcription, - speech recognition, - automatic speech recognition, - asr, - ] ---- - -import DocCardList from '@theme/DocCardList'; -import { Card, DataList, Text } from '@radix-ui/themes'; -import CodeBlock from "@theme/CodeBlock"; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import speakerIdEnrollmentPythonExample from "./assets/speaker-id-enrollment-file-example.py" -import speakerIdIdentificationPythonExample from "./assets/speaker-id-identification-file-example.py" - -# Realtime speaker identification - -:::tip -For an overview of the feature, see the [speaker identification](/speech-to-text/features/speaker-identification) page. -::: - -## Enrollment - -To generate identifiers for a desired speaker, run a [speaker diarization](/speech-to-text/features/diarization#diarization-modes) enabled transcription on an audio sample where the speaker is ideally speaking alone. -You can request the identifiers back from the engine by sending a `GetSpeakers` request. - -By default, the engine returns identifiers created up to the time of the request, but you can also wait until the end of the stream by setting the optional `final` flag in the `GetSpeakers` request: - -```json -{ - "message": "GetSpeakers", - "final": true -} -``` - -- final: false (default) — returns identifiers generated up to the point of the request. To avoid empty results, wait until the server has issued at least one `AddTranscript` message before sending the request. -- final: true — waits until the end of the stream and returns identifiers based on all audio. - -Alternatively, you can enable automatic speaker retrieval by setting the `get_speakers` option to true in the diarization configuration (recommended for enrollment). This guarantees that the engine automatically provides speaker identifiers once the transcription is complete, equivalent to manually calling `GetSpeakers(final=true)`. If the `get_speakers` option is not present in the configuration or is set to false, you can still request speakers explicitly by sending the `GetSpeakers(final=true)` message. In this case, the request takes precedence, and the engine will return the speaker identifiers at the end of the transcription. - -Example speaker diarization config: - -```json -{ - "type": "transcription", - "transcription_config": { - "model": "enhanced", - "language": "en", - "diarization": "speaker", - "speaker_diarization_config": { - // highlight-start - "get_speakers": true - // highlight-end - } - } -} -``` - -When the request is processed, the server replies with a `SpeakersResult` message that contains the identifiers for each diarized speaker: - -```json -{ - "message": "SpeakersResult", - "speakers": [ - {"label": "S1", "speaker_identifiers": [""]}, - {"label": "S2", "speaker_identifiers": [""]} - ] -} -``` - -## Identification - -Once you've generated speaker identifiers, you can provide them in your next transcription job to identify and tag known speakers. This is done through the `speakers` option in the speaker diarization configuration. - -All [speaker diarization options](/speech-to-text/realtime/realtime-diarization#configuration) work with speaker identification. The `max_speakers` parameter only applies to generic (non-enrolled) speakers. For example, if it’s set to 10 and 10 speakers are enrolled, the system can still add up to 10 additional generic speakers. The `speakers_sensitivity` parameter can also be used to adjust how strongly the system prefers enrolled speakers over detecting new generic ones. Lower values make it more likely to match existing enrolled speakers. - -An example configuration is shown below: - -```json -{ - "type": "transcription", - "transcription_config": { - "model": "enhanced", - "language": "en", - "diarization": "speaker", - "speaker_diarization_config": { - // highlight-start - "speakers": [ - {"label": "Alice", "speaker_identifiers": ["", ""]}, - {"label": "Bob", "speaker_identifiers": [""]} - ] - // highlight-end - } - } -} -``` - -With the config above, transcript segments should be tagged with `"Alice"` and `"Bob"` whenever these speakers are detected, whereas any other speakers should be tagged with the internal labels: - -```json -{ - "results": [ - { - "alternatives": [ - { - "confidence": 1.0, - "content": "Hello", - "language": "en", - // highlight-start - "speaker": "Alice" - // highlight-end - } - ] - }, - { - "alternatives": [ - { - "confidence": 1.0, - "content": "Hi", - "language": "en", - // highlight-start - "speaker": "S1" - // highlight-end - } - ] - }, - { - "alternatives": [ - { - "confidence": 1.0, - "content": "Nice", - "language": "en", - // highlight-start - "speaker": "Bob" - // highlight-end - } - ] - } - ] -} -``` - -## Code examples - - - - - Realtime speakers enrollment example. - - - {speakerIdEnrollmentPythonExample} - - - - - - Realtime speakers identification example. - - - {speakerIdIdentificationPythonExample} - - - - diff --git a/docs/speech-to-text/sidebar.ts b/docs/speech-to-text/sidebar.ts index c5be0cb6..cd5e7d4e 100644 --- a/docs/speech-to-text/sidebar.ts +++ b/docs/speech-to-text/sidebar.ts @@ -1,5 +1,6 @@ -import batchSidebar from "./batch/sidebar"; -import realtimeSidebar from "./realtime/sidebar"; +import agentSttSidebar from "./agent-stt/sidebar"; +import preRecordedSidebar from "./pre-recorded/sidebar"; +import streamingSidebar from "./streaming/sidebar"; export default { type: "category", @@ -12,17 +13,46 @@ export default { id: "speech-to-text/index", label: "Overview", }, - realtimeSidebar, - batchSidebar, + { + type: "doc", + id: "speech-to-text/interaction-patterns", + }, + // Order follows the Level 1 sequence in product-architecture.md: + // pre-recorded, streaming, then agent STT. + preRecordedSidebar, + streamingSidebar, + agentSttSidebar, { type: "doc", id: "speech-to-text/models", }, { type: "doc", - id: "speech-to-text/languages", + id: "speech-to-text/languages/index", + }, + { + type: "doc", + id: "speech-to-text/languages/transcription", + }, + { + // Declared explicitly rather than autogenerated: sidebar-generator.ts + // title-cases autogenerated category labels word by word. + type: "category", + label: "Availability", + items: [ + { + type: "doc", + id: "speech-to-text/availability/index", + }, + { + type: "doc", + id: "speech-to-text/availability/on-prem", + }, + ], }, { + // Declared explicitly rather than autogenerated: sidebar-generator.ts + // title-cases autogenerated category labels word by word. type: "category", label: "Features", items: [ @@ -33,9 +63,14 @@ export default { ], }, { - type: "doc", - label: "Formatting", - id: "speech-to-text/formatting", + type: "category", + label: "Add-ons", + items: [ + { + type: "autogenerated", + dirName: "speech-to-text/add-ons", + }, + ], }, { type: "doc", diff --git a/docs/speech-to-text/realtime/assets/end-of-utterance-file-example.py b/docs/speech-to-text/streaming/assets/end-of-utterance-file-example.py similarity index 100% rename from docs/speech-to-text/realtime/assets/end-of-utterance-file-example.py rename to docs/speech-to-text/streaming/assets/end-of-utterance-file-example.py diff --git a/docs/speech-to-text/realtime/assets/end-of-utterance-streaming-example.py b/docs/speech-to-text/streaming/assets/end-of-utterance-streaming-example.py similarity index 100% rename from docs/speech-to-text/realtime/assets/end-of-utterance-streaming-example.py rename to docs/speech-to-text/streaming/assets/end-of-utterance-streaming-example.py diff --git a/docs/speech-to-text/realtime/assets/javascript-radio-example.js b/docs/speech-to-text/streaming/assets/javascript-radio-example.js similarity index 100% rename from docs/speech-to-text/realtime/assets/javascript-radio-example.js rename to docs/speech-to-text/streaming/assets/javascript-radio-example.js diff --git a/docs/speech-to-text/realtime/assets/javascript-realtime-example.mjs b/docs/speech-to-text/streaming/assets/javascript-realtime-example.mjs similarity index 100% rename from docs/speech-to-text/realtime/assets/javascript-realtime-example.mjs rename to docs/speech-to-text/streaming/assets/javascript-realtime-example.mjs diff --git a/docs/speech-to-text/realtime/assets/sm-rt-example.py b/docs/speech-to-text/streaming/assets/sm-rt-example.py similarity index 100% rename from docs/speech-to-text/realtime/assets/sm-rt-example.py rename to docs/speech-to-text/streaming/assets/sm-rt-example.py diff --git a/docs/speech-to-text/realtime/assets/speaker-id-enrollment-file-example.py b/docs/speech-to-text/streaming/assets/speaker-id-enrollment-file-example.py similarity index 100% rename from docs/speech-to-text/realtime/assets/speaker-id-enrollment-file-example.py rename to docs/speech-to-text/streaming/assets/speaker-id-enrollment-file-example.py diff --git a/docs/speech-to-text/realtime/assets/speaker-id-identification-file-example.py b/docs/speech-to-text/streaming/assets/speaker-id-identification-file-example.py similarity index 100% rename from docs/speech-to-text/realtime/assets/speaker-id-identification-file-example.py rename to docs/speech-to-text/streaming/assets/speaker-id-identification-file-example.py diff --git a/docs/speech-to-text/realtime/assets/url-example.py b/docs/speech-to-text/streaming/assets/url-example.py similarity index 100% rename from docs/speech-to-text/realtime/assets/url-example.py rename to docs/speech-to-text/streaming/assets/url-example.py diff --git a/docs/speech-to-text/streaming/channels.mdx b/docs/speech-to-text/streaming/channels.mdx new file mode 100644 index 00000000..bdb34379 --- /dev/null +++ b/docs/speech-to-text/streaming/channels.mdx @@ -0,0 +1,69 @@ +--- +title: Audio channels +sidebar_label: Channels +description: Send audio per channel, close channels, and check the channel limits for your deployment. +keywords: [channels, channel diarization, AddChannelAudio, EndOfChannel, streaming] +--- + +When you use [channel diarization](/speech-to-text/features/channel-diarization) with streaming transcription, you send audio for each channel separately and close each channel when you are finished with it. This page covers that protocol. + +Set up channel diarization first: see [Channel diarization](/speech-to-text/features/channel-diarization) for the `diarization` and `channel_diarization_labels` config. + +## Send audio to a channel + +To send audio for a specific channel, use the `AddChannelAudio` message. Encode the data in base64 format: + +```json +{ + "message": "AddChannelAudio", + "channel": "New_York", + "data": +} +``` + +The server acknowledges with a `ChannelAudioAdded` message carrying a sequence number for that channel: + +```json +{ + "message": "ChannelAudioAdded", + "channel": "New_York", + "seq_no": <10> +} +``` + +## Close a channel + +When you are finished with a channel, signal that it is no longer in use by sending an [`EndOfChannel`](/api-ref/realtime-transcription-websocket#endofchannel) message: + +```json +{ + // highlight-start + "message": "EndOfChannel", + "channel": "New_York", + "last_seq_no": 2564 + // highlight-end +} +``` + +Once this happens, the channel stops accepting and processing additional data. When all channels are closed, the session ends. This is convenient when you do not want to track the number of open channels in a stream. + +You can also use [`EndOfStream`](/api-ref/realtime-transcription-websocket#endofstream) to close all channels simultaneously. This is convenient when you know you want to close every channel at the same time. + +:::info +Closing an individual channel has no impact on pricing. +::: + +## Channel limits + +For SaaS on Cloud, the maximum number of channels is 2. + +For on-prem container deployments, the maximum number of channels depends on your [multi-session container's](/deployments/container/cpu-speech-to-text#multi-session-containers) maximum number of connections. + +The Speechmatics Python client CLI is currently limited to transcribing multi-channel audio from files, not from a stream or raw audio. + +## Channels on-prem + +To run `channel` or `channel_and_speaker` diarization with an on-prem deployment, configure your environment as follows: + +- Use a [GPU Speech to Text container](/deployments/container/gpu-speech-to-text). Handling multiple audio streams is computationally intensive and benefits from GPU acceleration. +- Set the `SM_MAX_CONCURRENT_CONNECTIONS` environment variable to match the number of channels you want to process. diff --git a/docs/speech-to-text/realtime/guides/python-using-ffmpeg.mdx b/docs/speech-to-text/streaming/ffmpeg-audio.mdx similarity index 100% rename from docs/speech-to-text/realtime/guides/python-using-ffmpeg.mdx rename to docs/speech-to-text/streaming/ffmpeg-audio.mdx diff --git a/docs/speech-to-text/realtime/input.mdx b/docs/speech-to-text/streaming/input.mdx similarity index 83% rename from docs/speech-to-text/realtime/input.mdx rename to docs/speech-to-text/streaming/input.mdx index 01e3a3f0..3ccaac8a 100644 --- a/docs/speech-to-text/realtime/input.mdx +++ b/docs/speech-to-text/streaming/input.mdx @@ -31,6 +31,4 @@ After receiving a `RecognitionStarted` message, you can start sending audio over ## Next steps -View our guides: -- [using a microphone](docs/speech-to-text/realtime/guides/python-using-microphone.mdx) to learn how to capture audio from a microphone. -- [using FFMPEG](docs/speech-to-text/realtime/guides/python-using-ffmpeg.mdx) to find out how to pipe microphone audio to the API. \ No newline at end of file +To capture audio from a microphone, see the [Quickstart](/speech-to-text/streaming/quickstart). To pipe audio from another source, see [Pipe audio from FFmpeg](/speech-to-text/streaming/ffmpeg-audio). diff --git a/docs/speech-to-text/realtime/limits.mdx b/docs/speech-to-text/streaming/limits.mdx similarity index 100% rename from docs/speech-to-text/realtime/limits.mdx rename to docs/speech-to-text/streaming/limits.mdx diff --git a/docs/speech-to-text/realtime/output.mdx b/docs/speech-to-text/streaming/output.mdx similarity index 85% rename from docs/speech-to-text/realtime/output.mdx rename to docs/speech-to-text/streaming/output.mdx index 2d9932db..24a3004b 100644 --- a/docs/speech-to-text/realtime/output.mdx +++ b/docs/speech-to-text/streaming/output.mdx @@ -47,7 +47,7 @@ We recommend experimenting with different settings for the `max_delay` to find t Based on our own testing and experience, we can offer a few guidelines on `max_delay` settings to get you started: -- **0.7 - 1.5 seconds**: For use cases where an ultra-fast response is needed such as voice agents. This gives a minor accuracy degradation of less than 5% relative to the Batch transcription service. +- **0.7 - 1.5 seconds**: For conversational applications that need a fast response. For turn-based transcripts instead of a word stream, see [agent STT](/speech-to-text/agent-stt/). This gives a minor accuracy degradation of less than 5% relative to the Batch transcription service. - **2.0 seconds**: Recommended for most use cases, needing the optimal trade-off between acccuracy and latency, such as captioning or contact centres. This gives a negligible degradation of around 1% relative to the Batch transcription service. - **4.0 seconds**: For use cases where accuracy is more important than latency, such as legal transcription. This gives accuracy equivalent to our Batch transcription service. You can also use Partial transcripts to give users early feedback of the recognized text. @@ -77,9 +77,13 @@ Note that Partial transcripts have some limitations: - Accuracy is usually 10-25% lower than the Final transcript. This includes punctuation and capitalization of words. - The `confidence` field for Partial transcripts has no meaning and should not be relied on. +## Mixed-language transcripts + +With the Melia 1 model, each word carries the language detected for it, and the server sends a `LanguageInfo` message the first time it detects a new language. See [Mixed-language transcription](/speech-to-text/features/mixed-language-transcription#language-info). + ## Smart formatting -[Smart formatting](/speech-to-text/formatting#smart-formatting) ensures readability of your transcripts by formatting numbers, dates, currencies and other important _entities_ into their written form. +[Smart formatting](/speech-to-text/features/formatting#smart-formatting) ensures readability of your transcripts by formatting numbers, dates, currencies and other important _entities_ into their written form. When the `max_delay_mode` is set to `flexible`, and an entity is being spoken, the Final transcript would be delayed until the entity is fully spoken to enable proper formatting. This option should be used in most use-cases for improved accuracy and readability for numbers, currencies, and dates. @@ -115,14 +119,4 @@ With `Partials` enabled and `max_delay_mode` as `fixed`, messages received could ## App usage tracking -:::tip -First, please contact [Support](https://support.speechmatics.com) to enable this feature. -::: - -For integrations where customers use their own Speechmatics API keys, gain insights into application usage by aggregating data on unique users, processing hours, languages used, and more. - -Once enabled, use the `sm-app` query parameter in the connection string when starting a session. For example: - -```bash -wss://eu.rt.speechmatics.com/v2?sm-app=YourAppID -``` +To attribute usage across applications, add an app ID to your requests. See [App usage tracking](/speech-to-text/features/app-usage-tracking). diff --git a/docs/speech-to-text/realtime/quickstart.mdx b/docs/speech-to-text/streaming/quickstart.mdx similarity index 90% rename from docs/speech-to-text/realtime/quickstart.mdx rename to docs/speech-to-text/streaming/quickstart.mdx index 54ac8766..49c4004f 100644 --- a/docs/speech-to-text/realtime/quickstart.mdx +++ b/docs/speech-to-text/streaming/quickstart.mdx @@ -23,7 +23,7 @@ The quickest way to try Realtime transcription is via the [web portal](https://p ## Using the Realtime API -The Realtime API streams audio over a WebSocket connection and returns transcript results as you speak. Unlike the [Batch API](/speech-to-text/batch/quickstart), results arrive continuously — within milliseconds of the spoken words. +The Realtime API streams audio over a WebSocket connection and returns transcript results as you speak. Unlike the [Batch API](/speech-to-text/pre-recorded/quickstart), results arrive continuously — within milliseconds of the spoken words. ### 1. Create an API key @@ -42,7 +42,13 @@ Enterprise customers may need to speak to [Support](https://support.speechmatics pip install speechmatics-rt pyaudio ``` :::note - `pyaudio` is required for microphone input in this quickstart. + `pyaudio` is required for microphone input in this quickstart. On Mac M1 and M2, install it with: + ```bash + brew install portaudio + brew link portaudio + BREW_PREFIX=$(brew --prefix) + CFLAGS="-I$BREW_PREFIX/include -L$BREW_PREFIX/lib" python3 -m pip install pyaudio + ``` :::
@@ -155,7 +161,7 @@ Now that you have Realtime transcription working, explore these features to buil } /> } /> } /> } /> } /> diff --git a/docs/speech-to-text/streaming/sidebar.ts b/docs/speech-to-text/streaming/sidebar.ts new file mode 100644 index 00000000..7db30fac --- /dev/null +++ b/docs/speech-to-text/streaming/sidebar.ts @@ -0,0 +1,42 @@ +export default { + type: "category", + label: "Streaming transcription", + items: [ + { + type: "doc", + id: "speech-to-text/streaming/quickstart", + }, + { + // Anchor link: sidebar hrefs are not checked by onBrokenLinks, so the + // target is verified against the built HTML instead. + type: "link", + href: "/speech-to-text/models#streaming-models", + label: "Choosing a model", + }, + { + type: "doc", + id: "speech-to-text/streaming/input", + }, + { + type: "doc", + id: "speech-to-text/streaming/output", + }, + { + type: "doc", + id: "speech-to-text/streaming/limits", + }, + { + type: "doc", + id: "speech-to-text/streaming/channels", + }, + { + type: "doc", + id: "speech-to-text/streaming/ffmpeg-audio", + }, + { + type: "link", + label: "API reference", + href: "/api-ref/realtime-transcription-websocket", + }, + ], +} as const; diff --git a/docs/voice-agents/assets/basic-quickstart.py b/docs/voice-agents/assets/basic-quickstart.py deleted file mode 100644 index 621f902d..00000000 --- a/docs/voice-agents/assets/basic-quickstart.py +++ /dev/null @@ -1,50 +0,0 @@ -import asyncio -import os -from speechmatics.rt import Microphone -from speechmatics.voice import VoiceAgentClient, AgentServerMessageType - -async def main(): - """Stream microphone audio to Speechmatics Voice Agent using 'scribe' preset""" - - # Audio configuration - SAMPLE_RATE = 16000 # Hz - CHUNK_SIZE = 160 # Samples per read - PRESET = "scribe" # Configuration preset - - # Create client with preset - client = VoiceAgentClient( - api_key=os.getenv("YOUR_API_KEY"), - preset=PRESET - ) - - # Handle final segments - @client.on(AgentServerMessageType.ADD_SEGMENT) - def on_segment(message): - for segment in message["segments"]: - speaker = segment["speaker_id"] - text = segment["text"] - print(f"{speaker}: {text}") - - # Setup microphone - mic = Microphone(SAMPLE_RATE, CHUNK_SIZE) - if not mic.start(): - print("Error: Microphone not available") - return - - # Connect to the Voice agent - await client.connect() - - # Stream microphone audio (interruptible using keyboard) - try: - while True: - audio_chunk = await mic.read(CHUNK_SIZE) - if not audio_chunk: - break # Microphone stopped producing data - await client.send_audio(audio_chunk) - except KeyboardInterrupt: - pass - finally: - await client.disconnect() - -if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file diff --git a/docs/voice-agents/assets/presets.py b/docs/voice-agents/assets/presets.py deleted file mode 100644 index a85b3028..00000000 --- a/docs/voice-agents/assets/presets.py +++ /dev/null @@ -1,19 +0,0 @@ -# Presets provide optimized configurations for common use cases: - -# External end of turn preset - endpointing handled by the client -client = VoiceAgentClient(api_key=os.getenv("YOUR_API_KEY"), preset="external") - -# Scribe preset - for note-taking -client = VoiceAgentClient(api_key=os.getenv("YOUR_API_KEY"), preset="scribe") - -# Low latency preset - for fast responses -client = VoiceAgentClient(api_key=os.getenv("YOUR_API_KEY"), preset="low_latency") - -# Conversation preset - for natural dialogue -client = VoiceAgentClient(api_key=os.getenv("YOUR_API_KEY"), preset="conversation_adaptive") - -# Advanced conversation with ML turn detection -client = VoiceAgentClient(api_key=os.getenv("YOUR_API_KEY"), preset="conversation_smart_turn") - -# Captions preset - for live captioning -client = VoiceAgentClient(api_key=os.getenv("YOUR_API_KEY"), preset="captions") diff --git a/docs/voice-agents/overview.mdx b/docs/voice-agents/overview.mdx deleted file mode 100644 index 788d7600..00000000 --- a/docs/voice-agents/overview.mdx +++ /dev/null @@ -1,56 +0,0 @@ ---- -description: Learn how to build voice agents with Speechmatics integrations and the Voice SDK. ---- -import { LinkCard } from "@site/src/theme/LinkCard"; -import { Grid } from "@radix-ui/themes"; - -# Voice agents overview - -There are two ways to build voice agents using Speechmatics: - -- Integration partners (LiveKit, Pipecat and VAPI): the fastest path to a production voice agent. -- Voice SDK: direct access for custom pipelines or working outside of supported integration platforms. - - -## Features - -Speechmatics provides building blocks you can use through integrations and the Voice SDK. - -Key features include: - -- **Turn detection**: detect when a speaker has finished talking. -- **Intelligent segmentation**: group partial transcripts into clean, speaker-attributed segments. -- **Diarization**: identify and label different speakers. -- **Speaker focus**: focus on or ignore specific speakers in multi-speaker scenarios. -- **Preset configurations**: start quickly with ready-to-use settings. - -## Integrations - -Use an integration to handle audio transport and wiring, so you can focus on your agent logic: - - - } - href="/integrations-and-sdks/vapi" - /> - } - href="/integrations-and-sdks/livekit" - /> - } - href="/integrations-and-sdks/pipecat" - /> - - -## Voice SDK - -Use the Voice SDK to handle turn detection, group transcripts into clean segments, and apply diarization for LLM workflows. - -See [Voice SDK](/voice-agents/voice-sdk) for information on getting started, presets, and configuration. diff --git a/docs/voice-agents/sidebar.ts b/docs/voice-agents/sidebar.ts deleted file mode 100644 index 3645a687..00000000 --- a/docs/voice-agents/sidebar.ts +++ /dev/null @@ -1,18 +0,0 @@ -export default { - type: "category", - label: "Voice agents", - collapsible: false, - collapsed: false, - items: [ - { - type: "doc", - id: "voice-agents/overview", - label: "Overview", - }, - { - type: "doc", - id: "voice-agents/voice-sdk", - label: "Voice SDK", - }, - ], -} as const; diff --git a/docusaurus.config.ts b/docusaurus.config.ts index ddfe8c69..7cddad05 100644 --- a/docusaurus.config.ts +++ b/docusaurus.config.ts @@ -36,6 +36,9 @@ const config: Config = { onBrokenLinks: "throw", onBrokenMarkdownLinks: "warn", + // Anchors default to "warn". The baseline is clean, so throwing keeps the + // ~21 anchors that inbound links and redirects depend on from breaking silently. + onBrokenAnchors: "throw", // Even if you don't use internationalization, you can use this field to set // useful metadata like html lang. For example, if your site is Chinese, you diff --git a/package.json b/package.json index 01f8a997..2f434fcb 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "start": "run-p build:jobs-api-ref build:mp-api-ref && dotenv -e .env -e .env.local node_modules/.bin/docusaurus start", "build": "run-p build:jobs-api-ref build:mp-api-ref && dotenv -e .env -e .env.local node_modules/.bin/docusaurus build", "build:jobs-spec": "tsx scripts/generate-batch-spec.ts", - "build:jobs-api-ref": "rimraf docs/api-ref/jobs && npm run build:jobs-spec && docusaurus gen-api-docs jobs", + "build:jobs-api-ref": "rimraf docs/api-ref/batch && npm run build:jobs-spec && docusaurus gen-api-docs jobs", "build:mp-spec": "tsx scripts/generate-mp-spec.ts", "build:mp-api-ref": "rimraf docs/api-ref/management && npm run build:mp-spec && docusaurus gen-api-docs mp", "swizzle": "docusaurus swizzle", diff --git a/scripts/redirects/redirects.json b/scripts/redirects/redirects.json index 0e049179..10b9001a 100644 --- a/scripts/redirects/redirects.json +++ b/scripts/redirects/redirects.json @@ -1,106 +1,342 @@ [ + { + "source": "/features/accuracy-language-packs", + "destination": "/speech-to-text/languages#bilingual-and-multi-language-packs" + }, { "source": "/speech-to-text/realtime/realtime_diarization", - "destination": "/speech-to-text/realtime/realtime-diarization" + "destination": "/speech-to-text/features/speaker-diarization" }, { "source": "/speech-to-text/batch/batch_diarization", - "destination": "/speech-to-text/batch/batch-diarization" + "destination": "/speech-to-text/features/speaker-diarization" }, { "source": "/speech-to-text/batch/tracking", - "destination": "/speech-to-text/batch/output#tracking-metadata" + "destination": "/speech-to-text/pre-recorded/output#tracking-metadata" }, { "source": "/speech-to-text/realtime/end-of-turn", - "destination": "/speech-to-text/realtime/turn-detection" + "destination": "/speech-to-text/features/turn-detection" }, { "source": "/speech-to-text/realtime/end-of-utterance", - "destination": "/speech-to-text/realtime/turn-detection" + "destination": "/speech-to-text/features/turn-detection" }, { "source": "/speech-to-text/realtime/realtime-speaker-identification", - "destination": "/speech-to-text/realtime/speaker-identification" + "destination": "/speech-to-text/features/speaker-identification" }, { "source": "/voice-agents-flow/features/application-inputs", - "destination": "/voice-agents/flow/features/application-inputs" + "destination": "/speech-to-text/agent-stt" }, { "source": "/voice-agents-flow/setup", - "destination": "/voice-agents/flow/setup" + "destination": "/speech-to-text/agent-stt/quickstart" }, { "source": "/voice-agents-flow/features/function-calling", - "destination": "/voice-agents/flow/features/function-calling" + "destination": "/integrations-and-sdks" }, { "source": "/voice-agents-flow", - "destination": "/voice-agents/flow" + "destination": "/speech-to-text/agent-stt" }, { "source": "/voice-agents-flow/supported-languages", - "destination": "/voice-agents/flow/supported-languages" + "destination": "/speech-to-text/languages" }, { "source": "/voice-agents-flow/features/webrtc-livekit", - "destination": "/voice-agents/flow/features/webrtc-livekit" + "destination": "/integrations-and-sdks/livekit" }, { "source": "/voice-agents-flow/guides/nextjs-guide", - "destination": "/voice-agents/flow/guides/nextjs-guide" + "destination": "/integrations-and-sdks" }, { "source": "/voice-agents-flow/guides/react-native", - "destination": "/voice-agents/flow/guides/react-native" + "destination": "/integrations-and-sdks" }, { "source": "/deployments/container", - "destination": "/deployements/container/accessing-images" - }, - { - "source": "/voice-agents-flow", - "destination": "/voice-agents/flow" - }, - { - "source": "/voice-agents-flow", - "destination": "/voice-agents/flow" + "destination": "/deployments/container/accessing-images" }, { "source": "/voice-agents/flow", - "destination": "/voice-agents/overview" + "destination": "/speech-to-text/agent-stt" }, { "source": "/voice-agents/flow/features/application-inputs", - "destination": "/voice-agents/overview" + "destination": "/speech-to-text/agent-stt" }, { "source": "/voice-agents/flow/features/webrtc-livekit", - "destination": "/voice-agents/overview" + "destination": "/integrations-and-sdks/livekit" }, { "source": "/voice-agents/flow/setup", - "destination": "/voice-agents/overview" + "destination": "/speech-to-text/agent-stt/quickstart" }, { "source": "/voice-agents/flow/supported-languages", - "destination": "/voice-agents/overview" + "destination": "/speech-to-text/languages" }, { "source": "/voice-agents/flow/features/function-calling", - "destination": "/voice-agents/overview" + "destination": "/integrations-and-sdks" }, { "source": "/voice-agents/flow/guides/nextjs-guide", - "destination": "/voice-agents/overview" + "destination": "/integrations-and-sdks" }, { "source": "/voice-agents/flow/guides/react-native", - "destination": "/voice-agents/overview" + "destination": "/integrations-and-sdks" }, { "source": "/guides/projects", "destination": "/administration/projects" + }, + { + "source": "/speech-to-text/batch", + "destination": "/speech-to-text/pre-recorded/quickstart" + }, + { + "source": "/speech-to-text/realtime", + "destination": "/speech-to-text/streaming/quickstart" + }, + { + "source": "/speech-to-text/batch/alignment", + "destination": "/speech-to-text/add-ons/audio-alignment" + }, + { + "source": "/speech-to-text/batch/batch-diarization", + "destination": "/speech-to-text/features/speaker-diarization" + }, + { + "source": "/speech-to-text/batch/input", + "destination": "/speech-to-text/pre-recorded/input" + }, + { + "source": "/speech-to-text/batch/language-identification", + "destination": "/speech-to-text/pre-recorded/language-identification" + }, + { + "source": "/speech-to-text/batch/limits", + "destination": "/speech-to-text/pre-recorded/limits" + }, + { + "source": "/speech-to-text/batch/notifications", + "destination": "/speech-to-text/pre-recorded/notifications" + }, + { + "source": "/speech-to-text/batch/output", + "destination": "/speech-to-text/pre-recorded/output" + }, + { + "source": "/speech-to-text/batch/quickstart", + "destination": "/speech-to-text/pre-recorded/quickstart" + }, + { + "source": "/speech-to-text/batch/speaker-identification", + "destination": "/speech-to-text/features/speaker-identification" + }, + { + "source": "/speech-to-text/batch/speech-intelligence/auto-chapters", + "destination": "/speech-to-text/add-ons/chapters" + }, + { + "source": "/speech-to-text/batch/speech-intelligence/sentiment-analysis", + "destination": "/speech-to-text/add-ons/sentiment" + }, + { + "source": "/speech-to-text/batch/speech-intelligence/summarization", + "destination": "/speech-to-text/add-ons/summaries" + }, + { + "source": "/speech-to-text/batch/speech-intelligence/topic-detection", + "destination": "/speech-to-text/add-ons/topics" + }, + { + "source": "/speech-to-text/batch/srt-format", + "destination": "/speech-to-text/pre-recorded/srt-format" + }, + { + "source": "/speech-to-text/batch/synchronous", + "destination": "/speech-to-text/pre-recorded/synchronous" + }, + { + "source": "/speech-to-text/batch/troubleshooting", + "destination": "/speech-to-text/pre-recorded/troubleshooting" + }, + { + "source": "/speech-to-text/batch/usage", + "destination": "/administration/usage" + }, + { + "source": "/speech-to-text/realtime/guides/python-using-ffmpeg", + "destination": "/speech-to-text/streaming/ffmpeg-audio" + }, + { + "source": "/speech-to-text/realtime/guides/python-using-microphone", + "destination": "/speech-to-text/streaming/quickstart" + }, + { + "source": "/speech-to-text/realtime/input", + "destination": "/speech-to-text/streaming/input" + }, + { + "source": "/speech-to-text/realtime/limits", + "destination": "/speech-to-text/streaming/limits" + }, + { + "source": "/speech-to-text/realtime/output", + "destination": "/speech-to-text/streaming/output" + }, + { + "source": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart" + }, + { + "source": "/speech-to-text/realtime/realtime-diarization", + "destination": "/speech-to-text/features/speaker-diarization" + }, + { + "source": "/speech-to-text/realtime/speaker-identification", + "destination": "/speech-to-text/features/speaker-identification" + }, + { + "source": "/speech-to-text/realtime/turn-detection", + "destination": "/speech-to-text/features/turn-detection" + }, + { + "source": "/speech-to-text/batch/input#fetch-url", + "destination": "/speech-to-text/pre-recorded/input#fetch-url" + }, + { + "source": "/speech-to-text/batch/output#tracking-metadata", + "destination": "/speech-to-text/pre-recorded/output#tracking-metadata" + }, + { + "source": "/speech-to-text/realtime/output#latency", + "destination": "/speech-to-text/streaming/output#latency" + }, + { + "source": "/speech-to-text/batch/speech-intelligence/auto-chapters/", + "destination": "/speech-to-text/add-ons/chapters" + }, + { + "source": "/private/voice-agent-api", + "destination": "/speech-to-text/agent-stt" + }, + { + "source": "/speech-to-text/formatting", + "destination": "/speech-to-text/features/formatting" + }, + { + "source": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting" + }, + { + "source": "/speech-to-text/formatting#punctuation", + "destination": "/speech-to-text/features/formatting#punctuation" + }, + { + "source": "/speech-to-text/formatting#output-formatting-options", + "destination": "/speech-to-text/features/disfluencies-and-profanities" + }, + { + "source": "/speech-to-text/features/diarization", + "destination": "/speech-to-text/features/speaker-diarization" + }, + { + "source": "/speech-to-text/pre-recorded/batch-diarization", + "destination": "/speech-to-text/features/speaker-diarization" + }, + { + "source": "/speech-to-text/streaming/realtime-diarization", + "destination": "/speech-to-text/features/speaker-diarization" + }, + { + "source": "/speech-to-text/pre-recorded/speaker-identification", + "destination": "/speech-to-text/features/speaker-identification" + }, + { + "source": "/speech-to-text/streaming/speaker-identification", + "destination": "/speech-to-text/features/speaker-identification" + }, + { + "source": "/speech-to-text/streaming/turn-detection", + "destination": "/speech-to-text/features/turn-detection" + }, + { + "source": "/speech-to-text/streaming/guides/python-using-ffmpeg", + "destination": "/speech-to-text/streaming/ffmpeg-audio" + }, + { + "source": "/speech-to-text/streaming/guides/python-using-microphone", + "destination": "/speech-to-text/streaming/quickstart" + }, + { + "source": "/speech-to-text/app-analytics", + "destination": "/speech-to-text/features/app-usage-tracking" + }, + { + "source": "/speech-to-text/streaming/turn-detection#configuration", + "destination": "/speech-to-text/features/turn-detection#configuration" + }, + { + "source": "/speech-to-text/pre-recorded/speech-intelligence/summarization", + "destination": "/speech-to-text/add-ons/summaries" + }, + { + "source": "/speech-to-text/pre-recorded/speech-intelligence/topic-detection", + "destination": "/speech-to-text/add-ons/topics" + }, + { + "source": "/speech-to-text/pre-recorded/speech-intelligence/auto-chapters", + "destination": "/speech-to-text/add-ons/chapters" + }, + { + "source": "/speech-to-text/pre-recorded/speech-intelligence/sentiment-analysis", + "destination": "/speech-to-text/add-ons/sentiment" + }, + { + "source": "/speech-to-text/features/translation", + "destination": "/speech-to-text/add-ons/translation" + }, + { + "source": "/speech-to-text/pre-recorded/alignment", + "destination": "/speech-to-text/add-ons/audio-alignment" + }, + { + "source": "/speech-to-text/pre-recorded/speech-intelligence", + "destination": "/speech-to-text/add-ons/summaries" + }, + { + "source": "/speech-to-text/features/translation#languages", + "destination": "/speech-to-text/add-ons/translation#languages" + }, + { + "source": "/voice-agents/overview", + "destination": "/speech-to-text/agent-stt" + }, + { + "source": "/voice-agents", + "destination": "/speech-to-text/agent-stt" + }, + { + "source": "/voice-agents/voice-sdk", + "destination": "/integrations-and-sdks/voice-sdk" + }, + { + "source": "/speech-to-text/pre-recorded/usage", + "destination": "/administration/usage" + }, + { + "source": "/get-started/quickstart", + "destination": "/" } ] diff --git a/sidebars.ts b/sidebars.ts index 3eab70ba..e9de13fc 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -5,14 +5,12 @@ import gettingStartedSidebar from "./docs/get-started/sidebar"; import integrationsAndSDKSidebar from "./docs/integrations-and-sdks/sidebar"; import speechToTextSidebar from "./docs/speech-to-text/sidebar"; import textToSpeechSidebar from "./docs/text-to-speech/sidebar"; -import voiceAgentsSidebar from "./docs/voice-agents/sidebar"; export default { docs: [ gettingStartedSidebar, speechToTextSidebar, textToSpeechSidebar, - voiceAgentsSidebar, integrationsAndSDKSidebar, deploymentsSidebar, administrationSidebar, diff --git a/spec/batch.yaml b/spec/batch.yaml index 112ddd74..64edba23 100644 --- a/spec/batch.yaml +++ b/spec/batch.yaml @@ -1768,7 +1768,7 @@ definitions: properties: text: type: string - description: Represents the transcript of the analysed segment + description: Represents the transcript of the analyzed segment sentiment: type: string description: The assigned sentiment to the segment, which can be positive, neutral or negative diff --git a/spec/flow-api.yaml b/spec/flow-api.yaml deleted file mode 100644 index b5f55428..00000000 --- a/spec/flow-api.yaml +++ /dev/null @@ -1,910 +0,0 @@ -asyncapi: 3.0.0 -id: "urn:com:speechmatics:flow-service" -defaultContentType: application/json -info: - title: Speechmatics Flow API - version: "1.0.0" - contact: - name: Speechmatics Support - url: https://www.speechmatics.com/product/support/ - email: support@speechmatics.com - externalDocs: - description: "Flow API Reference" - url: https://docs.speechmatics.com/api-ref/flow-voice-ai - -servers: - default: - host: flow.api.speechmatics.com/ - protocol: wss - protocolVersion: v13 (RFC 6455) - description: Flow server - variables: - ports: - default: "9000" - -channels: - publish: - address: /v1/flow - messages: - StartConversation: - $ref: "#/components/messages/StartConversation" - AddAudio: - $ref: "#/components/messages/AddAudio" - AudioReceived: - $ref: "#/components/messages/AudioReceived" - AudioEnded: - $ref: "#/components/messages/AudioEnded" - AddInput: - $ref: "#/components/messages/AddInput" - ToolResult: - $ref: "#/components/messages/ToolResult" - subscribe: - address: /v1/flow - messages: - ConversationStarted: - $ref: "#/components/messages/ConversationStarted" - AddAudio: - $ref: "#/components/messages/AddAudio" - AudioAdded: - $ref: "#/components/messages/AudioAdded" - AddPartialTranscript: - $ref: "#/components/messages/AddPartialTranscript" - AddTranscript: - $ref: "#/components/messages/AddTranscript" - ResponseStarted: - $ref: "#/components/messages/ResponseStarted" - ResponseCompleted: - $ref: "#/components/messages/ResponseCompleted" - ResponseInterrupted: - $ref: "#/components/messages/ResponseInterrupted" - ToolInvoke: - $ref: "#/components/messages/ToolInvoke" - Error: - $ref: "#/components/messages/Error" - Warning: - $ref: "#/components/messages/Warning" - Info: - $ref: "#/components/messages/Info" - ConversationEnding: - $ref: "#/components/messages/ConversationEnding" - ConversationEnded: - $ref: "#/components/messages/ConversationEnded" - -operations: - publish: - action: send - channel: - $ref: "#/channels/publish" - messages: - - $ref: "#/channels/publish/messages/StartConversation" - - $ref: "#/channels/publish/messages/AddAudio" - - $ref: "#/channels/publish/messages/AudioReceived" - - $ref: "#/channels/publish/messages/AudioEnded" - - $ref: "#/channels/publish/messages/AddInput" - - $ref: "#/channels/publish/messages/ToolResult" - - subscribe: - action: receive - channel: - $ref: "#/channels/subscribe" - messages: - - $ref: "#/channels/subscribe/messages/ConversationStarted" - - $ref: "#/channels/subscribe/messages/AddAudio" - - $ref: "#/channels/subscribe/messages/AudioAdded" - - $ref: "#/channels/subscribe/messages/AddPartialTranscript" - - $ref: "#/channels/subscribe/messages/AddTranscript" - - $ref: "#/channels/subscribe/messages/ResponseStarted" - - $ref: "#/channels/subscribe/messages/ResponseCompleted" - - $ref: "#/channels/subscribe/messages/ResponseInterrupted" - - $ref: "#/channels/subscribe/messages/ToolInvoke" - - $ref: "#/channels/subscribe/messages/Error" - - $ref: "#/channels/subscribe/messages/Warning" - - $ref: "#/channels/subscribe/messages/Info" - - $ref: "#/channels/subscribe/messages/ConversationEnding" - - $ref: "#/channels/subscribe/messages/ConversationEnded" - -components: - messages: - StartConversation: - summary: Initiates a new conversation session. - payload: - $ref: "#/components/schemas/StartConversation" - AddAudio: - summary: A binary chunk of audio. The server confirms receipt by sending an AudioAdded message. - contentType: application/octet-stream - payload: - $ref: "#/components/schemas/AddAudio" - AudioReceived: - summary: Client response to AddAudio, indicating that server audio has been added to the client successfully. - payload: - $ref: "#/components/schemas/AudioReceived" - AudioEnded: - summary: Declares that the client has no more audio to send. - payload: - $ref: "#/components/schemas/AudioEnded" - AddInput: - summary: Message used by the application client to send input to the LLM in order to influence the conversation. - payload: - $ref: "#/components/schemas/AddInput" - ConversationStarted: - summary: Server response to StartConversation, acknowledging that a conversation session has started. - payload: - $ref: "#/components/schemas/ConversationStarted" - AudioAdded: - summary: Server response to AddAudio, indicating that audio has been added successfully. - payload: - $ref: "#/components/schemas/AudioAdded" - AddPartialTranscript: - summary: Contains a work-in-progress transcript of a part of the audio that the client has sent. - payload: - $ref: "#/components/schemas/AddPartialTranscript" - AddTranscript: - summary: Contains the final transcript of a part of the audio that the client has sent. - payload: - $ref: "#/components/schemas/AddTranscript" - ResponseStarted: - summary: Indicates the start of a response from the agent. - payload: - $ref: "#/components/schemas/ResponseStarted" - ResponseCompleted: - summary: Indicates the completion of a response from the agent. - payload: - $ref: "#/components/schemas/ResponseCompleted" - ResponseInterrupted: - summary: Indicates that a response from the agent was interrupted. - payload: - $ref: "#/components/schemas/ResponseInterrupted" - ToolInvoke: - summary: Invokes a tool with the specified parameters. - payload: - $ref: "#/components/schemas/ToolInvoke" - ToolResult: - summary: Contains the result of a tool invocation. - payload: - $ref: "#/components/schemas/ToolResult" - Info: - summary: Additional information sent from the server to the client. - payload: - $ref: "#/components/schemas/Info" - Warning: - summary: Warning messages sent from the server to the client. - payload: - $ref: "#/components/schemas/Warning" - Error: - summary: Error messages sent from the server to the client. - payload: - $ref: "#/components/schemas/Error" - ConversationEnding: - summary: Indicates starting of the session transfer procedure - payload: - $ref: "#/components/schemas/ConversationEnding" - ConversationEnded: - summary: Server ends the conversation, after the server has finished sending all other messages. - payload: - $ref: "#/components/schemas/ConversationEnded" - - schemas: - TypeEnum: - type: string - enum: - - word - - punctuation - - AttachesToEnum: - type: string - enum: - - next - - previous - - none - - both - - StatusEnum: - type: string - enum: - - ok - - rejected - - failed - - WarningTypeEnum: - type: string - enum: - - high_asr_latency - - llm_error - - high_llm_latency - - llm_request_content_filter - - tts_error - - high_tts_latency - - protocol_error - - idle_timeout - - session_timeout - # conversation_termination has extra fields so it handled separately - - ErrorTypeEnum: - type: string - enum: - - asr_error - - protocol_error - - config_error - - idle_timeout - - session_timeout - - not_allowed - - not_authorised - - quota_exceeded - - timelimit_exceeded - - job_error - - internal_error - - unknown_error - - ToolTypeEnum: - type: string - description: The type of tool to use. At the moment, only `function` is supported. - enum: - - function - - ToolFunctionParameterTypeEnum: - type: string - enum: - - integer - - number - - string - - boolean - - AudioFormatRawEncodingEnum: - type: string - enum: - - pcm_f32le - - pcm_s16le - - mulaw - default: pcm_s16le - - ToolFunctionParametersTypeEnum: - type: string - enum: - - object - - OperatingPoint: - type: string - enum: - - standard - - enhanced - - DiarizationConfigEnum: - type: string - enum: - - none - - speaker - - MaxDelayModeConfigEnum: - type: string - enum: - - flexible - - fixed - - RecognitionDisplayDirectionEnum: - type: string - enum: - - ltr - - rtl - - StartConversation: - type: object - properties: - message: - const: StartConversation - audio_format: - $ref: "#/components/schemas/AudioFormat" - conversation_config: - $ref: "#/components/schemas/ConversationConfig" - tools: - $ref: "#/components/schemas/ToolsArray" - debug: - $ref: "#/components/schemas/DebugObject" - required: - - message - - conversation_config - - ToolsArray: - type: array - items: - $ref: "#/components/schemas/ToolConfig" - description: > - A list of tools that the LLM can use during the conversation. - - DebugObject: - type: object - properties: - llm: - type: boolean - additionalProperties: true - - AddAudio: - type: string - format: binary - - AudioReceived: - type: object - properties: - message: - const: AudioReceived - seq_no: - type: integer - required: - - message - - seq_no - - AudioEnded: - type: object - properties: - message: - const: AudioEnded - last_seq_no: - type: integer - required: - - message - - last_seq_no - - AddInput: - type: object - properties: - message: - const: AddInput - input: - type: string - description: > - The information that the LLM must incorporate in the response - interrupt_response: - type: boolean - default: false - description: > - If true, the response will be interrupted by the new input.
- If false, the response will continue until it is complete, defaults to false. - immediate: - type: boolean - default: false - description: > - If true, the input will be treated as urgent and will be sent to LLM immediately.
- If false, new input will be added to current prompt and sent to LLM as a part of the next request.
- required: - - message - - input - - ConversationStarted: - type: object - properties: - message: - const: ConversationStarted - id: - type: string - asr_session_id: - type: string - language_pack_info: - $ref: "#/components/schemas/LanguagePackInfo" - required: - - message - - AudioAdded: - type: object - properties: - message: - const: AudioAdded - seq_no: - type: integer - required: - - message - - seq_no - - AddPartialTranscript: - type: object - properties: - message: - const: AddPartialTranscript - format: - type: string - example: "2.1" - description: Speechmatics JSON output format version number. - metadata: - $ref: "#/components/schemas/RecognitionMetadata" - results: - $ref: "#/components/schemas/RecognitionResultArray" - required: - - message - - metadata - - results - - AddTranscript: - type: object - properties: - message: - const: AddTranscript - format: - type: string - example: "2.1" - description: Speechmatics JSON output format version number. - metadata: - $ref: "#/components/schemas/RecognitionMetadata" - results: - $ref: "#/components/schemas/RecognitionResultArray" - required: - - message - - metadata - - results - - RecognitionResultArray: - type: array - items: - $ref: "#/components/schemas/RecognitionResult" - - ResponseStarted: - type: object - properties: - message: - const: ResponseStarted - content: - type: string - description: The content that is spoken by the agent in the response. - start_time: - type: number - format: float - description: The start time of the spoken response, relative to the start of the session. - required: - - message - - content - - start_time - - ResponseCompleted: - type: object - properties: - message: - const: ResponseCompleted - content: - type: string - description: The content that is spoken by the agent in the response. - start_time: - type: number - format: float - description: The start time of the spoken response, relative to the start of the session. - end_time: - type: number - format: float - description: The end time of the spoken response, relative to the start of the session. - required: - - message - - content - - start_time - - end_time - - ResponseInterrupted: - type: object - properties: - message: - const: ResponseInterrupted - content: - type: string - description: The content that is spoken by the agent in the response. - start_time: - type: number - format: float - description: The start time of the spoken response, relative to the start of the session. - end_time: - type: number - format: float - description: The end time of the spoken response, relative to the start of the session. - required: - - message - - content - - start_time - - end_time - - ToolInvoke: - type: object - required: - - message - - id - - type - - function - properties: - message: - const: ToolInvoke - id: - type: string - description: The id of the tool invoke. - type: - const: function - function: - $ref: "#/components/schemas/ToolFunction" - - ToolFunction: - type: object - required: - - name - - arguments - properties: - name: - type: string - description: The name of the tool to invoke. - arguments: - type: object - additionalProperties: - oneOf: - - type: string - - type: integer - - type: boolean - - ToolResult: - type: object - required: - - message - - id - - status - properties: - message: - const: ToolResult - id: - type: string - description: The id of the tool invoke. - status: - $ref: "#/components/schemas/StatusEnum" - description: The status of the tool invocation. - content: - type: string - description: The content of the tool result. - - Info: - oneOf: - - $ref: "#/components/schemas/StatusUpdateInfo" - - $ref: "#/components/schemas/ConversationDurationLimitInfo" - - $ref: "#/components/schemas/ConcurrentSessionUsage" - discrimator: - propertyName: type - - BaseInfo: - type: object - properties: - message: - const: Info - reason: - type: string - required: - - message - - reason - - StatusUpdateInfo: - title: StatusUpdate - allOf: - - $ref: '#/components/schemas/BaseInfo' - - type: object - properties: - type: - const: status_update - event: - type: object - required: - - prev_status - - status - properties: - prev_status: - type: string - status: - type: string - required: - - type - - event - - ConversationDurationLimitInfo: - title: ConversationDurationLimit - allOf: - - $ref: '#/components/schemas/BaseInfo' - - type: object - properties: - type: - const: conversation_duration_limit - conversation_duration_limit: - type: integer - required: - - type - - conversation_duration_limit - - ConcurrentSessionUsage: - title: ConcurrentSessionUsage - allOf: - - $ref: '#/components/schemas/BaseInfo' - - type: object - properties: - type: - const: concurrent_session_usage - usage: - type: integer - quota: - type: integer - last_udpated: - type: string - format: date-time - required: - - type - - usage - - quota - - last_updated - - Warning: - oneOf: - - $ref: "#/components/schemas/DefaultWarning" - - $ref: "#/components/schemas/ConversationTermination" - discrimator: - propertyName: type - - BaseWarning: - type: object - properties: - message: - const: Warning - reason: - type: string - required: - - message - - reason - - DefaultWarning: - title: DefaultWarning - allOf: - - $ref: "#/components/schemas/BaseWarning" - - type: object - properties: - type: - $ref: "#/components/schemas/WarningTypeEnum" - required: - - type - - ConversationTermination: - title: ConversationTermination - allOf: - - $ref: "#/components/schemas/BaseWarning" - - type: object - properties: - type: - const: conversation_termination - conversation_termination: - type: integer - required: - - type - - conversation_termination - - Error: - type: object - properties: - message: - const: Error - type: - $ref: "#/components/schemas/ErrorTypeEnum" - reason: - type: string - required: - - message - - type - - reason - - ConversationEnding: - type: object - properties: - message: - const: ConversationEnding - required: - - message - - ConversationEnded: - type: object - properties: - message: - const: ConversationEnded - required: - - message - - ConversationConfig: - type: object - required: - - template_id - properties: - template_id: - type: string - # description: The id of the agent or persona to use during the conversation. - description: | - Required in the the `StartConversation` message in the Flow API. Generated from the [Speechmatics Portal](https://portal.speechmatics.com/). This maps to the [language supported](/voice-agents/flow/supported-languages), agent's prompt, LLM, TTS voice, & custom dictionary. These can be customised by creating or modifying agents in the Portal. - template_variables: - type: object - additionalProperties: - type: string - - ToolConfig: - type: object - additionalProperties: false - required: - - type - - function - properties: - type: - $ref: "#/components/schemas/ToolTypeEnum" - description: The type of the tool (currently, must be 'function'). - function: - $ref: "#/components/schemas/ToolFunctionConfig" - - ToolFunctionConfig: - type: object - additionalProperties: false - description: The function that the tool will call. - required: - - name - properties: - name: - type: string - description: The name of the function that should be called. This name is passed as a field in the ToolInvoke message - description: - type: string - description: | - A natural language string that instructs the LLM about the condition in which the function must be called - parameters: - $ref: "#/components/schemas/ToolFunctionParameters" - - ToolFunctionParameters: - type: object - additionalProperties: false - description: | - An object containing the `properties` of the function call which should be collected from the conversation. Each parameter is defined by: - properties: - type: - $ref: "#/components/schemas/ToolFunctionParametersTypeEnum" - description: The type of the parameters (currently, will always be object). - required: - type: array - description: | - (optional) The list of input parameters for the function which are required. - items: - type: string - properties: - $ref: "#/components/schemas/ToolFunctionParameterProperties" - - ToolFunctionParameterProperties: - type: object - description: | - Properties of the function parameter object - additionalProperties: - $ref: "#/components/schemas/ToolFunctionParameter" - - ToolFunctionParameter: - type: object - additionalProperties: false - required: - - type - properties: - type: - $ref: "#/components/schemas/ToolFunctionParameterTypeEnum" - description: The type of the parameter - description: - type: string - description: A description of the parameter. - enum: - type: array - items: - x-union-string-int: true - example: - type: string - description: An example value for the parameter. - - AudioFormat: - type: object - properties: - type: - const: raw - encoding: - $ref: "#/components/schemas/AudioFormatRawEncodingEnum" - sample_rate: - type: integer - default: 16000 - required: - - type - - LanguagePackInfo: - type: object - description: Properties of the language pack. - required: [ - word_delimiter - ] - properties: - language_description: - type: string - description: Full descriptive name of the language, e.g. 'Japanese'. - word_delimiter: - type: string - description: The character to use to separate words. - writing_direction: - type: string - enum: [ - left-to-right, - right-to-left - ] - description: The direction that words in the language should be written and read in. - itn: - type: boolean - description: Whether or not ITN (inverse text normalization) is available for the language pack. - adapted: - type: boolean - description: Whether or not language model adaptation has been applied to the language pack. - - RecognitionMetadata: - type: object - properties: - start_time: - type: number - format: float - end_time: - type: number - format: float - transcript: - type: string - required: - - start_time - - end_time - - transcript - - RecognitionResult: - type: object - properties: - type: - $ref: "#/components/schemas/TypeEnum" - start_time: - type: number - format: float - end_time: - type: number - format: float - channel: - type: string - attaches_to: - $ref: "#/components/schemas/AttachesToEnum" - is_eos: - type: boolean - alternatives: - $ref: "#/components/schemas/RecognitionAlternativeArray" - score: - type: number - format: float - minimum: 0 - maximum: 1 - volume: - type: number - format: float - minimum: 0 - maximum: 100 - required: - - type - - start_time - - end_time - - RecognitionAlternativeArray: - type: array - items: - $ref: "#/components/schemas/RecognitionAlternative" - - RecognitionAlternative: - type: object - properties: - content: - type: string - confidence: - type: number - format: float - language: - type: string - display: - $ref: "#/components/schemas/RecognitionDisplay" - speaker: - type: string - required: - - content - - confidence - - RecognitionDisplay: - type: object - required: - - direction - properties: - direction: - $ref: "#/components/schemas/RecognitionDisplayDirectionEnum" diff --git a/spec/realtime.yaml b/spec/realtime.yaml index 53bd0887..15683748 100644 --- a/spec/realtime.yaml +++ b/spec/realtime.yaml @@ -179,7 +179,7 @@ components: Partials will only be sent if `transcription_config.enable_partials` is set to `true` in the `StartRecognition` message. - The message structure is the same as `AddTranscript`, with a few [limitations](https://docs.speechmatics.com/speech-to-text/realtime/output#partial-transcripts). + The message structure is the same as `AddTranscript`, with a few [limitations](https://docs.speechmatics.com/speech-to-text/streaming/output#partial-transcripts). :::warning @@ -195,7 +195,7 @@ components: EndOfUtterance: summary: | Indicates the end of an utterance, triggered by a configurable period of non-speech. - The message is sent when no speech has been detected for a short period of time, configurable by the `end_of_utterance_silence_trigger` parameter in `conversation_config` (see [End Of Utterance](https://docs.speechmatics.com/speech-to-text/realtime/turn-detection#configuration)). + The message is sent when no speech has been detected for a short period of time, configurable by the `end_of_utterance_silence_trigger` parameter in `conversation_config` (see [End Of Utterance](https://docs.speechmatics.com/speech-to-text/features/turn-detection#configuration)). Like punctuation, an `EndOfUtterance` has zero duration. payload: @@ -764,7 +764,7 @@ components: max_delay: type: number description: | - This is the delay in seconds between the end of a spoken word and returning the Final transcript results. See [Latency](https://docs.speechmatics.com/speech-to-text/realtime/output#latency) for more details + This is the delay in seconds between the end of a spoken word and returning the Final transcript results. See [Latency](https://docs.speechmatics.com/speech-to-text/streaming/output#latency) for more details minimum: 0.7 maximum: 4 default: 4 @@ -780,7 +780,7 @@ components: type: boolean description: | Whether or not to send Partials (i.e. `AddPartialTranslation` messages) as well as Finals (i.e. `AddTranslation` messages) - See [Partial transcripts](https://docs.speechmatics.com/speech-to-text/realtime/output#partial-transcripts). + See [Partial transcripts](https://docs.speechmatics.com/speech-to-text/streaming/output#partial-transcripts). default: false enable_entities: type: boolean @@ -811,7 +811,7 @@ components: max_delay: type: number description: | - This is the delay in seconds between the end of a spoken word and returning the Final transcript results. See [Latency](https://docs.speechmatics.com/speech-to-text/realtime/output#latency) for more details + This is the delay in seconds between the end of a spoken word and returning the Final transcript results. See [Latency](https://docs.speechmatics.com/speech-to-text/streaming/output#latency) for more details minimum: 0.7 maximum: 4 default: 4 @@ -823,7 +823,7 @@ components: type: boolean description: | Whether or not to send Partials (i.e. `AddPartialTranslation` messages) as well as Finals (i.e. `AddTranslation` messages) - See [Partial transcripts](https://docs.speechmatics.com/speech-to-text/realtime/output#partial-transcripts). + See [Partial transcripts](https://docs.speechmatics.com/speech-to-text/streaming/output#partial-transcripts). default: false conversation_config: $ref: "#/components/schemas/ConversationConfig" @@ -839,7 +839,7 @@ components: Model: type: string description: | - Which model you wish to use. See [Models](http://docs.speechmatics.com/speech-to-text/models) for more details. + Which model you wish to use. See [Models](https://docs.speechmatics.com/speech-to-text/models) for more details. enum: - standard - enhanced @@ -848,7 +848,7 @@ components: PunctuationOverrides: type: object description: | - Options for controlling punctuation in the output transcripts. See [Punctuation Settings](https://docs.speechmatics.com/speech-to-text/formatting#punctuation) + Options for controlling punctuation in the output transcripts. See [Punctuation Settings](https://docs.speechmatics.com/speech-to-text/features/formatting#punctuation) properties: permitted_marks: type: array @@ -935,7 +935,7 @@ components: DiarizationConfig: type: string description: | - Set to `speaker` to apply [Speaker Diarization](https://docs.speechmatics.com/speech-to-text/features/diarization) to the audio. + Set to `speaker` to apply [Speaker Diarization](https://docs.speechmatics.com/speech-to-text/features/speaker-diarization) to the audio. enum: - none - speaker @@ -948,12 +948,12 @@ components: max_speakers: type: integer description: | - Configure the maximum number of speakers to detect. See [Max Speakers](http://docs.speechmatics.com/speech-to-text/features/diarization#max-speakers). + Configure the maximum number of speakers to detect. See [Max Speakers](https://docs.speechmatics.com/speech-to-text/features/speaker-diarization#max-speakers). minimum: 2 prefer_current_speaker: description: | When set to `true`, reduces the likelihood of incorrectly switching between similar sounding speakers. - See [Prefer Current Speaker](https://docs.speechmatics.com/speech-to-text/features/diarization#prefer-current-speaker). + See [Prefer Current Speaker](https://docs.speechmatics.com/speech-to-text/features/speaker-diarization#prefer-current-speaker). type: boolean default: false speaker_sensitivity: @@ -991,16 +991,16 @@ components: remove_disfluencies: type: boolean description: | - When set to `true`, removes disfluencies from the transcript. See [Removing disfluencies](https://docs.speechmatics.com/speech-to-text/formatting#removing-disfluencies) + When set to `true`, removes disfluencies from the transcript. See [Removing disfluencies](https://docs.speechmatics.com/speech-to-text/features/disfluencies-and-profanities#removing-disfluencies) replacements: - description: A list of replacement rules to apply to the transcript. Each rule consists of a pattern to match and a replacement string. See [Word replacement](https://docs.speechmatics.com/speech-to-text/formatting#word-replacement) + description: A list of replacement rules to apply to the transcript. Each rule consists of a pattern to match and a replacement string. See [Word replacement](https://docs.speechmatics.com/speech-to-text/features/text-replacement) type: array items: $ref: "#/components/schemas/WordReplacementItem" OutputLocale: type: string description: | - Configure locale for outputted transcription. See [output formatting](https://docs.speechmatics.com/speech-to-text/formatting#output-locale). + Configure locale for outputted transcription. See [output formatting](https://docs.speechmatics.com/speech-to-text/features/formatting#output-locale). minLength: 1 LanguagePackInfo: type: object @@ -1123,11 +1123,11 @@ components: speaker: type: string description: | - Label indicating who said that word. Only set if [diarization](https://docs.speechmatics.com/speech-to-text/features/diarization) is enabled. + Label indicating who said that word. Only set if [diarization](https://docs.speechmatics.com/speech-to-text/features/speaker-diarization) is enabled. tags: type: array description: | - This is a set list of profanities and disfluencies respectively that cannot be altered by the end user. `[disfluency]` is present in the [supported languages](https://docs.speechmatics.com/speech-to-text/formatting#supported-languages-for-disfluencies), and `[profanity]` is present in English, Spanish, and Italian + This is a set list of profanities and disfluencies respectively that cannot be altered by the end user. `[disfluency]` is present in the [supported languages](https://docs.speechmatics.com/speech-to-text/features/disfluencies-and-profanities#supported-languages-for-disfluencies), and `[profanity]` is present in English, Spanish, and Italian items: $ref: "#/components/schemas/RecognitionAlternativeTagsEnum" required: @@ -1150,7 +1150,7 @@ components: MaxDelayModeConfig: type: string description: | - This allows some additional time for [Smart Formatting](https://docs.speechmatics.com/speech-to-text/formatting#smart-formatting). + This allows some additional time for [Smart Formatting](https://docs.speechmatics.com/speech-to-text/features/formatting#smart-formatting). enum: - flexible - fixed @@ -1266,8 +1266,8 @@ components: | --- | --- | | `duration_limit_exceeded` | The maximum allowed duration of a single utterance to process has been exceeded. Any `AddAudio` messages received that exceed this limit are confirmed with `AudioAdded`, but are ignored by the transcription engine. Exceeding the limit triggers the same mechanism as receiving an `EndOfStream` message, so the Server will eventually send an `EndOfTranscript` message and suspend. | `unsupported_translation_pair` | One of the requested translation target languages is unsupported (given the source audio language). The error message specifies the unsupported language pair. - | `idle_timeout` | Informs that the session is approaching the idle duration limit (no audio data sent within the last hour), with a `reason` of the form:

`Session will timeout in {time_remaining}m due to inactivity, no audio sent within the last {time_elapsed}m`

Currently the server will send messages at 15, 10 and 5m prior to timeout, and will send a final error message on timeout, before closing the connection with the code 1008. (see [Realtime limits](https://docs.speechmatics.com/speech-to-text/realtime/limits) for more information). - | `session_timeout` | Informs that the session is approaching the max session duration limit (maximum session duration of 48 hours), with a `reason` of the form:

`Session will timeout in {time_remaining}m due to max duration, session has been active for {time_elapsed}m`

Currently the server will send messages at 45, 30 and 15m prior to timeout, and will send a final error message on timeout, before closing the connection with the code 1008. (see [Realtime limits](https://docs.speechmatics.com/speech-to-text/realtime/limits) for more information).| + | `idle_timeout` | Informs that the session is approaching the idle duration limit (no audio data sent within the last hour), with a `reason` of the form:

`Session will timeout in {time_remaining}m due to inactivity, no audio sent within the last {time_elapsed}m`

Currently the server will send messages at 15, 10 and 5m prior to timeout, and will send a final error message on timeout, before closing the connection with the code 1008. (see [Realtime limits](https://docs.speechmatics.com/speech-to-text/streaming/limits) for more information). + | `session_timeout` | Informs that the session is approaching the max session duration limit (maximum session duration of 48 hours), with a `reason` of the form:

`Session will timeout in {time_remaining}m due to max duration, session has been active for {time_elapsed}m`

Currently the server will send messages at 45, 30 and 15m prior to timeout, and will send a final error message on timeout, before closing the connection with the code 1008. (see [Realtime limits](https://docs.speechmatics.com/speech-to-text/streaming/limits) for more information).| | `empty_translation_target_list` | No supported translation target languages specified. Translation will not run. | `add_audio_after_eos` | Protocol specification doesn't allow adding audio after `EndOfStream` has been received. Any `AddAudio messages after this, will be ignored. | `speaker_id` | Informs the client about any speaker ID related issues. | diff --git a/src/theme/Root.tsx b/src/theme/Root.tsx index b91822b0..aa3bf1a5 100644 --- a/src/theme/Root.tsx +++ b/src/theme/Root.tsx @@ -3,16 +3,24 @@ import MixpanelProvider from "@site/src/components/MixpanelProvider"; import React, { useEffect } from "react"; // The medical/healthcare domain docs moved from the Languages page to the Models -// page (DEL-33741). Server-side redirects can't match a URL hash, so redirect the -// legacy anchor client-side, on load and on hash change. +// page (DEL-33741), and then from the Models page to their own feature page. +// Server-side redirects can't match a URL hash, so redirect the legacy anchors +// client-side, on load and on hash change. +const MEDICAL_DOMAIN_PAGE = "/speech-to-text/features/medical-domain"; +const LEGACY_HEALTHCARE_ANCHORS = [ + "/speech-to-text/languages", + "/speech-to-text/models", +]; + function useLegacyHealthcareAnchorRedirect() { useEffect(() => { function redirect() { const { pathname, hash } = window.location; - const onLanguagesPage = - pathname.replace(/\/$/, "") === "/speech-to-text/languages"; - if (onLanguagesPage && hash === "#healthcare-domain") { - window.location.replace("/speech-to-text/models#healthcare-domain"); + const onLegacyPage = LEGACY_HEALTHCARE_ANCHORS.includes( + pathname.replace(/\/$/, ""), + ); + if (onLegacyPage && hash === "#healthcare-domain") { + window.location.replace(MEDICAL_DOMAIN_PAGE); } } redirect(); diff --git a/vercel.json b/vercel.json index 9f9a98cc..43c48ab2 100644 --- a/vercel.json +++ b/vercel.json @@ -1,129 +1,124 @@ { "trailingSlash": false, "redirects": [ + { + "source": "/features/accuracy-language-packs", + "destination": "/speech-to-text/languages#bilingual-and-multi-language-packs", + "permanent": true + }, { "source": "/speech-to-text/realtime/realtime_diarization", - "destination": "/speech-to-text/realtime/realtime-diarization", + "destination": "/speech-to-text/features/speaker-diarization", "permanent": true }, { "source": "/speech-to-text/batch/batch_diarization", - "destination": "/speech-to-text/batch/batch-diarization", + "destination": "/speech-to-text/features/speaker-diarization", "permanent": true }, { "source": "/speech-to-text/batch/tracking", - "destination": "/speech-to-text/batch/output#tracking-metadata", + "destination": "/speech-to-text/pre-recorded/output#tracking-metadata", "permanent": true }, { "source": "/speech-to-text/realtime/end-of-turn", - "destination": "/speech-to-text/realtime/turn-detection", + "destination": "/speech-to-text/features/turn-detection", "permanent": true }, { "source": "/speech-to-text/realtime/end-of-utterance", - "destination": "/speech-to-text/realtime/turn-detection", + "destination": "/speech-to-text/features/turn-detection", "permanent": true }, { "source": "/speech-to-text/realtime/realtime-speaker-identification", - "destination": "/speech-to-text/realtime/speaker-identification", + "destination": "/speech-to-text/features/speaker-identification", "permanent": true }, { "source": "/voice-agents-flow/features/application-inputs", - "destination": "/voice-agents/flow/features/application-inputs", + "destination": "/speech-to-text/agent-stt", "permanent": true }, { "source": "/voice-agents-flow/setup", - "destination": "/voice-agents/flow/setup", + "destination": "/speech-to-text/agent-stt/quickstart", "permanent": true }, { "source": "/voice-agents-flow/features/function-calling", - "destination": "/voice-agents/flow/features/function-calling", + "destination": "/integrations-and-sdks", "permanent": true }, { "source": "/voice-agents-flow", - "destination": "/voice-agents/flow", + "destination": "/speech-to-text/agent-stt", "permanent": true }, { "source": "/voice-agents-flow/supported-languages", - "destination": "/voice-agents/flow/supported-languages", + "destination": "/speech-to-text/languages", "permanent": true }, { "source": "/voice-agents-flow/features/webrtc-livekit", - "destination": "/voice-agents/flow/features/webrtc-livekit", + "destination": "/integrations-and-sdks/livekit", "permanent": true }, { "source": "/voice-agents-flow/guides/nextjs-guide", - "destination": "/voice-agents/flow/guides/nextjs-guide", + "destination": "/integrations-and-sdks", "permanent": true }, { "source": "/voice-agents-flow/guides/react-native", - "destination": "/voice-agents/flow/guides/react-native", + "destination": "/integrations-and-sdks", "permanent": true }, { "source": "/deployments/container", - "destination": "/deployements/container/accessing-images", - "permanent": true - }, - { - "source": "/voice-agents-flow", - "destination": "/voice-agents/flow", - "permanent": true - }, - { - "source": "/voice-agents-flow", - "destination": "/voice-agents/flow", + "destination": "/deployments/container/accessing-images", "permanent": true }, { "source": "/voice-agents/flow", - "destination": "/voice-agents/overview", + "destination": "/speech-to-text/agent-stt", "permanent": true }, { "source": "/voice-agents/flow/features/application-inputs", - "destination": "/voice-agents/overview", + "destination": "/speech-to-text/agent-stt", "permanent": true }, { "source": "/voice-agents/flow/features/webrtc-livekit", - "destination": "/voice-agents/overview", + "destination": "/integrations-and-sdks/livekit", "permanent": true }, { "source": "/voice-agents/flow/setup", - "destination": "/voice-agents/overview", + "destination": "/speech-to-text/agent-stt/quickstart", "permanent": true }, { "source": "/voice-agents/flow/supported-languages", - "destination": "/voice-agents/overview", + "destination": "/speech-to-text/languages", "permanent": true }, { "source": "/voice-agents/flow/features/function-calling", - "destination": "/voice-agents/overview", + "destination": "/integrations-and-sdks", "permanent": true }, { "source": "/voice-agents/flow/guides/nextjs-guide", - "destination": "/voice-agents/overview", + "destination": "/integrations-and-sdks", "permanent": true }, { "source": "/voice-agents/flow/guides/react-native", - "destination": "/voice-agents/overview", + "destination": "/integrations-and-sdks", "permanent": true }, { @@ -131,6 +126,306 @@ "destination": "/administration/projects", "permanent": true }, + { + "source": "/speech-to-text/batch", + "destination": "/speech-to-text/pre-recorded/quickstart", + "permanent": true + }, + { + "source": "/speech-to-text/realtime", + "destination": "/speech-to-text/streaming/quickstart", + "permanent": true + }, + { + "source": "/speech-to-text/batch/alignment", + "destination": "/speech-to-text/add-ons/audio-alignment", + "permanent": true + }, + { + "source": "/speech-to-text/batch/batch-diarization", + "destination": "/speech-to-text/features/speaker-diarization", + "permanent": true + }, + { + "source": "/speech-to-text/batch/input", + "destination": "/speech-to-text/pre-recorded/input", + "permanent": true + }, + { + "source": "/speech-to-text/batch/language-identification", + "destination": "/speech-to-text/pre-recorded/language-identification", + "permanent": true + }, + { + "source": "/speech-to-text/batch/limits", + "destination": "/speech-to-text/pre-recorded/limits", + "permanent": true + }, + { + "source": "/speech-to-text/batch/notifications", + "destination": "/speech-to-text/pre-recorded/notifications", + "permanent": true + }, + { + "source": "/speech-to-text/batch/output", + "destination": "/speech-to-text/pre-recorded/output", + "permanent": true + }, + { + "source": "/speech-to-text/batch/quickstart", + "destination": "/speech-to-text/pre-recorded/quickstart", + "permanent": true + }, + { + "source": "/speech-to-text/batch/speaker-identification", + "destination": "/speech-to-text/features/speaker-identification", + "permanent": true + }, + { + "source": "/speech-to-text/batch/speech-intelligence/auto-chapters", + "destination": "/speech-to-text/add-ons/chapters", + "permanent": true + }, + { + "source": "/speech-to-text/batch/speech-intelligence/sentiment-analysis", + "destination": "/speech-to-text/add-ons/sentiment", + "permanent": true + }, + { + "source": "/speech-to-text/batch/speech-intelligence/summarization", + "destination": "/speech-to-text/add-ons/summaries", + "permanent": true + }, + { + "source": "/speech-to-text/batch/speech-intelligence/topic-detection", + "destination": "/speech-to-text/add-ons/topics", + "permanent": true + }, + { + "source": "/speech-to-text/batch/srt-format", + "destination": "/speech-to-text/pre-recorded/srt-format", + "permanent": true + }, + { + "source": "/speech-to-text/batch/synchronous", + "destination": "/speech-to-text/pre-recorded/synchronous", + "permanent": true + }, + { + "source": "/speech-to-text/batch/troubleshooting", + "destination": "/speech-to-text/pre-recorded/troubleshooting", + "permanent": true + }, + { + "source": "/speech-to-text/batch/usage", + "destination": "/administration/usage", + "permanent": true + }, + { + "source": "/speech-to-text/realtime/guides/python-using-ffmpeg", + "destination": "/speech-to-text/streaming/ffmpeg-audio", + "permanent": true + }, + { + "source": "/speech-to-text/realtime/guides/python-using-microphone", + "destination": "/speech-to-text/streaming/quickstart", + "permanent": true + }, + { + "source": "/speech-to-text/realtime/input", + "destination": "/speech-to-text/streaming/input", + "permanent": true + }, + { + "source": "/speech-to-text/realtime/limits", + "destination": "/speech-to-text/streaming/limits", + "permanent": true + }, + { + "source": "/speech-to-text/realtime/output", + "destination": "/speech-to-text/streaming/output", + "permanent": true + }, + { + "source": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart", + "permanent": true + }, + { + "source": "/speech-to-text/realtime/realtime-diarization", + "destination": "/speech-to-text/features/speaker-diarization", + "permanent": true + }, + { + "source": "/speech-to-text/realtime/speaker-identification", + "destination": "/speech-to-text/features/speaker-identification", + "permanent": true + }, + { + "source": "/speech-to-text/realtime/turn-detection", + "destination": "/speech-to-text/features/turn-detection", + "permanent": true + }, + { + "source": "/speech-to-text/batch/input", + "destination": "/speech-to-text/pre-recorded/input#fetch-url", + "permanent": true + }, + { + "source": "/speech-to-text/batch/output", + "destination": "/speech-to-text/pre-recorded/output#tracking-metadata", + "permanent": true + }, + { + "source": "/speech-to-text/realtime/output", + "destination": "/speech-to-text/streaming/output#latency", + "permanent": true + }, + { + "source": "/speech-to-text/batch/speech-intelligence/auto-chapters", + "destination": "/speech-to-text/add-ons/chapters", + "permanent": true + }, + { + "source": "/private/voice-agent-api", + "destination": "/speech-to-text/agent-stt", + "permanent": true + }, + { + "source": "/speech-to-text/formatting", + "destination": "/speech-to-text/features/formatting", + "permanent": true + }, + { + "source": "/speech-to-text/formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", + "permanent": true + }, + { + "source": "/speech-to-text/formatting", + "destination": "/speech-to-text/features/formatting#punctuation", + "permanent": true + }, + { + "source": "/speech-to-text/formatting", + "destination": "/speech-to-text/features/disfluencies-and-profanities", + "permanent": true + }, + { + "source": "/speech-to-text/features/diarization", + "destination": "/speech-to-text/features/speaker-diarization", + "permanent": true + }, + { + "source": "/speech-to-text/pre-recorded/batch-diarization", + "destination": "/speech-to-text/features/speaker-diarization", + "permanent": true + }, + { + "source": "/speech-to-text/streaming/realtime-diarization", + "destination": "/speech-to-text/features/speaker-diarization", + "permanent": true + }, + { + "source": "/speech-to-text/pre-recorded/speaker-identification", + "destination": "/speech-to-text/features/speaker-identification", + "permanent": true + }, + { + "source": "/speech-to-text/streaming/speaker-identification", + "destination": "/speech-to-text/features/speaker-identification", + "permanent": true + }, + { + "source": "/speech-to-text/streaming/turn-detection", + "destination": "/speech-to-text/features/turn-detection", + "permanent": true + }, + { + "source": "/speech-to-text/streaming/guides/python-using-ffmpeg", + "destination": "/speech-to-text/streaming/ffmpeg-audio", + "permanent": true + }, + { + "source": "/speech-to-text/streaming/guides/python-using-microphone", + "destination": "/speech-to-text/streaming/quickstart", + "permanent": true + }, + { + "source": "/speech-to-text/app-analytics", + "destination": "/speech-to-text/features/app-usage-tracking", + "permanent": true + }, + { + "source": "/speech-to-text/streaming/turn-detection", + "destination": "/speech-to-text/features/turn-detection#configuration", + "permanent": true + }, + { + "source": "/speech-to-text/pre-recorded/speech-intelligence/summarization", + "destination": "/speech-to-text/add-ons/summaries", + "permanent": true + }, + { + "source": "/speech-to-text/pre-recorded/speech-intelligence/topic-detection", + "destination": "/speech-to-text/add-ons/topics", + "permanent": true + }, + { + "source": "/speech-to-text/pre-recorded/speech-intelligence/auto-chapters", + "destination": "/speech-to-text/add-ons/chapters", + "permanent": true + }, + { + "source": "/speech-to-text/pre-recorded/speech-intelligence/sentiment-analysis", + "destination": "/speech-to-text/add-ons/sentiment", + "permanent": true + }, + { + "source": "/speech-to-text/features/translation", + "destination": "/speech-to-text/add-ons/translation", + "permanent": true + }, + { + "source": "/speech-to-text/pre-recorded/alignment", + "destination": "/speech-to-text/add-ons/audio-alignment", + "permanent": true + }, + { + "source": "/speech-to-text/pre-recorded/speech-intelligence", + "destination": "/speech-to-text/add-ons/summaries", + "permanent": true + }, + { + "source": "/speech-to-text/features/translation", + "destination": "/speech-to-text/add-ons/translation#languages", + "permanent": true + }, + { + "source": "/voice-agents/overview", + "destination": "/speech-to-text/agent-stt", + "permanent": true + }, + { + "source": "/voice-agents", + "destination": "/speech-to-text/agent-stt", + "permanent": true + }, + { + "source": "/voice-agents/voice-sdk", + "destination": "/integrations-and-sdks/voice-sdk", + "permanent": true + }, + { + "source": "/speech-to-text/pre-recorded/usage", + "destination": "/administration/usage", + "permanent": true + }, + { + "source": "/get-started/quickstart", + "destination": "/", + "permanent": true + }, { "source": "/jobsapi", "destination": "/api-ref/batch/create-a-new-job", @@ -148,7 +443,7 @@ }, { "source": "/features-other/auto-chapters", - "destination": "/speech-to-text/batch/speech-intelligence/auto-chapters/", + "destination": "/speech-to-text/add-ons/chapters", "permanent": true }, { @@ -158,52 +453,52 @@ }, { "source": "/features-other/fetch-url", - "destination": "/speech-to-text/batch/input#fetch-url", + "destination": "/speech-to-text/pre-recorded/input#fetch-url", "permanent": true }, { "source": "/features-other/lang-id", - "destination": "/speech-to-text/batch/language-identification", + "destination": "/speech-to-text/pre-recorded/language-identification", "permanent": true }, { "source": "/features-other/notifications", - "destination": "/speech-to-text/batch/notifications", + "destination": "/speech-to-text/pre-recorded/notifications", "permanent": true }, { "source": "/features-other/sentiment-analysis", - "destination": "/speech-to-text/batch/speech-intelligence/sentiment-analysis", + "destination": "/speech-to-text/add-ons/sentiment", "permanent": true }, { "source": "/features-other/srt-format", - "destination": "/speech-to-text/batch/srt-format", + "destination": "/speech-to-text/pre-recorded/srt-format", "permanent": true }, { "source": "/features-other/summarization", - "destination": "/speech-to-text/batch/speech-intelligence/summarization", + "destination": "/speech-to-text/add-ons/summaries", "permanent": true }, { "source": "/features-other/topic-detection", - "destination": "/speech-to-text/batch/speech-intelligence/topic-detection", + "destination": "/speech-to-text/add-ons/topics", "permanent": true }, { "source": "/features-other/tracking", - "destination": "/speech-to-text/batch/output#tracking-metadata", + "destination": "/speech-to-text/pre-recorded/output#tracking-metadata", "permanent": true }, { "source": "/features-other/translation", - "destination": "/speech-to-text/features/translation", + "destination": "/speech-to-text/add-ons/translation", "permanent": true }, { "source": "/features-other/word-alignment", - "destination": "/speech-to-text/batch/alignment", + "destination": "/speech-to-text/add-ons/audio-alignment", "permanent": true }, { @@ -228,77 +523,77 @@ }, { "source": "/features/diarization", - "destination": "/speech-to-text/features/diarization", + "destination": "/speech-to-text/features/speaker-diarization", "permanent": true }, { "source": "/features/end-of-turn", - "destination": "/speech-to-text/realtime/turn-detection", + "destination": "/speech-to-text/features/turn-detection", "permanent": true }, { "source": "/features/entities", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/features/punctuation-settings", - "destination": "/speech-to-text/formatting#punctuation", + "destination": "/speech-to-text/features/formatting#punctuation", "permanent": true }, { "source": "/features/realtime-latency", - "destination": "/speech-to-text/realtime/output#latency", + "destination": "/speech-to-text/streaming/output#latency", "permanent": true }, { "source": "/features/word-tagging", - "destination": "/speech-to-text/formatting#output-formatting-options", + "destination": "/speech-to-text/features/disfluencies-and-profanities", "permanent": true }, { "source": "/flow-api-ref", - "destination": "/voice-agents/overview", + "destination": "/speech-to-text/agent-stt", "permanent": true }, { "source": "/flow/application-inputs", - "destination": "/voice-agents/overview", + "destination": "/speech-to-text/agent-stt", "permanent": true }, { "source": "/flow/config", - "destination": "/voice-agents/overview", + "destination": "/speech-to-text/agent-stt/quickstart", "permanent": true }, { "source": "/flow/function-calling", - "destination": "/voice-agents/overview", + "destination": "/integrations-and-sdks", "permanent": true }, { "source": "/flow/introduction", - "destination": "/voice-agents/overview", + "destination": "/speech-to-text/agent-stt", "permanent": true }, { "source": "/flow/languages-supported", - "destination": "/voice-agents/overview", + "destination": "/speech-to-text/languages", "permanent": true }, { "source": "/flow/livekit-webrtc", - "destination": "/voice-agents/overview", + "destination": "/integrations-and-sdks/livekit", "permanent": true }, { "source": "/flow/nextjs-guide", - "destination": "/voice-agents/overview", + "destination": "/integrations-and-sdks", "permanent": true }, { "source": "/flow/react-native-guide", - "destination": "/voice-agents/overview", + "destination": "/integrations-and-sdks", "permanent": true }, { @@ -308,7 +603,7 @@ }, { "source": "/introduction/additional-info", - "destination": "/speech-to-text/batch/output", + "destination": "/speech-to-text/pre-recorded/output", "permanent": true }, { @@ -318,17 +613,17 @@ }, { "source": "/introduction/batch-guide", - "destination": "/speech-to-text/batch/quickstart", + "destination": "/speech-to-text/pre-recorded/quickstart", "permanent": true }, { "source": "/introduction/errors-rate-limits", - "destination": "/speech-to-text/batch/input", + "destination": "/speech-to-text/pre-recorded/input", "permanent": true }, { "source": "/introduction/rt-guide", - "destination": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart", "permanent": true }, { @@ -338,17 +633,17 @@ }, { "source": "/introduction/troubleshooting", - "destination": "/speech-to-text/batch/troubleshooting", + "destination": "/speech-to-text/pre-recorded/troubleshooting", "permanent": true }, { "source": "/introduction/usage", - "destination": "/speech-to-text/batch/usage", + "destination": "/administration/usage", "permanent": true }, { "source": "/on-prem/containers", - "destination": "/deployements/container/accessing-images", + "destination": "/deployments/container/accessing-images", "permanent": true }, { @@ -668,12 +963,12 @@ }, { "source": "/tutorials/using-ffmpeg", - "destination": "/speech-to-text/realtime/guides/python-using-ffmpeg", + "destination": "/speech-to-text/streaming/ffmpeg-audio", "permanent": true }, { "source": "/tutorials/using-mic", - "destination": "/speech-to-text/realtime/guides/python-using-microphone", + "destination": "/speech-to-text/streaming/quickstart", "permanent": true }, { @@ -758,12 +1053,12 @@ }, { "source": "/en/cloud/understanding-saas-usage", - "destination": "/speech-to-text/batch/input", + "destination": "/speech-to-text/pre-recorded/input", "permanent": true }, { "source": "/en/cloud/troubleshooting", - "destination": "/speech-to-text/batch/input", + "destination": "/speech-to-text/pre-recorded/input", "permanent": true }, { @@ -803,7 +1098,7 @@ }, { "source": "/en/alignment-service", - "destination": "/speech-to-text/batch/alignment", + "destination": "/speech-to-text/add-ons/audio-alignment", "permanent": true }, { @@ -933,27 +1228,27 @@ }, { "source": "/features/diarization/diarization/speaker-diarization", - "destination": "/speech-to-text/features/diarization", + "destination": "/speech-to-text/features/speaker-diarization", "permanent": true }, { "source": "/features/diarization/speaker-diarization", - "destination": "/speech-to-text/features/diarization", + "destination": "/speech-to-text/features/speaker-diarization", "permanent": true }, { "source": "/features/diarization/channel-diarization", - "destination": "/speech-to-text/features/diarization", + "destination": "/speech-to-text/features/speaker-diarization", "permanent": true }, { "source": "/features/diarization/speaker-change", - "destination": "/speech-to-text/features/diarization", + "destination": "/speech-to-text/features/speaker-diarization", "permanent": true }, { "source": "/features/diarization/speaker-change-channel", - "destination": "/speech-to-text/features/diarization", + "destination": "/speech-to-text/features/speaker-diarization", "permanent": true }, { @@ -963,12 +1258,12 @@ }, { "source": "/flow", - "destination": "/voice-agents/overview", + "destination": "/speech-to-text/agent-stt", "permanent": true }, { "source": "/flow/getting-started", - "destination": "/voice-agents/overview", + "destination": "/speech-to-text/agent-stt", "permanent": true }, { @@ -988,277 +1283,277 @@ }, { "source": "/flow/flow-api-ref", - "destination": "/voice-agents/overview", + "destination": "/speech-to-text/agent-stt", "permanent": true }, { "source": "/en/cloud/howto", - "destination": "/speech-to-text/batch/quickstart", + "destination": "/speech-to-text/pre-recorded/quickstart", "permanent": true }, { "source": "/en/batch-appliance/api-guide/api-howto/v4.2.0", - "destination": "/speech-to-text/batch/quickstart", + "destination": "/speech-to-text/pre-recorded/quickstart", "permanent": true }, { "source": "/en/batch-appliance/api-guide/api-howto/v4.1.0", - "destination": "/speech-to-text/batch/quickstart", + "destination": "/speech-to-text/pre-recorded/quickstart", "permanent": true }, { "source": "/en/batch-appliance/api-guide/api-howto/v4.0.0", - "destination": "/speech-to-text/batch/quickstart", + "destination": "/speech-to-text/pre-recorded/quickstart", "permanent": true }, { "source": "/en/batch-appliance/api-guide/api-howto/v3.8.0", - "destination": "/speech-to-text/batch/quickstart", + "destination": "/speech-to-text/pre-recorded/quickstart", "permanent": true }, { "source": "/en/batch-appliance/api-guide/api-howto/v3.7.1", - "destination": "/speech-to-text/batch/quickstart", + "destination": "/speech-to-text/pre-recorded/quickstart", "permanent": true }, { "source": "/en/batch-appliance/api-guide/api-howto/v3.7.0", - "destination": "/speech-to-text/batch/quickstart", + "destination": "/speech-to-text/pre-recorded/quickstart", "permanent": true }, { "source": "/en/batch-appliance/api-guide/api-howto/v3.6.0", - "destination": "/speech-to-text/batch/quickstart", + "destination": "/speech-to-text/pre-recorded/quickstart", "permanent": true }, { "source": "/en/batch-appliance/api-guide/api-howto/v3.5.0", - "destination": "/speech-to-text/batch/quickstart", + "destination": "/speech-to-text/pre-recorded/quickstart", "permanent": true }, { "source": "/en/batch-appliance/api-guide/api-howto/v3.4.0", - "destination": "/speech-to-text/batch/quickstart", + "destination": "/speech-to-text/pre-recorded/quickstart", "permanent": true }, { "source": "/en/batch-appliance/api-guide/api-howto/v3.3.0", - "destination": "/speech-to-text/batch/quickstart", + "destination": "/speech-to-text/pre-recorded/quickstart", "permanent": true }, { "source": "/en/batch-appliance/api-guide/api-howto", - "destination": "/speech-to-text/batch/quickstart", + "destination": "/speech-to-text/pre-recorded/quickstart", "permanent": true }, { "source": "/en/cloud/entities", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/real-time-saas/entities", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/batch-appliance/entities/v4.2.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/batch-appliance/entities/v4.1.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/batch-appliance/entities/v4.0.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/batch-appliance/entities/v3.8.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/batch-appliance/entities/v3.7.1", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/batch-appliance/entities/v3.7.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/batch-appliance/entities/v3.6.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/batch-appliance/entities/v3.5.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/batch-appliance/entities/v3.4.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/batch-appliance/entities/v3.3.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/batch-appliance/entities", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/real-time-appliance/entities/v4.2.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/real-time-appliance/entities/v4.1.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/real-time-appliance/entities/v4.0.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/real-time-appliance/entities/v3.8.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/real-time-appliance/entities/v3.7.1", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/real-time-appliance/entities/v3.7.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/real-time-appliance/entities/v3.6.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/real-time-appliance/entities/v3.5.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/real-time-appliance/entities/v3.4.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/real-time-appliance/entities/v3.3.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/real-time-appliance/entities", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/batch-container/entities/v9.0.1", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/batch-container/entities/v9.0.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/batch-container/entities/v8.2.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/batch-container/entities/v8.1.2", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/batch-container/entities/v8.1.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/batch-container/entities/v8.0.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/batch-container/entities/v7.0.3", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/batch-container/entities/v6.3.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/batch-container/entities", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/real-time-container/entities/v2.0.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/real-time-container/entities/v1.4.1", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/real-time-container/entities/v1.4.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/real-time-container/entities/v1.3.1", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/real-time-container/entities/v1.3.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/real-time-container/entities/v1.2.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/real-time-container/entities/v1.1.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/real-time-container/entities/v1.0.0", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { "source": "/en/real-time-container/entities", - "destination": "/speech-to-text/formatting#smart-formatting", + "destination": "/speech-to-text/features/formatting#smart-formatting", "permanent": true }, { @@ -1488,122 +1783,122 @@ }, { "source": "/en/real-time-saas/user-guide", - "destination": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart", "permanent": true }, { "source": "/en/real-time-appliance/api-v2/speech-api-guide/v4.2.0", - "destination": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart", "permanent": true }, { "source": "/en/real-time-appliance/api-v2/speech-api-guide/v4.1.0", - "destination": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart", "permanent": true }, { "source": "/en/real-time-appliance/api-v2/speech-api-guide/v4.0.0", - "destination": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart", "permanent": true }, { "source": "/en/real-time-appliance/api-v2/speech-api-guide/v3.8.0", - "destination": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart", "permanent": true }, { "source": "/en/real-time-appliance/api-v2/speech-api-guide/v3.7.1", - "destination": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart", "permanent": true }, { "source": "/en/real-time-appliance/api-v2/speech-api-guide/v3.7.0", - "destination": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart", "permanent": true }, { "source": "/en/real-time-appliance/api-v2/speech-api-guide/v3.6.0", - "destination": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart", "permanent": true }, { "source": "/en/real-time-appliance/api-v2/speech-api-guide/v3.5.0", - "destination": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart", "permanent": true }, { "source": "/en/real-time-appliance/api-v2/speech-api-guide/v3.4.0", - "destination": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart", "permanent": true }, { "source": "/en/real-time-appliance/api-v2/speech-api-guide/v3.3.0", - "destination": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart", "permanent": true }, { "source": "/en/real-time-appliance/api-v2/speech-api-guide", - "destination": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart", "permanent": true }, { "source": "/en/real-time-appliance/api-v2/api-example-usage/v4.2.0", - "destination": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart", "permanent": true }, { "source": "/en/real-time-appliance/api-v2/api-example-usage/v4.1.0", - "destination": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart", "permanent": true }, { "source": "/en/real-time-appliance/api-v2/api-example-usage/v4.0.0", - "destination": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart", "permanent": true }, { "source": "/en/real-time-appliance/api-v2/api-example-usage/v3.8.0", - "destination": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart", "permanent": true }, { "source": "/en/real-time-appliance/api-v2/api-example-usage/v3.7.1", - "destination": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart", "permanent": true }, { "source": "/en/real-time-appliance/api-v2/api-example-usage/v3.7.0", - "destination": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart", "permanent": true }, { "source": "/en/real-time-appliance/api-v2/api-example-usage/v3.6.0", - "destination": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart", "permanent": true }, { "source": "/en/real-time-appliance/api-v2/api-example-usage/v3.5.0", - "destination": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart", "permanent": true }, { "source": "/en/real-time-appliance/api-v2/api-example-usage/v3.4.0", - "destination": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart", "permanent": true }, { "source": "/en/real-time-appliance/api-v2/api-example-usage/v3.3.0", - "destination": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart", "permanent": true }, { "source": "/en/real-time-appliance/api-v2/api-example-usage", - "destination": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart", "permanent": true }, { "source": "/en/real-time-saas", - "destination": "/speech-to-text/realtime/quickstart", + "destination": "/speech-to-text/streaming/quickstart", "permanent": true }, {