diff --git a/ai-logic/firebase-ai/CHANGELOG.md b/ai-logic/firebase-ai/CHANGELOG.md index 28a884b5f47..3c8437ead69 100644 --- a/ai-logic/firebase-ai/CHANGELOG.md +++ b/ai-logic/firebase-ai/CHANGELOG.md @@ -1,5 +1,6 @@ # Unreleased +- [feature] Added support for `RealtimeInputConfig` and `ActivityDetectionConfig` to configure voice activity detection in Live API. Added `sendStartActivityRealtime` and `sendStopActivityRealtime` to `LiveSession` for manual activity control. (#8080) - [feature] Added `getOnDeviceModelName` to `GenerativeModel` (#8247) - [changed] Deprecated `GenerativeBackend.vertexAI` in favor of `GenerativeBackend.agentPlatform` to reflect the renaming of Vertex diff --git a/ai-logic/firebase-ai/api.txt b/ai-logic/firebase-ai/api.txt index e5d63c2ef91..9b02089f183 100644 --- a/ai-logic/firebase-ai/api.txt +++ b/ai-logic/firebase-ai/api.txt @@ -307,6 +307,8 @@ package com.google.firebase.ai.java { method public abstract com.google.common.util.concurrent.ListenableFuture sendAudioRealtime(com.google.firebase.ai.type.InlineData audio); method public abstract com.google.common.util.concurrent.ListenableFuture sendFunctionResponse(java.util.List functionList); method @Deprecated public abstract com.google.common.util.concurrent.ListenableFuture sendMediaStream(java.util.List mediaChunks); + method public abstract com.google.common.util.concurrent.ListenableFuture sendStartActivityRealtime(); + method public abstract com.google.common.util.concurrent.ListenableFuture sendStopActivityRealtime(); method public abstract com.google.common.util.concurrent.ListenableFuture sendTextRealtime(String text); method public abstract com.google.common.util.concurrent.ListenableFuture sendVideoRealtime(com.google.firebase.ai.type.InlineData video); method @RequiresPermission(android.Manifest.permission.RECORD_AUDIO) public abstract com.google.common.util.concurrent.ListenableFuture startAudioConversation(); @@ -368,6 +370,43 @@ package com.google.firebase.ai.type { public final class APINotConfiguredException extends com.google.firebase.ai.type.FirebaseAIException { } + @com.google.firebase.ai.type.PublicPreviewAPI public final class ActivityDetectionConfig { + method public static com.google.firebase.ai.type.ActivityDetectionConfig.Builder builder(); + method public static com.google.firebase.ai.type.ActivityDetectionConfig disabled(); + field public static final com.google.firebase.ai.type.ActivityDetectionConfig.Companion Companion; + } + + public static final class ActivityDetectionConfig.Builder { + ctor public ActivityDetectionConfig.Builder(); + method public com.google.firebase.ai.type.ActivityDetectionConfig build(); + method public com.google.firebase.ai.type.ActivityDetectionConfig.Builder setEndSensitivity(com.google.firebase.ai.type.ActivityDetectionConfig.Sensitivity sensitivity); + method public com.google.firebase.ai.type.ActivityDetectionConfig.Builder setPrefixPaddingMs(int paddingMs); + method public com.google.firebase.ai.type.ActivityDetectionConfig.Builder setSilenceDurationMs(int durationMs); + method public com.google.firebase.ai.type.ActivityDetectionConfig.Builder setStartSensitivity(com.google.firebase.ai.type.ActivityDetectionConfig.Sensitivity sensitivity); + field public com.google.firebase.ai.type.ActivityDetectionConfig.Sensitivity? endSensitivity; + field public Integer? prefixPaddingMs; + field public Integer? silenceDurationMs; + field public com.google.firebase.ai.type.ActivityDetectionConfig.Sensitivity? startSensitivity; + } + + public static final class ActivityDetectionConfig.Companion { + method public com.google.firebase.ai.type.ActivityDetectionConfig.Builder builder(); + method public com.google.firebase.ai.type.ActivityDetectionConfig disabled(); + } + + public static final class ActivityDetectionConfig.Sensitivity { + field public static final com.google.firebase.ai.type.ActivityDetectionConfig.Sensitivity.Companion Companion; + field public static final com.google.firebase.ai.type.ActivityDetectionConfig.Sensitivity HIGH; + field public static final com.google.firebase.ai.type.ActivityDetectionConfig.Sensitivity LOW; + } + + public static final class ActivityDetectionConfig.Sensitivity.Companion { + } + + public final class ActivityDetectionConfigKt { + method public static com.google.firebase.ai.type.ActivityDetectionConfig activityDetectionConfig(kotlin.jvm.functions.Function1 init); + } + public final class AspectRatio { field public static final com.google.firebase.ai.type.AspectRatio.Companion Companion; field public static final com.google.firebase.ai.type.AspectRatio LANDSCAPE_16x9; @@ -1452,6 +1491,7 @@ package com.google.firebase.ai.type { method public com.google.firebase.ai.type.LiveGenerationConfig.Builder setMaxOutputTokens(Integer? maxOutputTokens); method public com.google.firebase.ai.type.LiveGenerationConfig.Builder setOutputAudioTranscription(com.google.firebase.ai.type.AudioTranscriptionConfig? config); method public com.google.firebase.ai.type.LiveGenerationConfig.Builder setPresencePenalty(Float? presencePenalty); + method public com.google.firebase.ai.type.LiveGenerationConfig.Builder setRealtimeInputConfig(com.google.firebase.ai.type.RealtimeInputConfig config); method public com.google.firebase.ai.type.LiveGenerationConfig.Builder setResponseModality(com.google.firebase.ai.type.ResponseModality? responseModality); method public com.google.firebase.ai.type.LiveGenerationConfig.Builder setSpeechConfig(com.google.firebase.ai.type.SpeechConfig? speechConfig); method public com.google.firebase.ai.type.LiveGenerationConfig.Builder setTemperature(Float? temperature); @@ -1463,6 +1503,7 @@ package com.google.firebase.ai.type { field public Integer? maxOutputTokens; field public com.google.firebase.ai.type.AudioTranscriptionConfig? outputAudioTranscription; field public Float? presencePenalty; + field public com.google.firebase.ai.type.RealtimeInputConfig? realtimeInputConfig; field public com.google.firebase.ai.type.ResponseModality? responseModality; field public com.google.firebase.ai.type.SpeechConfig? speechConfig; field public Float? temperature; @@ -1536,6 +1577,8 @@ package com.google.firebase.ai.type { method public suspend Object? sendAudioRealtime(com.google.firebase.ai.type.InlineData audio, kotlin.coroutines.Continuation); method public suspend Object? sendFunctionResponse(java.util.List functionList, kotlin.coroutines.Continuation); method @Deprecated public suspend Object? sendMediaStream(java.util.List mediaChunks, kotlin.coroutines.Continuation); + method public suspend Object? sendStartActivityRealtime(kotlin.coroutines.Continuation); + method public suspend Object? sendStopActivityRealtime(kotlin.coroutines.Continuation); method public suspend Object? sendTextRealtime(String text, kotlin.coroutines.Continuation); method public suspend Object? sendVideoRealtime(com.google.firebase.ai.type.InlineData video, kotlin.coroutines.Continuation); method @RequiresPermission(android.Manifest.permission.RECORD_AUDIO) public suspend Object? startAudioConversation(com.google.firebase.ai.type.LiveAudioConversationConfig liveAudioConversationConfig, kotlin.coroutines.Continuation); @@ -1617,6 +1660,48 @@ package com.google.firebase.ai.type { public final class QuotaExceededException extends com.google.firebase.ai.type.FirebaseAIException { } + @com.google.firebase.ai.type.PublicPreviewAPI public final class RealtimeInputConfig { + method public static com.google.firebase.ai.type.RealtimeInputConfig.Builder builder(); + field public static final com.google.firebase.ai.type.RealtimeInputConfig.Companion Companion; + } + + public static final class RealtimeInputConfig.ActivityHandling { + field public static final com.google.firebase.ai.type.RealtimeInputConfig.ActivityHandling.Companion Companion; + field public static final com.google.firebase.ai.type.RealtimeInputConfig.ActivityHandling INTERRUPT; + field public static final com.google.firebase.ai.type.RealtimeInputConfig.ActivityHandling NO_INTERRUPT; + } + + public static final class RealtimeInputConfig.ActivityHandling.Companion { + } + + public static final class RealtimeInputConfig.Builder { + ctor public RealtimeInputConfig.Builder(); + method public com.google.firebase.ai.type.RealtimeInputConfig build(); + method public com.google.firebase.ai.type.RealtimeInputConfig.Builder setActivityHandling(com.google.firebase.ai.type.RealtimeInputConfig.ActivityHandling handling); + method public com.google.firebase.ai.type.RealtimeInputConfig.Builder setAutomaticActivityDetection(com.google.firebase.ai.type.ActivityDetectionConfig config); + method public com.google.firebase.ai.type.RealtimeInputConfig.Builder setTurnCoverage(com.google.firebase.ai.type.RealtimeInputConfig.TurnCoverage coverage); + field public com.google.firebase.ai.type.RealtimeInputConfig.ActivityHandling? activityHandling; + field public com.google.firebase.ai.type.ActivityDetectionConfig? automaticActivityDetection; + field public com.google.firebase.ai.type.RealtimeInputConfig.TurnCoverage? turnCoverage; + } + + public static final class RealtimeInputConfig.Companion { + method public com.google.firebase.ai.type.RealtimeInputConfig.Builder builder(); + } + + public static final class RealtimeInputConfig.TurnCoverage { + field public static final com.google.firebase.ai.type.RealtimeInputConfig.TurnCoverage ALL_INPUT; + field public static final com.google.firebase.ai.type.RealtimeInputConfig.TurnCoverage.Companion Companion; + field public static final com.google.firebase.ai.type.RealtimeInputConfig.TurnCoverage ONLY_ACTIVITY; + } + + public static final class RealtimeInputConfig.TurnCoverage.Companion { + } + + public final class RealtimeInputConfigKt { + method public static com.google.firebase.ai.type.RealtimeInputConfig realtimeInputConfig(kotlin.jvm.functions.Function1 init); + } + public final class RequestOptions { ctor public RequestOptions(); ctor public RequestOptions(long timeoutInMillis = 180.seconds.inWholeMilliseconds); diff --git a/ai-logic/firebase-ai/src/androidTest/kotlin/com/google/firebase/ai/LiveSessionTests.kt b/ai-logic/firebase-ai/src/androidTest/kotlin/com/google/firebase/ai/LiveSessionTests.kt index 39ac45de7dd..8c605a3769a 100644 --- a/ai-logic/firebase-ai/src/androidTest/kotlin/com/google/firebase/ai/LiveSessionTests.kt +++ b/ai-logic/firebase-ai/src/androidTest/kotlin/com/google/firebase/ai/LiveSessionTests.kt @@ -20,6 +20,7 @@ import android.content.Context import android.graphics.Bitmap import android.media.MediaMetadataRetriever import androidx.test.core.app.ApplicationProvider +import com.google.firebase.ai.type.ActivityDetectionConfig import com.google.firebase.ai.type.AudioTranscriptionConfig import com.google.firebase.ai.type.FunctionDeclaration import com.google.firebase.ai.type.FunctionResponsePart @@ -29,14 +30,17 @@ import com.google.firebase.ai.type.LiveServerToolCall import com.google.firebase.ai.type.LiveSession import com.google.firebase.ai.type.LiveSessionResumptionUpdate import com.google.firebase.ai.type.PublicPreviewAPI +import com.google.firebase.ai.type.RealtimeInputConfig import com.google.firebase.ai.type.ResponseModality import com.google.firebase.ai.type.Schema import com.google.firebase.ai.type.SessionResumptionConfig import com.google.firebase.ai.type.SpeechConfig import com.google.firebase.ai.type.Tool import com.google.firebase.ai.type.Voice +import com.google.firebase.ai.type.activityDetectionConfig import com.google.firebase.ai.type.content import com.google.firebase.ai.type.liveGenerationConfig +import com.google.firebase.ai.type.realtimeInputConfig import io.kotest.matchers.ints.shouldBeGreaterThan import io.kotest.matchers.longs.shouldBeGreaterThan import io.kotest.matchers.nulls.shouldNotBeNull @@ -323,4 +327,64 @@ class LiveSessionTests { .collect {} return transcriptBuilder.toString() } + + @Test + fun testRealtimeInputConfig_manualActivity(): Unit = runBlocking { + val config = liveGenerationConfig { + responseModality = ResponseModality.AUDIO + outputAudioTranscription = AudioTranscriptionConfig() + realtimeInputConfig = realtimeInputConfig { + automaticActivityDetection = ActivityDetectionConfig.disabled() + } + } + val liveModel = + AIModels.getGoogleLiveModel( + modelName = modelName, + config = config, + systemInstruction = SystemInstructions.yesOrNo + ) + val session = liveModel.connect() + try { + session.sendStartActivityRealtime() + session.sendTextRealtime("Does five plus five equal ten?") + session.sendStopActivityRealtime() + + val text = withTimeoutOrNull(15.seconds) { session.collectNextAudioOutputTranscript() } ?: "" + text.toLowerCasePreservingASCIIRules() shouldContain "yes" + } finally { + session.close() + } + } + + @Test + fun testRealtimeInputConfig_fullConfiguration(): Unit = runBlocking { + val config = liveGenerationConfig { + responseModality = ResponseModality.AUDIO + outputAudioTranscription = AudioTranscriptionConfig() + realtimeInputConfig = realtimeInputConfig { + activityHandling = RealtimeInputConfig.ActivityHandling.NO_INTERRUPT + turnCoverage = RealtimeInputConfig.TurnCoverage.ONLY_ACTIVITY + automaticActivityDetection = activityDetectionConfig { + startSensitivity = ActivityDetectionConfig.Sensitivity.HIGH + endSensitivity = ActivityDetectionConfig.Sensitivity.LOW + prefixPaddingMs = 100 + silenceDurationMs = 500 + } + } + } + val liveModel = + AIModels.getGoogleLiveModel( + modelName = modelName, + config = config, + systemInstruction = SystemInstructions.yesOrNo + ) + val session = liveModel.connect() + try { + session.sendTextRealtime("Is the sky blue?") + val text = withTimeoutOrNull(15.seconds) { session.collectNextAudioOutputTranscript() } ?: "" + text.toLowerCasePreservingASCIIRules() shouldContain "yes" + } finally { + session.close() + } + } } diff --git a/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/LiveGenerativeModel.kt b/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/LiveGenerativeModel.kt index 5fa381dd16a..55e2008382a 100644 --- a/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/LiveGenerativeModel.kt +++ b/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/LiveGenerativeModel.kt @@ -125,7 +125,8 @@ internal constructor( config?.inputAudioTranscription?.toInternal(), config?.outputAudioTranscription?.toInternal(), resumption?.toInternal(), - config?.contextWindowCompression?.toInternal() + config?.contextWindowCompression?.toInternal(), + config?.realtimeInputConfig?.toInternal() ) .toInternal() val data: String = JSON.encodeToString(clientMessage) diff --git a/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/LiveSessionFutures.kt b/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/LiveSessionFutures.kt index 604f0180f61..eb85bed2f9f 100644 --- a/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/LiveSessionFutures.kt +++ b/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/LiveSessionFutures.kt @@ -220,6 +220,18 @@ public abstract class LiveSessionFutures internal constructor() { */ public abstract fun sendTextRealtime(text: String): ListenableFuture + /** + * Manually marks the start of user activity. Required only when automatic activity detection is + * disabled. + */ + public abstract fun sendStartActivityRealtime(): ListenableFuture + + /** + * Manually marks the end of user activity. Required only when automatic activity detection is + * disabled. + */ + public abstract fun sendStopActivityRealtime(): ListenableFuture + /** * Streams client data to the model. * @@ -294,6 +306,12 @@ public abstract class LiveSessionFutures internal constructor() { override fun sendTextRealtime(text: String): ListenableFuture = SuspendToFutureAdapter.launchFuture { session.sendTextRealtime(text) } + override fun sendStartActivityRealtime(): ListenableFuture = + SuspendToFutureAdapter.launchFuture { session.sendStartActivityRealtime() } + + override fun sendStopActivityRealtime(): ListenableFuture = + SuspendToFutureAdapter.launchFuture { session.sendStopActivityRealtime() } + override fun sendMediaStream(mediaChunks: List) = SuspendToFutureAdapter.launchFuture { session.sendMediaStream(mediaChunks) } diff --git a/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ActivityDetectionConfig.kt b/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ActivityDetectionConfig.kt new file mode 100644 index 00000000000..278b4fe4562 --- /dev/null +++ b/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ActivityDetectionConfig.kt @@ -0,0 +1,154 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.ai.type + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +/** Configures the model's automatic detection of user activity. */ +@PublicPreviewAPI +public class ActivityDetectionConfig +private constructor( + internal val startSensitivity: Sensitivity?, + internal val endSensitivity: Sensitivity?, + internal val prefixPaddingMs: Int?, + internal val silenceDurationMs: Int?, + internal val disabled: Boolean? +) { + + /** How sensitive the model interprets speech activity. */ + public class Sensitivity private constructor(internal val value: String) { + public companion object { + /** + * The model will detect speech less often. In other words, a higher volume of speech is + * required for the model to consider the user is speaking. + */ + @JvmField public val LOW: Sensitivity = Sensitivity("LOW") + + /** + * The model will detect speech more often. In other words, a lower volume of speech is + * required for the model to consider the user is speaking. + */ + @JvmField public val HIGH: Sensitivity = Sensitivity("HIGH") + } + } + + /** Builder for creating an [ActivityDetectionConfig]. */ + public class Builder { + /** Determines how likely the start of speech is detected. */ + @JvmField public var startSensitivity: Sensitivity? = null + + /** Determines how likely the end of speech is detected. */ + @JvmField public var endSensitivity: Sensitivity? = null + + /** + * How long (in milliseconds) detected speech should be present before start-of-speech is + * committed. + * + * The lower this value, the more sensitive the start-of-speech detection is and the shorter the + * speech that can be recognized. However, this also increases the probability of false + * positives. + */ + @JvmField public var prefixPaddingMs: Int? = null + + /** + * How long (in milliseconds) silence (or non-speech) should be present before end-of-speech is + * committed. + * + * The larger this value, the longer speech gaps can be without interrupting the user's + * activity, but this will increase the model's latency. + */ + @JvmField public var silenceDurationMs: Int? = null + + /** Sets [startSensitivity]. */ + public fun setStartSensitivity(sensitivity: Sensitivity): Builder = apply { + startSensitivity = sensitivity + } + + /** Sets [endSensitivity]. */ + public fun setEndSensitivity(sensitivity: Sensitivity): Builder = apply { + endSensitivity = sensitivity + } + + /** Sets [prefixPaddingMs] in milliseconds. */ + public fun setPrefixPaddingMs(paddingMs: Int): Builder = apply { prefixPaddingMs = paddingMs } + + /** Sets [silenceDurationMs] in milliseconds. */ + public fun setSilenceDurationMs(durationMs: Int): Builder = apply { + silenceDurationMs = durationMs + } + + /** Creates a new [ActivityDetectionConfig] with the configured options. */ + public fun build(): ActivityDetectionConfig = + ActivityDetectionConfig( + startSensitivity, + endSensitivity, + prefixPaddingMs, + silenceDurationMs, + null + ) + } + + internal fun toInternal(): Internal = + Internal( + startSensitivity = startSensitivity?.let { "START_SENSITIVITY_${it.value}" }, + endSensitivity = endSensitivity?.let { "END_SENSITIVITY_${it.value}" }, + prefixPaddingMs = prefixPaddingMs, + silenceDurationMs = silenceDurationMs, + disabled = disabled + ) + + @Serializable + internal data class Internal( + @SerialName("start_of_speech_sensitivity") val startSensitivity: String? = null, + @SerialName("end_of_speech_sensitivity") val endSensitivity: String? = null, + @SerialName("prefix_padding_ms") val prefixPaddingMs: Int? = null, + @SerialName("silence_duration_ms") val silenceDurationMs: Int? = null, + val disabled: Boolean? = null + ) + + public companion object { + /** Creates a new [Builder]. */ + @JvmStatic public fun builder(): Builder = Builder() + + /** + * Disables automatic activity detection. + * + * When automatic activity detection is disabled, the user must send activity signals manually + * using [LiveSession.sendStartActivityRealtime] and [LiveSession.sendStopActivityRealtime]. + */ + @JvmStatic + public fun disabled(): ActivityDetectionConfig = + ActivityDetectionConfig( + startSensitivity = null, + endSensitivity = null, + prefixPaddingMs = null, + silenceDurationMs = null, + disabled = true + ) + } +} + +/** Helper method to construct an [ActivityDetectionConfig] in a DSL-like manner. */ +@OptIn(PublicPreviewAPI::class) +public fun activityDetectionConfig( + init: ActivityDetectionConfig.Builder.() -> Unit +): ActivityDetectionConfig { + val builder = ActivityDetectionConfig.builder() + builder.init() + return builder.build() +} diff --git a/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveClientSetupMessage.kt b/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveClientSetupMessage.kt index bbe1633b97e..8a521afc234 100644 --- a/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveClientSetupMessage.kt +++ b/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveClientSetupMessage.kt @@ -38,6 +38,7 @@ internal class LiveClientSetupMessage( val outputAudioTranscription: AudioTranscriptionConfig.Internal?, val sessionResumption: SessionResumptionConfig.Internal?, val contextWindowCompression: ContextWindowCompressionConfig.Internal?, + val realtimeInputConfig: RealtimeInputConfig.Internal?, ) { @Serializable internal class Internal(val setup: LiveClientSetup) { @@ -53,6 +54,8 @@ internal class LiveClientSetupMessage( val sessionResumption: SessionResumptionConfig.Internal? = null, @SerialName("context_window_compression") val contextWindowCompression: ContextWindowCompressionConfig.Internal? = null, + @SerialName("realtime_input_config") + val realtimeInputConfig: RealtimeInputConfig.Internal? = null, ) } @@ -66,7 +69,8 @@ internal class LiveClientSetupMessage( inputAudioTranscription, outputAudioTranscription, sessionResumption, - contextWindowCompression + contextWindowCompression, + realtimeInputConfig ) ) } diff --git a/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveGenerationConfig.kt b/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveGenerationConfig.kt index 764100fe9b2..057a7d3015c 100644 --- a/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveGenerationConfig.kt +++ b/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveGenerationConfig.kt @@ -58,6 +58,8 @@ import kotlinx.serialization.Serializable * @property outputAudioTranscription Specifies the configuration for transcribing output audio from * the model. * + * @property realtimeInputConfig Configures realtime input for the session. + * * Refer to the * [Control generated output](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/control-generated-output) * guide for more details. @@ -76,6 +78,7 @@ private constructor( internal val inputAudioTranscription: AudioTranscriptionConfig?, internal val outputAudioTranscription: AudioTranscriptionConfig?, internal val contextWindowCompression: ContextWindowCompressionConfig?, + internal val realtimeInputConfig: RealtimeInputConfig?, ) { /** @@ -105,6 +108,8 @@ private constructor( * @property outputAudioTranscription see [LiveGenerationConfig.outputAudioTranscription] * * @property contextWindowCompression see [LiveGenerationConfig.contextWindowCompression] + * + * @property realtimeInputConfig see [LiveGenerationConfig.realtimeInputConfig] */ public class Builder { @JvmField public var temperature: Float? = null @@ -118,6 +123,7 @@ private constructor( @JvmField public var inputAudioTranscription: AudioTranscriptionConfig? = null @JvmField public var outputAudioTranscription: AudioTranscriptionConfig? = null @JvmField public var contextWindowCompression: ContextWindowCompressionConfig? = null + @JvmField public var realtimeInputConfig: RealtimeInputConfig? = null public fun setTemperature(temperature: Float?): Builder = apply { this.temperature = temperature @@ -153,6 +159,10 @@ private constructor( this.contextWindowCompression = config } + public fun setRealtimeInputConfig(config: RealtimeInputConfig): Builder = apply { + this.realtimeInputConfig = config + } + /** Create a new [LiveGenerationConfig] with the attached arguments. */ public fun build(): LiveGenerationConfig = LiveGenerationConfig( @@ -167,6 +177,7 @@ private constructor( inputAudioTranscription = inputAudioTranscription, outputAudioTranscription = outputAudioTranscription, contextWindowCompression = contextWindowCompression, + realtimeInputConfig = realtimeInputConfig, ) } diff --git a/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt b/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt index 23a549607a1..c8491cd8a05 100644 --- a/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt +++ b/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt @@ -61,6 +61,7 @@ import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.isActive import kotlinx.coroutines.launch import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable import kotlinx.serialization.encodeToString @@ -432,6 +433,36 @@ internal constructor( sendFrame(BidiGenerateContentRealtimeInputSetup(text = text).toInternal()) } + /** + * Manually marks the start of user activity, using the realtime API. + * + * The start of user activity is effectively the start of a user's turn, but depending on the + * configuration defined in [RealtimeInputConfig], it may not be interpreted as an interruption. + * An example of the start of user activity could be the user speaking (not silence). + * + * Should be followed with a call to [sendStopActivityRealtime] after all the data has been sent + * for the user's turn. + * + * Only required when automatic activity detection is disabled via [RealtimeInputConfig]. + */ + public suspend fun sendStartActivityRealtime() { + sendFrame(BidiGenerateContentRealtimeInputSetup(activityStart = true).toInternal()) + } + + /** + * Manually marks the end of user activity, using the realtime API. + * + * The end of user activity is effectively the end of a user's turn, and signals that the model + * can start sending responses. + * + * Should follow after a previous call to [sendStartActivityRealtime]. + * + * Only required when automatic activity detection is disabled via [RealtimeInputConfig]. + */ + public suspend fun sendStopActivityRealtime() { + sendFrame(BidiGenerateContentRealtimeInputSetup(activityEnd = true).toInternal()) + } + /** * Streams client data to the model. * @@ -716,8 +747,13 @@ internal constructor( val mediaChunks: List? = null, val audio: InlineData? = null, val video: InlineData? = null, - val text: String? = null + val text: String? = null, + val activityStart: Boolean = false, + val activityEnd: Boolean = false ) { + @Serializable internal class ActivityStart + @Serializable internal class ActivityEnd + @Serializable internal class Internal(val realtimeInput: BidiGenerateContentRealtimeInput) { @Serializable @@ -725,7 +761,9 @@ internal constructor( val mediaChunks: List?, val audio: InlineData.Internal?, val video: InlineData.Internal?, - val text: String? + val text: String?, + @SerialName("activity_start") val activityStart: ActivityStart? = null, + @SerialName("activity_end") val activityEnd: ActivityEnd? = null ) } fun toInternal() = @@ -734,7 +772,9 @@ internal constructor( mediaChunks?.map { it.toInternal() }, audio?.toInternal(), video?.toInternal(), - text + text, + if (activityStart) ActivityStart() else null, + if (activityEnd) ActivityEnd() else null ) ) } diff --git a/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/RealtimeInputConfig.kt b/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/RealtimeInputConfig.kt new file mode 100644 index 00000000000..af00f46f593 --- /dev/null +++ b/ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/RealtimeInputConfig.kt @@ -0,0 +1,132 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.ai.type + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +/** + * Configures model input behavior when generating content in the Live API via realtime supported + * methods. + */ +@PublicPreviewAPI +public class RealtimeInputConfig +private constructor( + internal val automaticActivityDetection: ActivityDetectionConfig?, + internal val activityHandling: ActivityHandling?, + internal val turnCoverage: TurnCoverage? +) { + + /** How a model handles user input activity. */ + public class ActivityHandling private constructor(internal val value: String) { + public companion object { + /** + * When the user sends input marking the start of activity, the model's current response will + * be cut-off immediately. + */ + @JvmField + public val INTERRUPT: ActivityHandling = ActivityHandling("START_OF_ACTIVITY_INTERRUPTS") + + /** + * When the user sends input marking the start of activity, the model will process it, but + * won't cut-off its current response. + */ + @JvmField public val NO_INTERRUPT: ActivityHandling = ActivityHandling("NO_INTERRUPTION") + } + } + + /** How the model considers which input is included in the user's turn. */ + public class TurnCoverage private constructor(internal val value: String) { + public companion object { + /** + * The model will exclude inactivity (e.g, silence on the audio stream) from the user's input. + */ + @JvmField public val ONLY_ACTIVITY: TurnCoverage = TurnCoverage("TURN_INCLUDES_ONLY_ACTIVITY") + + /** + * The model will include all input (including inactivity) since the last turn as the user's + * input. + */ + @JvmField public val ALL_INPUT: TurnCoverage = TurnCoverage("TURN_INCLUDES_ALL_INPUT") + } + } + + /** Builder for creating a [RealtimeInputConfig]. */ + public class Builder { + /** + * Configures automatic activity detection on the model. + * + * When not set, automatic activity detection is enabled by default. If set, the user must send + * activity signals. + */ + @JvmField public var automaticActivityDetection: ActivityDetectionConfig? = null + + /** Defines how the model treats user input activity. */ + @JvmField public var activityHandling: ActivityHandling? = null + + /** + * Defines which input is included in the user's turn, relative to the starting and ending of + * the activity. + */ + @JvmField public var turnCoverage: TurnCoverage? = null + + /** Sets [automaticActivityDetection]. */ + public fun setAutomaticActivityDetection(config: ActivityDetectionConfig): Builder = apply { + automaticActivityDetection = config + } + + /** Sets [activityHandling]. */ + public fun setActivityHandling(handling: ActivityHandling): Builder = apply { + activityHandling = handling + } + + /** Sets [turnCoverage]. */ + public fun setTurnCoverage(coverage: TurnCoverage): Builder = apply { turnCoverage = coverage } + + /** Creates a new [RealtimeInputConfig] with the configured options. */ + public fun build(): RealtimeInputConfig = + RealtimeInputConfig(automaticActivityDetection, activityHandling, turnCoverage) + } + + internal fun toInternal(): Internal = + Internal( + automaticActivityDetection = automaticActivityDetection?.toInternal(), + activityHandling = activityHandling?.value, + turnCoverage = turnCoverage?.value + ) + + @Serializable + internal data class Internal( + @SerialName("automatic_activity_detection") + val automaticActivityDetection: ActivityDetectionConfig.Internal? = null, + @SerialName("activity_handling") val activityHandling: String? = null, + @SerialName("turn_coverage") val turnCoverage: String? = null + ) + + public companion object { + /** Creates a new [Builder]. */ + @JvmStatic public fun builder(): Builder = Builder() + } +} + +/** Helper method to construct a [RealtimeInputConfig] in a DSL-like manner. */ +@OptIn(PublicPreviewAPI::class) +public fun realtimeInputConfig(init: RealtimeInputConfig.Builder.() -> Unit): RealtimeInputConfig { + val builder = RealtimeInputConfig.builder() + builder.init() + return builder.build() +} diff --git a/ai-logic/firebase-ai/src/test/java/com/google/firebase/ai/type/RealtimeInputConfigTest.kt b/ai-logic/firebase-ai/src/test/java/com/google/firebase/ai/type/RealtimeInputConfigTest.kt new file mode 100644 index 00000000000..f903e4448c4 --- /dev/null +++ b/ai-logic/firebase-ai/src/test/java/com/google/firebase/ai/type/RealtimeInputConfigTest.kt @@ -0,0 +1,153 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.ai.type + +import com.google.firebase.ai.common.JSON +import com.google.firebase.ai.type.RealtimeInputConfig.ActivityHandling +import com.google.firebase.ai.type.RealtimeInputConfig.TurnCoverage +import io.kotest.assertions.json.shouldEqualJson +import io.kotest.matchers.equals.shouldBeEqual +import kotlinx.serialization.encodeToString +import org.junit.Test + +@OptIn(PublicPreviewAPI::class) +internal class RealtimeInputConfigTest { + + @Test + fun `Basic RealtimeInputConfig`() { + val config = realtimeInputConfig { + activityHandling = ActivityHandling.NO_INTERRUPT + turnCoverage = TurnCoverage.ONLY_ACTIVITY + automaticActivityDetection = ActivityDetectionConfig.disabled() + } + + val expectedJson = + """ + { + "activity_handling": "NO_INTERRUPTION", + "turn_coverage": "TURN_INCLUDES_ONLY_ACTIVITY", + "automatic_activity_detection": { + "disabled": true + } + } + """ + .trimIndent() + + JSON.encodeToString(config.toInternal()).shouldEqualJson(expectedJson) + } + + @Test + fun `ActivityDetectionConfig full config`() { + val detection = activityDetectionConfig { + startSensitivity = ActivityDetectionConfig.Sensitivity.HIGH + endSensitivity = ActivityDetectionConfig.Sensitivity.LOW + prefixPaddingMs = 100 + silenceDurationMs = 500 + } + + val expectedJson = + """ + { + "start_of_speech_sensitivity": "START_SENSITIVITY_HIGH", + "end_of_speech_sensitivity": "END_SENSITIVITY_LOW", + "prefix_padding_ms": 100, + "silence_duration_ms": 500 + } + """ + .trimIndent() + + JSON.encodeToString(detection.toInternal()).shouldEqualJson(expectedJson) + } + + @Test + fun `DSL correctly delegates to Builder`() { + val config = + RealtimeInputConfig.builder().setActivityHandling(ActivityHandling.INTERRUPT).build() + + val configDsl = realtimeInputConfig { activityHandling = ActivityHandling.INTERRUPT } + + config.activityHandling?.shouldBeEqual(configDsl.activityHandling as ActivityHandling) + } + + @Test + fun `LiveClientSetupMessage includes RealtimeInputConfig in serialization`() { + val config = liveGenerationConfig { + realtimeInputConfig = realtimeInputConfig { activityHandling = ActivityHandling.NO_INTERRUPT } + } + val message = + LiveClientSetupMessage( + model = "my-model", + generationConfig = null, + tools = null, + systemInstruction = null, + inputAudioTranscription = null, + outputAudioTranscription = null, + sessionResumption = null, + contextWindowCompression = null, + realtimeInputConfig = config.realtimeInputConfig?.toInternal() + ) + + val expectedJson = + """ + { + "setup": { + "model": "my-model", + "realtime_input_config": { + "activity_handling": "NO_INTERRUPTION" + } + } + } + """ + .trimIndent() + + JSON.encodeToString(message.toInternal()).shouldEqualJson(expectedJson) + } + + @Test + fun `BidiGenerateContentRealtimeInputSetup with activityStart serializes correctly`() { + val setup = LiveSession.BidiGenerateContentRealtimeInputSetup(activityStart = true) + + val expectedJson = + """ + { + "realtimeInput": { + "activity_start": {} + } + } + """ + .trimIndent() + + JSON.encodeToString(setup.toInternal()).shouldEqualJson(expectedJson) + } + + @Test + fun `BidiGenerateContentRealtimeInputSetup with activityEnd serializes correctly`() { + val setup = LiveSession.BidiGenerateContentRealtimeInputSetup(activityEnd = true) + + val expectedJson = + """ + { + "realtimeInput": { + "activity_end": {} + } + } + """ + .trimIndent() + + JSON.encodeToString(setup.toInternal()).shouldEqualJson(expectedJson) + } +} diff --git a/ai-logic/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java b/ai-logic/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java index 5ab0122f186..dbf7bb0f14a 100644 --- a/ai-logic/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java +++ b/ai-logic/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java @@ -34,6 +34,7 @@ import com.google.firebase.ai.java.LiveSessionFutures; import com.google.firebase.ai.java.TemplateChatFutures; import com.google.firebase.ai.java.TemplateGenerativeModelFutures; +import com.google.firebase.ai.type.ActivityDetectionConfig; import com.google.firebase.ai.type.AspectRatio; import com.google.firebase.ai.type.BlockReason; import com.google.firebase.ai.type.Candidate; @@ -75,6 +76,7 @@ import com.google.firebase.ai.type.Part; import com.google.firebase.ai.type.PromptFeedback; import com.google.firebase.ai.type.PublicPreviewAPI; +import com.google.firebase.ai.type.RealtimeInputConfig; import com.google.firebase.ai.type.ResponseModality; import com.google.firebase.ai.type.RetrievalConfig; import com.google.firebase.ai.type.SafetyRating; @@ -193,6 +195,18 @@ private LiveGenerationConfig getLiveConfig() { .setPresencePenalty(2.0F) .setResponseModality(ResponseModality.AUDIO) .setSpeechConfig(new SpeechConfig(new Voice("AOEDE"))) + .setRealtimeInputConfig( + new RealtimeInputConfig.Builder() + .setActivityHandling(RealtimeInputConfig.ActivityHandling.NO_INTERRUPT) + .setTurnCoverage(RealtimeInputConfig.TurnCoverage.ONLY_ACTIVITY) + .setAutomaticActivityDetection( + new ActivityDetectionConfig.Builder() + .setStartSensitivity(ActivityDetectionConfig.Sensitivity.HIGH) + .setEndSensitivity(ActivityDetectionConfig.Sensitivity.LOW) + .setPrefixPaddingMs(100) + .setSilenceDurationMs(500) + .build()) + .build()) .build(); } @@ -470,6 +484,8 @@ public void onComplete() { session.sendAudioRealtime(new InlineData(bytes, "audio/jxl", null)); session.sendVideoRealtime(new InlineData(bytes, "image/jxl", null)); session.sendTextRealtime("text"); + session.sendStartActivityRealtime(); + session.sendStopActivityRealtime(); FunctionResponsePart functionResponse = new FunctionResponsePart("myFunction", new JsonObject(Map.of()));