[AI] Implement RealtimeInputConfig and Manual Activity Signals for Live API

ai-logic/firebase-ai/CHANGELOG.md

@@ -1,5 +1,6 @@
 # Unreleased
 
+- [feature] Added support for `RealtimeInputConfig` and `ActivityDetectionConfig` to configure voice activity detection in Live API. Added `sendStartActivityRealtime` and `sendStopActivityRealtime` to `LiveSession` for manual activity control. (#8080)
 - [feature] Added `getOnDeviceModelName` to `GenerativeModel` (#8247)
 - [changed] Deprecated `GenerativeBackend.vertexAI` in favor of
   `GenerativeBackend.agentPlatform` to reflect the renaming of Vertex

ai-logic/firebase-ai/api.txt

@@ -307,6 +307,8 @@ package com.google.firebase.ai.java {
     method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> sendAudioRealtime(com.google.firebase.ai.type.InlineData audio);
     method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> sendFunctionResponse(java.util.List<com.google.firebase.ai.type.FunctionResponsePart> functionList);
     method @Deprecated public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> sendMediaStream(java.util.List<com.google.firebase.ai.type.MediaData> mediaChunks);
+    method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> sendStartActivityRealtime();
+    method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> sendStopActivityRealtime();
     method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> sendTextRealtime(String text);
     method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> sendVideoRealtime(com.google.firebase.ai.type.InlineData video);
     method @RequiresPermission(android.Manifest.permission.RECORD_AUDIO) public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> startAudioConversation();
@@ -368,6 +370,43 @@ package com.google.firebase.ai.type {
   public final class APINotConfiguredException extends com.google.firebase.ai.type.FirebaseAIException {
   }
 
+  @com.google.firebase.ai.type.PublicPreviewAPI public final class ActivityDetectionConfig {
+    method public static com.google.firebase.ai.type.ActivityDetectionConfig.Builder builder();
+    method public static com.google.firebase.ai.type.ActivityDetectionConfig disabled();
+    field public static final com.google.firebase.ai.type.ActivityDetectionConfig.Companion Companion;
+  }
+
+  public static final class ActivityDetectionConfig.Builder {
+    ctor public ActivityDetectionConfig.Builder();
+    method public com.google.firebase.ai.type.ActivityDetectionConfig build();
+    method public com.google.firebase.ai.type.ActivityDetectionConfig.Builder setEndSensitivity(com.google.firebase.ai.type.ActivityDetectionConfig.Sensitivity sensitivity);
+    method public com.google.firebase.ai.type.ActivityDetectionConfig.Builder setPrefixPaddingMs(int paddingMs);
+    method public com.google.firebase.ai.type.ActivityDetectionConfig.Builder setSilenceDurationMs(int durationMs);
+    method public com.google.firebase.ai.type.ActivityDetectionConfig.Builder setStartSensitivity(com.google.firebase.ai.type.ActivityDetectionConfig.Sensitivity sensitivity);
+    field public com.google.firebase.ai.type.ActivityDetectionConfig.Sensitivity? endSensitivity;
+    field public Integer? prefixPaddingMs;
+    field public Integer? silenceDurationMs;
+    field public com.google.firebase.ai.type.ActivityDetectionConfig.Sensitivity? startSensitivity;
+  }
+
+  public static final class ActivityDetectionConfig.Companion {
+    method public com.google.firebase.ai.type.ActivityDetectionConfig.Builder builder();
+    method public com.google.firebase.ai.type.ActivityDetectionConfig disabled();
+  }
+
+  public static final class ActivityDetectionConfig.Sensitivity {
+    field public static final com.google.firebase.ai.type.ActivityDetectionConfig.Sensitivity.Companion Companion;
+    field public static final com.google.firebase.ai.type.ActivityDetectionConfig.Sensitivity HIGH;
+    field public static final com.google.firebase.ai.type.ActivityDetectionConfig.Sensitivity LOW;
+  }
+
+  public static final class ActivityDetectionConfig.Sensitivity.Companion {
+  }
+
+  public final class ActivityDetectionConfigKt {
+    method public static com.google.firebase.ai.type.ActivityDetectionConfig activityDetectionConfig(kotlin.jvm.functions.Function1<? super com.google.firebase.ai.type.ActivityDetectionConfig.Builder,kotlin.Unit> init);
+  }
+
   public final class AspectRatio {
     field public static final com.google.firebase.ai.type.AspectRatio.Companion Companion;
     field public static final com.google.firebase.ai.type.AspectRatio LANDSCAPE_16x9;
@@ -1452,6 +1491,7 @@ package com.google.firebase.ai.type {
     method public com.google.firebase.ai.type.LiveGenerationConfig.Builder setMaxOutputTokens(Integer? maxOutputTokens);
     method public com.google.firebase.ai.type.LiveGenerationConfig.Builder setOutputAudioTranscription(com.google.firebase.ai.type.AudioTranscriptionConfig? config);
     method public com.google.firebase.ai.type.LiveGenerationConfig.Builder setPresencePenalty(Float? presencePenalty);
+    method public com.google.firebase.ai.type.LiveGenerationConfig.Builder setRealtimeInputConfig(com.google.firebase.ai.type.RealtimeInputConfig config);
     method public com.google.firebase.ai.type.LiveGenerationConfig.Builder setResponseModality(com.google.firebase.ai.type.ResponseModality? responseModality);
     method public com.google.firebase.ai.type.LiveGenerationConfig.Builder setSpeechConfig(com.google.firebase.ai.type.SpeechConfig? speechConfig);
     method public com.google.firebase.ai.type.LiveGenerationConfig.Builder setTemperature(Float? temperature);
@@ -1463,6 +1503,7 @@ package com.google.firebase.ai.type {
     field public Integer? maxOutputTokens;
     field public com.google.firebase.ai.type.AudioTranscriptionConfig? outputAudioTranscription;
     field public Float? presencePenalty;
+    field public com.google.firebase.ai.type.RealtimeInputConfig? realtimeInputConfig;
     field public com.google.firebase.ai.type.ResponseModality? responseModality;
     field public com.google.firebase.ai.type.SpeechConfig? speechConfig;
     field public Float? temperature;
@@ -1536,6 +1577,8 @@ package com.google.firebase.ai.type {
     method public suspend Object? sendAudioRealtime(com.google.firebase.ai.type.InlineData audio, kotlin.coroutines.Continuation<? super kotlin.Unit>);
     method public suspend Object? sendFunctionResponse(java.util.List<com.google.firebase.ai.type.FunctionResponsePart> functionList, kotlin.coroutines.Continuation<? super kotlin.Unit>);
     method @Deprecated public suspend Object? sendMediaStream(java.util.List<com.google.firebase.ai.type.MediaData> mediaChunks, kotlin.coroutines.Continuation<? super kotlin.Unit>);
+    method public suspend Object? sendStartActivityRealtime(kotlin.coroutines.Continuation<? super kotlin.Unit>);
+    method public suspend Object? sendStopActivityRealtime(kotlin.coroutines.Continuation<? super kotlin.Unit>);
     method public suspend Object? sendTextRealtime(String text, kotlin.coroutines.Continuation<? super kotlin.Unit>);
     method public suspend Object? sendVideoRealtime(com.google.firebase.ai.type.InlineData video, kotlin.coroutines.Continuation<? super kotlin.Unit>);
     method @RequiresPermission(android.Manifest.permission.RECORD_AUDIO) public suspend Object? startAudioConversation(com.google.firebase.ai.type.LiveAudioConversationConfig liveAudioConversationConfig, kotlin.coroutines.Continuation<? super kotlin.Unit>);
@@ -1617,6 +1660,48 @@ package com.google.firebase.ai.type {
   public final class QuotaExceededException extends com.google.firebase.ai.type.FirebaseAIException {
   }
 
+  @com.google.firebase.ai.type.PublicPreviewAPI public final class RealtimeInputConfig {
+    method public static com.google.firebase.ai.type.RealtimeInputConfig.Builder builder();
+    field public static final com.google.firebase.ai.type.RealtimeInputConfig.Companion Companion;
+  }
+
+  public static final class RealtimeInputConfig.ActivityHandling {
+    field public static final com.google.firebase.ai.type.RealtimeInputConfig.ActivityHandling.Companion Companion;
+    field public static final com.google.firebase.ai.type.RealtimeInputConfig.ActivityHandling INTERRUPT;
+    field public static final com.google.firebase.ai.type.RealtimeInputConfig.ActivityHandling NO_INTERRUPT;
+  }
+
+  public static final class RealtimeInputConfig.ActivityHandling.Companion {
+  }
+
+  public static final class RealtimeInputConfig.Builder {
+    ctor public RealtimeInputConfig.Builder();
+    method public com.google.firebase.ai.type.RealtimeInputConfig build();
+    method public com.google.firebase.ai.type.RealtimeInputConfig.Builder setActivityHandling(com.google.firebase.ai.type.RealtimeInputConfig.ActivityHandling handling);
+    method public com.google.firebase.ai.type.RealtimeInputConfig.Builder setAutomaticActivityDetection(com.google.firebase.ai.type.ActivityDetectionConfig config);
+    method public com.google.firebase.ai.type.RealtimeInputConfig.Builder setTurnCoverage(com.google.firebase.ai.type.RealtimeInputConfig.TurnCoverage coverage);
+    field public com.google.firebase.ai.type.RealtimeInputConfig.ActivityHandling? activityHandling;
+    field public com.google.firebase.ai.type.ActivityDetectionConfig? automaticActivityDetection;
+    field public com.google.firebase.ai.type.RealtimeInputConfig.TurnCoverage? turnCoverage;
+  }
+
+  public static final class RealtimeInputConfig.Companion {
+    method public com.google.firebase.ai.type.RealtimeInputConfig.Builder builder();
+  }
+
+  public static final class RealtimeInputConfig.TurnCoverage {
+    field public static final com.google.firebase.ai.type.RealtimeInputConfig.TurnCoverage ALL_INPUT;
+    field public static final com.google.firebase.ai.type.RealtimeInputConfig.TurnCoverage.Companion Companion;
+    field public static final com.google.firebase.ai.type.RealtimeInputConfig.TurnCoverage ONLY_ACTIVITY;
+  }
+
+  public static final class RealtimeInputConfig.TurnCoverage.Companion {
+  }
+
+  public final class RealtimeInputConfigKt {
+    method public static com.google.firebase.ai.type.RealtimeInputConfig realtimeInputConfig(kotlin.jvm.functions.Function1<? super com.google.firebase.ai.type.RealtimeInputConfig.Builder,kotlin.Unit> init);
+  }
+
   public final class RequestOptions {
     ctor public RequestOptions();
     ctor public RequestOptions(long timeoutInMillis = 180.seconds.inWholeMilliseconds);

ai-logic/firebase-ai/src/androidTest/kotlin/com/google/firebase/ai/LiveSessionTests.kt

@@ -20,6 +20,7 @@ import android.content.Context
 import android.graphics.Bitmap
 import android.media.MediaMetadataRetriever
 import androidx.test.core.app.ApplicationProvider
+import com.google.firebase.ai.type.ActivityDetectionConfig
 import com.google.firebase.ai.type.AudioTranscriptionConfig
 import com.google.firebase.ai.type.FunctionDeclaration
 import com.google.firebase.ai.type.FunctionResponsePart
@@ -29,14 +30,17 @@ import com.google.firebase.ai.type.LiveServerToolCall
 import com.google.firebase.ai.type.LiveSession
 import com.google.firebase.ai.type.LiveSessionResumptionUpdate
 import com.google.firebase.ai.type.PublicPreviewAPI
+import com.google.firebase.ai.type.RealtimeInputConfig
 import com.google.firebase.ai.type.ResponseModality
 import com.google.firebase.ai.type.Schema
 import com.google.firebase.ai.type.SessionResumptionConfig
 import com.google.firebase.ai.type.SpeechConfig
 import com.google.firebase.ai.type.Tool
 import com.google.firebase.ai.type.Voice
+import com.google.firebase.ai.type.activityDetectionConfig
 import com.google.firebase.ai.type.content
 import com.google.firebase.ai.type.liveGenerationConfig
+import com.google.firebase.ai.type.realtimeInputConfig
 import io.kotest.matchers.ints.shouldBeGreaterThan
 import io.kotest.matchers.longs.shouldBeGreaterThan
 import io.kotest.matchers.nulls.shouldNotBeNull
@@ -323,4 +327,64 @@ class LiveSessionTests {
       .collect {}
     return transcriptBuilder.toString()
   }
+
+  @Test
+  fun testRealtimeInputConfig_manualActivity(): Unit = runBlocking {
+    val config = liveGenerationConfig {
+      responseModality = ResponseModality.AUDIO
+      outputAudioTranscription = AudioTranscriptionConfig()
+      realtimeInputConfig = realtimeInputConfig {
+        automaticActivityDetection = ActivityDetectionConfig.disabled()
+      }
+    }
+    val liveModel =
+      AIModels.getGoogleLiveModel(
+        modelName = modelName,
+        config = config,
+        systemInstruction = SystemInstructions.yesOrNo
+      )
+    val session = liveModel.connect()
+    try {
+      session.sendStartActivityRealtime()
+      session.sendTextRealtime("Does five plus five equal ten?")
+      session.sendStopActivityRealtime()
+
+      val text = withTimeoutOrNull(15.seconds) { session.collectNextAudioOutputTranscript() } ?: ""
+      text.toLowerCasePreservingASCIIRules() shouldContain "yes"
+    } finally {
+      session.close()
+    }
+  }
+
+  @Test
+  fun testRealtimeInputConfig_fullConfiguration(): Unit = runBlocking {
+    val config = liveGenerationConfig {
+      responseModality = ResponseModality.AUDIO
+      outputAudioTranscription = AudioTranscriptionConfig()
+      realtimeInputConfig = realtimeInputConfig {
+        activityHandling = RealtimeInputConfig.ActivityHandling.NO_INTERRUPT
+        turnCoverage = RealtimeInputConfig.TurnCoverage.ONLY_ACTIVITY
+        automaticActivityDetection = activityDetectionConfig {
+          startSensitivity = ActivityDetectionConfig.Sensitivity.HIGH
+          endSensitivity = ActivityDetectionConfig.Sensitivity.LOW
+          prefixPaddingMs = 100
+          silenceDurationMs = 500
+        }
+      }
+    }
+    val liveModel =
+      AIModels.getGoogleLiveModel(
+        modelName = modelName,
+        config = config,
+        systemInstruction = SystemInstructions.yesOrNo
+      )
+    val session = liveModel.connect()
+    try {
+      session.sendTextRealtime("Is the sky blue?")
+      val text = withTimeoutOrNull(15.seconds) { session.collectNextAudioOutputTranscript() } ?: ""
+      text.toLowerCasePreservingASCIIRules() shouldContain "yes"
+    } finally {
+      session.close()
+    }
+  }
 }

ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/LiveGenerativeModel.kt

@@ -125,7 +125,8 @@ internal constructor(
               config?.inputAudioTranscription?.toInternal(),
               config?.outputAudioTranscription?.toInternal(),
               resumption?.toInternal(),
-              config?.contextWindowCompression?.toInternal()
+              config?.contextWindowCompression?.toInternal(),
+              config?.realtimeInputConfig?.toInternal()
             )
             .toInternal()
         val data: String = JSON.encodeToString(clientMessage)

ai-logic/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/LiveSessionFutures.kt

@@ -220,6 +220,18 @@ public abstract class LiveSessionFutures internal constructor() {
    */
   public abstract fun sendTextRealtime(text: String): ListenableFuture<Unit>
 
+  /**
+   * Manually marks the start of user activity. Required only when automatic activity detection is
+   * disabled.
+   */
+  public abstract fun sendStartActivityRealtime(): ListenableFuture<Unit>
+
+  /**
+   * Manually marks the end of user activity. Required only when automatic activity detection is
+   * disabled.
+   */
+  public abstract fun sendStopActivityRealtime(): ListenableFuture<Unit>
+
   /**
    * Streams client data to the model.
    *
@@ -294,6 +306,12 @@ public abstract class LiveSessionFutures internal constructor() {
     override fun sendTextRealtime(text: String): ListenableFuture<Unit> =
       SuspendToFutureAdapter.launchFuture { session.sendTextRealtime(text) }
 
+    override fun sendStartActivityRealtime(): ListenableFuture<Unit> =
+      SuspendToFutureAdapter.launchFuture { session.sendStartActivityRealtime() }
+
+    override fun sendStopActivityRealtime(): ListenableFuture<Unit> =
+      SuspendToFutureAdapter.launchFuture { session.sendStopActivityRealtime() }
+
     override fun sendMediaStream(mediaChunks: List<MediaData>) =
       SuspendToFutureAdapter.launchFuture { session.sendMediaStream(mediaChunks) }