diff --git a/app/build.gradle.kts b/app/build.gradle.kts index d57c4303..0402910c 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -86,13 +86,13 @@ dependencies { implementation(project(":samples:genai-summarization")) implementation(project(":samples:genai-image-description")) implementation(project(":samples:genai-writing-assistance")) - implementation(project(":samples:imagen")) - implementation(project(":samples:imagen-editing")) + implementation(project(":samples:nanobanana")) implementation(project(":samples:magic-selfie")) implementation(project(":samples:gemini-video-summarization")) implementation(project(":samples:gemini-live-todo")) implementation(project(":samples:gemini-video-metadata-creation")) implementation(project(":samples:gemini-image-chat")) + implementation(project(":samples:gemini-hybrid")) testImplementation(libs.junit) androidTestImplementation(libs.androidx.junit) diff --git a/app/src/main/java/com/android/ai/catalog/domain/SampleCatalog.kt b/app/src/main/java/com/android/ai/catalog/domain/SampleCatalog.kt index c4b10287..51b15282 100644 --- a/app/src/main/java/com/android/ai/catalog/domain/SampleCatalog.kt +++ b/app/src/main/java/com/android/ai/catalog/domain/SampleCatalog.kt @@ -31,13 +31,25 @@ import com.android.ai.samples.geminivideosummary.ui.VideoSummarizationScreen import com.android.ai.samples.genai_image_description.GenAIImageDescriptionScreen import com.android.ai.samples.genai_summarization.GenAISummarizationScreen import com.android.ai.samples.genai_writing_assistance.GenAIWritingAssistanceScreen -import com.android.ai.samples.imagen.ui.ImagenScreen -import com.android.ai.samples.imagenediting.ui.ImagenEditingScreen +import com.android.ai.samples.geminihybrid.GeminiHybridScreen +import com.android.ai.samples.nanobanana.ui.NanobananaScreen import com.android.ai.samples.magicselfie.ui.MagicSelfieScreen import com.android.ai.theme.extendedColorScheme +import com.google.firebase.ai.type.PublicPreviewAPI +@OptIn(PublicPreviewAPI::class) @RequiresPermission(Manifest.permission.RECORD_AUDIO) val sampleCatalog = listOf( + SampleCatalogItem( + title = R.string.gemini_hybrid_sample_list_title, + description = R.string.gemini_hybrid_sample_list_description, + route = "GeminiHybridScreen", + sampleEntryScreen = { GeminiHybridScreen() }, + tags = listOf(SampleTags.GEMINI_NANO, SampleTags.GEMINI_FLASH, SampleTags.ML_KIT, SampleTags.FIREBASE), + needsFirebase = true, + keyArt = R.drawable.img_keyart_text, + isFeatured = true, + ), SampleCatalogItem( title = R.string.gemini_image_chat_list_title, description = R.string.gemini_image_chat_list_description, @@ -48,16 +60,6 @@ val sampleCatalog = listOf( needsFirebase = true, isFeatured = true, ), - SampleCatalogItem( - title = R.string.imagen_editing_sample_list_title, - description = R.string.imagen_editing_sample_list_description, - route = "ImagenMaskEditing", - sampleEntryScreen = { ImagenEditingScreen() }, - tags = listOf(SampleTags.IMAGEN, SampleTags.FIREBASE), - needsFirebase = true, - keyArt = R.drawable.img_keyart_imagen, - isFeatured = true, - ), SampleCatalogItem( title = R.string.gemini_multimodal_sample_list_title, description = R.string.gemini_multimodal_sample_list_description, @@ -102,11 +104,11 @@ val sampleCatalog = listOf( keyArt = R.drawable.img_keyart_text, ), SampleCatalogItem( - title = R.string.imagen_sample_list_title, - description = R.string.imagen_sample_list_description, - route = "ImagenImageGenerationScreen", - sampleEntryScreen = { ImagenScreen() }, - tags = listOf(SampleTags.IMAGEN, SampleTags.FIREBASE), + title = R.string.nanobanana_sample_list_title, + description = R.string.nanobanana_sample_list_description, + route = "NanobananaImageGenerationScreen", + sampleEntryScreen = { NanobananaScreen() }, + tags = listOf(SampleTags.GEMINI_FLASH, SampleTags.FIREBASE), needsFirebase = true, keyArt = R.drawable.img_keyart_imagen, ), @@ -115,7 +117,7 @@ val sampleCatalog = listOf( description = R.string.magic_selfie_sample_list_description, route = "MagicSelfieScreen", sampleEntryScreen = { MagicSelfieScreen() }, - tags = listOf(SampleTags.IMAGEN, SampleTags.FIREBASE, SampleTags.ML_KIT), + tags = listOf(SampleTags.GEMINI_FLASH, SampleTags.FIREBASE), needsFirebase = true, keyArt = R.drawable.img_keyart_magic_selfie, ), diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 553b4ed2..bc8f5a91 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,6 +1,7 @@ [versions] agp = "9.2.0" coilCompose = "3.1.0" +firebaseAiOndevice = "16.0.0-beta01" firebaseBom = "34.11.0" lifecycleRuntimeCompose = "2.9.1" mlkitGenAi = "1.0.0-beta1" @@ -25,10 +26,10 @@ mlkitSegmentation = "16.0.0-beta1" runtimeLivedata = "1.7.6" media3 = "1.8.0" firebaseCommonKtx = "21.0.0" -uiToolingPreviewAndroid = "1.11.0-rc01" +uiToolingPreviewAndroid = "1.8.1" spotless = "7.0.4" -uiToolingPreview = "1.11.0-rc01" -uiTooling = "1.11.0-rc01" +uiToolingPreview = "1.8.3" +uiTooling = "1.8.3" lifecycleViewmodelAndroid = "2.8.7" material3 = "1.5.0-alpha01" uiTextGoogleFonts = "1.8.1" @@ -38,13 +39,13 @@ richtext = "1.0.0-alpha02" glimmer = "1.0.0-alpha12" projected = "1.0.0-alpha07" - [libraries] androidx-core-ktx = { group = "androidx.core", name = "core-ktx", version.ref = "coreKtx" } androidx-lifecycle-runtime-compose = { module = "androidx.lifecycle:lifecycle-runtime-compose", version.ref = "lifecycleRuntimeCompose" } coil-compose = { module = "io.coil-kt.coil3:coil-compose", version.ref = "coilCompose" } +firebase-ai-ondevice = { module = "com.google.firebase:firebase-ai-ondevice", version.ref = "firebaseAiOndevice" } firebase-bom = { module = "com.google.firebase:firebase-bom", version.ref = "firebaseBom" } -firebase-ai = { group = "com.google.firebase", name = "firebase-ai" } +firebase-ai = { module = "com.google.firebase:firebase-ai"} firebase-common-ktx = { group = "com.google.firebase", name = "firebase-common-ktx", version.ref = "firebaseCommonKtx" } genai-image-description = { module = "com.google.mlkit:genai-image-description", version.ref = "mlkitGenAi" } genai-proofreading = { module = "com.google.mlkit:genai-proofreading", version.ref = "mlkitGenAi" } @@ -82,7 +83,6 @@ androidx-media3-ui = { module = "androidx.media3:media3-ui", version.ref = "medi androidx-media3-ui-compose = { module = "androidx.media3:media3-ui-compose", version.ref = "media3"} androidx-media3-transformer = { module = "androidx.media3:media3-transformer", version.ref = "media3" } androidx-ui-tooling-preview-android = { group = "androidx.compose.ui", name = "ui-tooling-preview-android", version.ref = "uiToolingPreviewAndroid" } -mlkit-segmentation = { module = "com.google.android.gms:play-services-mlkit-subject-segmentation", version.ref = "mlkitSegmentation" } ui-tooling-preview = { group = "androidx.compose.ui", name = "ui-tooling-preview", version.ref = "uiToolingPreview" } ui-tooling = { group = "androidx.compose.ui", name = "ui-tooling", version.ref = "uiTooling" } androidx-lifecycle-viewmodel-android = { group = "androidx.lifecycle", name = "lifecycle-viewmodel-android", version.ref = "lifecycleViewmodelAndroid" } @@ -99,4 +99,4 @@ google-gms-google-services = { id = "com.google.gms.google-services", version.re hilt-plugin = { id = "com.google.dagger.hilt.android", version.ref = "hilt"} ksp = { id = "com.google.devtools.ksp", version.ref = "ksp" } compose-compiler = { id = "org.jetbrains.kotlin.plugin.compose", version.ref = "kotlin" } -spotless = { id = "com.diffplug.spotless", version.ref = "spotless" } \ No newline at end of file +spotless = { id = "com.diffplug.spotless", version.ref = "spotless" } diff --git a/samples/gemini-live-todo/README.md b/samples/gemini-live-todo/README.md index 477c0fe0..ab1abecb 100644 --- a/samples/gemini-live-todo/README.md +++ b/samples/gemini-live-todo/README.md @@ -1,4 +1,4 @@ -# Gemini Live Todo AI Glasses Sample +# Gemini Live Todo Sample This sample is part of the [AI Sample Catalog](../../). To build and run this sample, you should clone the entire repository. @@ -6,10 +6,10 @@ This sample was built with Android Studio Quail 1 Canary 3. Get the latest [Andr ## Description -This sample demonstrates how to use the Gemini Live API for real-time, voice-based interactions in a simple ToDo application. Users can add, remove, and update tasks by speaking to the app, showcasing a hands-free, conversational user experience powered by the Gemini API. +This sample demonstrates how to use the Gemini Live API for real-time, voice-based interactions in a simple to-do application. Users can add, remove, and update tasks by speaking to the app, showcasing a hands-free, conversational user experience powered by the Gemini API.
-Gemini Live Todo in action +Gemini Live API to-do in action
## How it works @@ -20,7 +20,7 @@ Here is the key snippet of code that initializes the model and connects to a liv ```kotlin val generativeModel = Firebase.ai(backend = GenerativeBackend.vertexAI()).liveModel( - "gemini-2.0-flash-live-preview-04-09", + "gemini-2.5-flash-native-audio-preview-12-2025", generationConfig = liveGenerationConfig, systemInstruction = systemInstruction, tools = listOf( @@ -37,27 +37,5 @@ try { liveSessionState.value = LiveSessionState.Error } ``` -# Google AI Glasses Support -This prototype sample demonstrates how to extend the Gemini Live experience to Google AI Glasses. -The prototype illustrates how to leverage the glasses' form factor for a heads-up display (HUD) experience while maintaining the core application logic on the host device. - -
-AI Glasses List in action -
- -
-AI Glasses List in action scrolled -
- -## Tech Stack -The glasses integration is built using the following libraries: - -### Jetpack Projected: -Used to manage the connection and service lifecycle between the host application (phone) and the client display (glasses). This allows the application to "project" its content onto the glasses. - -### Jetpack Compose Glimmer: -The UI for the glasses is constructed using Glimmer, an Android UI toolkit optimized for transparent, wearable displays. - -Read more about the Gemini Live API in the Android Documentation. -Read more about the [Gemini Live API](https://developer.android.com/ai/gemini/live) in the Android Documentation. \ No newline at end of file +Read more about the [Gemini Live API](https://developer.android.com/ai/gemini/live) in the Android Documentation. diff --git a/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/GlassesActivity.kt b/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/GlassesActivity.kt index 534e1c4c..b6f6efcb 100644 --- a/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/GlassesActivity.kt +++ b/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/GlassesActivity.kt @@ -2,10 +2,13 @@ package com.android.ai.samples.geminilivetodo import android.Manifest import android.content.pm.PackageManager +import android.os.Build import android.os.Bundle +import android.util.Log import androidx.activity.ComponentActivity import androidx.activity.compose.setContent import androidx.activity.result.ActivityResultLauncher +import androidx.activity.viewModels import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.getValue @@ -15,19 +18,30 @@ import androidx.compose.ui.Modifier import androidx.compose.ui.res.stringResource import androidx.compose.ui.tooling.preview.Preview import androidx.core.content.ContextCompat +import androidx.lifecycle.lifecycleScope import androidx.xr.glimmer.GlimmerTheme +import com.android.ai.samples.geminilivetodo.ui.AudioExperience import com.android.ai.samples.geminilivetodo.ui.GlimmerTodoScreen +import com.android.ai.samples.geminilivetodo.ui.TodoScreenViewModel +import androidx.xr.projected.ProjectedDeviceController +import androidx.xr.projected.ProjectedDeviceController.Capability +import androidx.xr.projected.ProjectedDisplayController +import androidx.xr.projected.ProjectedDisplayController.PresentationMode import androidx.xr.projected.experimental.ExperimentalProjectedApi import androidx.xr.projected.permissions.ProjectedPermissionsRequestParams import androidx.xr.projected.permissions.ProjectedPermissionsResultContract import dagger.hilt.android.AndroidEntryPoint +import kotlinx.coroutines.launch private const val TAG = "GlassesActivity" @AndroidEntryPoint class GlassesActivity : ComponentActivity() { + private val viewModel: TodoScreenViewModel by viewModels() private var isPermissionsGranted by mutableStateOf(false) + private var isDisplayCapable by mutableStateOf(false) + private var areVisualsOn by mutableStateOf(false) private val requiredPermissions = listOf( Manifest.permission.RECORD_AUDIO @@ -43,11 +57,31 @@ class GlassesActivity : ComponentActivity() { setupContent() } + @OptIn(ExperimentalProjectedApi::class) override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) + + viewModel.initializeGeminiLive(this) + val allGranted = checkAllPermissionsGranted() isPermissionsGranted = allGranted + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) { + lifecycleScope.launch { + try { + val deviceController = ProjectedDeviceController.create(this@GlassesActivity) + isDisplayCapable = deviceController.capabilities.contains(Capability.CAPABILITY_VISUAL_UI) + + val displayController = ProjectedDisplayController.create(this@GlassesActivity) + displayController.addPresentationModeChangedListener { flags -> + areVisualsOn = flags.hasPresentationMode(PresentationMode.VISUALS_ON) + } + } catch (e: Exception) { + Log.e(TAG, "Error initializing projected display controllers", e) + } + } + } + setupContent() @@ -66,7 +100,12 @@ class GlassesActivity : ComponentActivity() { private fun setupContent() { setContent { GlimmerTheme { - RootScreen(isGranted = isPermissionsGranted) + RootScreen( + isGranted = isPermissionsGranted, + isDisplayCapable = isDisplayCapable, + areVisualsOn = areVisualsOn, + viewModel = viewModel + ) } } } @@ -86,14 +125,24 @@ class GlassesActivity : ComponentActivity() { @Composable -fun RootScreen(isGranted: Boolean, modifier: Modifier = Modifier) { - if (isGranted) { - GlimmerTodoScreen(modifier = modifier) - } else { +fun RootScreen( + isGranted: Boolean, + isDisplayCapable: Boolean, + areVisualsOn: Boolean, + viewModel: TodoScreenViewModel, + modifier: Modifier = Modifier +) { + if (!isGranted) { Text( text = stringResource(R.string.permissions_denied_mic_access), modifier = modifier ) + } else if (isDisplayCapable && areVisualsOn) { + // VISUAL MODE: Render UI for display-active glasses + GlimmerTodoScreen(viewModel = viewModel, modifier = modifier) + } else { + // AUDIO MODE: Fall back to Gemini for audio-only or display-off states + AudioExperience(viewModel = viewModel) } } @@ -102,6 +151,11 @@ fun RootScreen(isGranted: Boolean, modifier: Modifier = Modifier) { @Composable fun PreviewRootScreen() { GlimmerTheme { - RootScreen(isGranted = false) + RootScreen( + isGranted = false, + isDisplayCapable = true, + areVisualsOn = true, + viewModel = TodoScreenViewModel(com.android.ai.samples.geminilivetodo.data.TodoRepository()) + ) } } \ No newline at end of file diff --git a/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/data/Todo.kt b/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/data/Todo.kt index 4805b658..c2971496 100644 --- a/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/data/Todo.kt +++ b/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/data/Todo.kt @@ -18,6 +18,7 @@ package com.android.ai.samples.geminilivetodo.data import java.util.UUID.randomUUID const val MIC_TODO_ID = 111 +const val MIC_STATUS_TODO_ID = -999 sealed interface GlassesListItem { val id: Int diff --git a/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/data/TodoRepository.kt b/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/data/TodoRepository.kt index a6617903..0a59ab32 100644 --- a/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/data/TodoRepository.kt +++ b/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/data/TodoRepository.kt @@ -15,6 +15,7 @@ */ package com.android.ai.samples.geminilivetodo.data +import android.util.Log import javax.inject.Inject import javax.inject.Singleton import kotlin.collections.filterNot @@ -26,10 +27,10 @@ import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.update -private const val MIC_STATUS_TODO_ID = -999 @Singleton class TodoRepository @Inject constructor() { + private val TAG = "TodoRepository" private val _todos = MutableStateFlow>( listOf( @@ -48,13 +49,15 @@ class TodoRepository @Inject constructor() { fun getTodoList(): List = _todos.value - fun addTodo(taskDescription: String) { + fun addTodo(taskDescription: String): Int? { if (taskDescription.isNotBlank()) { val newTodo = Todo(task = taskDescription, isCompleted = false) _todos.update { currentList -> listOf(newTodo) + currentList } + return newTodo.id } + return null } @@ -87,6 +90,7 @@ class TodoRepository @Inject constructor() { } fun toggleTodoStatus(todoId: Int) { + Log.d(TAG, "Toggling task status for ID: $todoId") _todos.update { currentList -> currentList.map { item -> when (item) { diff --git a/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/ui/AudioExperience.kt b/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/ui/AudioExperience.kt new file mode 100644 index 00000000..7a9b020b --- /dev/null +++ b/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/ui/AudioExperience.kt @@ -0,0 +1,69 @@ +/* + * Copyright 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.ai.samples.geminilivetodo.ui + +import android.app.Activity +import android.util.Log +import androidx.activity.compose.LocalActivity +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.ui.Modifier +import androidx.compose.ui.focus.focusTarget +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import androidx.xr.glimmer.onIndirectPointerGesture +import com.google.firebase.ai.type.PublicPreviewAPI +import com.google.firebase.ai.type.content +import kotlinx.coroutines.launch + +private const val TAG = "AudioExperience" + +@OptIn(PublicPreviewAPI::class) +@Composable +fun AudioExperience(viewModel: TodoScreenViewModel) { + val activity = LocalActivity.current as Activity + val uiState by viewModel.uiState.collectAsStateWithLifecycle() + val isMicOn = (uiState as? TodoScreenUiState.Success)?.isMicOn ?: false + val scope = rememberCoroutineScope() + + Box( + modifier = Modifier + .fillMaxSize() + .onIndirectPointerGesture( + enabled = true, + onSwipeForward = { /* onSwipeForward */ }, + onClick = { + val wasOn = isMicOn + Log.d(TAG, "Screen clicked - currently isMicOn: $wasOn") + if (wasOn) { + // Turn OFF + viewModel.toggleLiveSession(activity) + } else { + viewModel.toggleLiveSession(activity) + scope.launch { + // Give Gemini Live a moment to open the channel + kotlinx.coroutines.delay(500) + viewModel.session?.send(content { text("Turning Microphone On") }) + } + } + }, + ) + .focusTarget() + ) +} diff --git a/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/ui/GlimmerToDoScreen.kt b/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/ui/GlimmerToDoScreen.kt index f8509916..648678da 100644 --- a/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/ui/GlimmerToDoScreen.kt +++ b/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/ui/GlimmerToDoScreen.kt @@ -18,14 +18,15 @@ package com.android.ai.samples.geminilivetodo.ui import android.app.Activity import android.util.Log -import androidx.compose.foundation.Image import androidx.compose.foundation.background import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.PaddingValues import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.offset import androidx.compose.foundation.layout.size +import androidx.compose.material3.Icon import androidx.compose.runtime.Composable import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.getValue @@ -43,6 +44,7 @@ import androidx.lifecycle.compose.collectAsStateWithLifecycle import androidx.xr.glimmer.GlimmerTheme import androidx.xr.glimmer.ListItem import androidx.xr.glimmer.Text +import androidx.xr.glimmer.TitleChip import androidx.xr.glimmer.list.VerticalList import com.android.ai.samples.geminilivetodo.R import com.android.ai.samples.geminilivetodo.data.Todo @@ -51,9 +53,8 @@ import kotlin.math.min private val DefaultListItemHeight = 64.dp private const val MaxItemsInList = 4 -private val IconSize = 30.dp +private val IconSize = 24.dp private const val TAG = "GlimmerTodoScreen" -private const val MIC_CONTROL_ID = 111 @Composable fun GlimmerTodoScreen( @@ -75,7 +76,7 @@ fun GlimmerTodoScreen( GlimmerTheme { Box( - contentAlignment = Alignment.BottomCenter, + contentAlignment = Alignment.Center, modifier = modifier .fillMaxSize() .background(GlimmerTheme.colors.background) @@ -83,7 +84,8 @@ fun GlimmerTodoScreen( GlimmerScreenContent( uiState = uiState, - onToggleItem = viewModel::toggleTodoStatus, + viewModel = viewModel, + activity = activity, onExit = { onExit() } ) } @@ -93,7 +95,8 @@ fun GlimmerTodoScreen( @Composable private fun GlimmerScreenContent( uiState: TodoScreenUiState, - onToggleItem: (Int) -> Unit, + viewModel: TodoScreenViewModel, + activity: Activity?, onExit: () -> Unit ) { when (uiState) { @@ -104,7 +107,8 @@ private fun GlimmerScreenContent( TodoListView( todoItems = uiState.todoItems, isMicOn = uiState.isMicOn, - onToggleItem = onToggleItem, + viewModel = viewModel, + activity = activity, onExit = onExit ) } @@ -118,16 +122,17 @@ private fun GlimmerScreenContent( private fun TodoListView( todoItems: List, isMicOn: Boolean, - onToggleItem: (Int) -> Unit, + viewModel: TodoScreenViewModel, + activity: Activity?, onExit: () -> Unit ) { - - val totalItems = todoItems.size + 2 + val totalItems = todoItems.size + 2 // Mic + Exit val listHeight = (min(totalItems, MaxItemsInList) * DefaultListItemHeight.value + min(totalItems - 1, MaxItemsInList) * 12f) VerticalList( + title = { TitleChip { Text("To-Do List") } }, modifier = Modifier.height(listHeight.dp), contentPadding = PaddingValues(horizontal = 16.dp, vertical = 24.dp), verticalArrangement = Arrangement.spacedBy(12.dp), @@ -136,14 +141,14 @@ private fun TodoListView( item { GlimmerMicControlItem( isMicOn = isMicOn, - onToggle = { onToggleItem(MIC_CONTROL_ID) } + onToggle = { activity?.let { viewModel.toggleLiveSession(it) } } ) } items(todoItems.size, key = { index -> todoItems[index].id }) { index -> GlimmerTodoItem( task = todoItems[index], - onToggle = onToggleItem + onToggle = { viewModel.toggleTodoStatus(it) } ) } @@ -151,10 +156,13 @@ private fun TodoListView( ListItem( onClick = onExit, leadingIcon = { - Image( + Icon( painter = painterResource(id = UiComponentR.drawable.ic_close), contentDescription = stringResource(R.string.exit_app), - modifier = Modifier.size(IconSize) + modifier = Modifier + .size(IconSize) + .offset(y = 12.dp), + tint = Color.White ) } ) { @@ -187,10 +195,13 @@ private fun GlimmerMicControlItem( ListItem( onClick = onToggle, leadingIcon = { - Image( + Icon( painter = painterResource(id = icon), contentDescription = contentDesc, - modifier = Modifier.size(IconSize) + modifier = Modifier + .size(IconSize) + .offset(y = 12.dp), + tint = Color.White ) } ) { @@ -208,13 +219,16 @@ private fun GlimmerTodoItem( ListItem( onClick = { onToggle(task.id) }, leadingIcon = { - Image( + Icon( painter = painterResource(id = icon), contentDescription = if (task.isCompleted) stringResource(R.string.status_completed) else stringResource(R.string.status_pending), - modifier = Modifier.size(IconSize) + modifier = Modifier + .size(IconSize) + .offset(y = 12.dp), + tint = Color.White ) } ) { @@ -247,9 +261,10 @@ private fun GlimmerTodoScreenPreview() { isMicOn = true, liveSessionState = LiveSessionState.Running ), - onToggleItem = { _ -> }, + viewModel = TodoScreenViewModel(com.android.ai.samples.geminilivetodo.data.TodoRepository()), + activity = null, onExit = {} ) } } -} \ No newline at end of file +} diff --git a/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/ui/TodoScreen.kt b/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/ui/TodoScreen.kt index 37dbb8dc..69f74775 100644 --- a/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/ui/TodoScreen.kt +++ b/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/ui/TodoScreen.kt @@ -154,7 +154,6 @@ fun TodoScreen(viewModel: TodoScreenViewModel = hiltViewModel()) { .imePadding() .fillMaxSize(), ) { - when (val state = uiState) { is TodoScreenUiState.Initial -> { Box( diff --git a/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/ui/TodoScreenViewModel.kt b/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/ui/TodoScreenViewModel.kt index 7a80ede1..fe111b10 100644 --- a/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/ui/TodoScreenViewModel.kt +++ b/samples/gemini-live-todo/src/main/java/com/android/ai/samples/geminilivetodo/ui/TodoScreenViewModel.kt @@ -17,14 +17,15 @@ package com.android.ai.samples.geminilivetodo.ui import android.Manifest +import android.annotation.SuppressLint import android.app.Activity import android.content.pm.PackageManager import android.util.Log -import androidx.annotation.RequiresPermission import androidx.core.app.ActivityCompat import androidx.core.content.ContextCompat import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope +import com.android.ai.samples.geminilivetodo.data.MIC_STATUS_TODO_ID import com.android.ai.samples.geminilivetodo.data.MicControl import com.android.ai.samples.geminilivetodo.data.Todo import com.android.ai.samples.geminilivetodo.data.TodoRepository @@ -46,7 +47,6 @@ import com.google.firebase.ai.type.liveGenerationConfig import dagger.hilt.android.lifecycle.HiltViewModel import java.lang.ref.WeakReference import javax.inject.Inject -import kotlinx.coroutines.CancellationException import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.StateFlow @@ -56,25 +56,20 @@ import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.JsonPrimitive -import kotlinx.serialization.json.jsonPrimitive import kotlinx.serialization.json.int - -private const val MIC_TODO_ID = 111 -private const val MIC_STATUS_TODO_ID = -999 +import kotlinx.serialization.json.jsonPrimitive @OptIn(PublicPreviewAPI::class) @HiltViewModel class TodoScreenViewModel @Inject constructor(private val todoRepository: TodoRepository) : ViewModel() { private val TAG = "TodoScreenViewModel" - private var session: LiveSession? = null + internal var session: LiveSession? = null private var hostActivityRef: WeakReference? = null private val liveSessionState = MutableStateFlow(LiveSessionState.NotReady) private val todos = todoRepository.todos val uiState: StateFlow = combine(liveSessionState, todos) { liveSessionState, currentTodos -> - - val micItem = currentTodos.filterIsInstance().firstOrNull() val isMicOn = micItem?.isMicOn ?: false @@ -94,76 +89,50 @@ class TodoScreenViewModel @Inject constructor(private val todoRepository: TodoRe initialValue = TodoScreenUiState.Initial, ) - fun addTodo(taskDescription: String) { - todoRepository.addTodo(taskDescription) + fun addTodo(taskDescription: String): Int? { + return todoRepository.addTodo(taskDescription) } fun removeTodo(todoId: Int) { - if (todoId == MIC_TODO_ID || todoId == MIC_STATUS_TODO_ID) return todoRepository.removeTodo(todoId) } fun toggleTodoStatus(todoId: Int) { - if (todoId == MIC_TODO_ID) { - todoRepository.toggleTodoStatus(MIC_TODO_ID) - return - } - if (todoId == MIC_STATUS_TODO_ID) return todoRepository.toggleTodoStatus(todoId) } - @RequiresPermission(Manifest.permission.RECORD_AUDIO) - private fun startLiveSession() { - val activity = hostActivityRef?.get() ?: run { - Log.e(TAG, "Cannot start Live Session: Host Activity reference lost.") - todoRepository.updateMicStatus(micIsOn = false) - return - } - + @SuppressLint("MissingPermission") + fun toggleLiveSession(activity: Activity) { viewModelScope.launch { if (liveSessionState.value is LiveSessionState.NotReady) return@launch session?.let { currentSession -> - if (ContextCompat.checkSelfPermission( - activity, - Manifest.permission.RECORD_AUDIO, - ) == PackageManager.PERMISSION_GRANTED - ) { - try { - liveSessionState.update { LiveSessionState.Running } - Log.i(TAG, "API Sync: Live Session Started.") - currentSession.startAudioConversation(::handleFunctionCall) - } - - catch (e: CancellationException) { - throw e - } - catch (e: Exception) { - Log.e(TAG, "Error starting Live Session: ${e.message}", e) - todoRepository.updateMicStatus(micIsOn = false) - liveSessionState.update { LiveSessionState.Ready } + if (liveSessionState.value is LiveSessionState.Ready) { + if (ContextCompat.checkSelfPermission( + activity, + Manifest.permission.RECORD_AUDIO, + ) == PackageManager.PERMISSION_GRANTED + ) { + try { + Log.i(TAG, "API Sync: Live Session Starting...") + currentSession.startAudioConversation(::handleFunctionCall) + liveSessionState.update { LiveSessionState.Running } + todoRepository.updateMicStatus(micIsOn = true) + } catch (e: Exception) { + Log.e(TAG, "Error starting Live Session: ${e.message}", e) + liveSessionState.update { LiveSessionState.Ready } + todoRepository.updateMicStatus(micIsOn = false) + } + } else { + requestAudioPermissionIfNeeded(activity) } } else { - requestAudioPermissionIfNeeded(activity) - todoRepository.updateMicStatus(micIsOn = false) - } - } - } - } - - private fun stopLiveSession() { - viewModelScope.launch { - session?.let { currentSession -> - if (liveSessionState.value is LiveSessionState.Running) { try { currentSession.stopAudioConversation() liveSessionState.update { LiveSessionState.Ready } + todoRepository.updateMicStatus(micIsOn = false) Log.i(TAG, "API Sync: Live Session Stopped.") - } - catch (e: CancellationException) { - throw e - } - catch (e: Exception) { + } catch (e: Exception) { Log.e(TAG, "Error stopping Live Session: ${e.message}", e) liveSessionState.update { LiveSessionState.Ready } } @@ -172,31 +141,14 @@ class TodoScreenViewModel @Inject constructor(private val todoRepository: TodoRe } } - fun toggleLiveSession(activity: Activity) { - todoRepository.toggleTodoStatus(MIC_TODO_ID) - } - fun initializeGeminiLive(activity: Activity) { + if (liveSessionState.value !is LiveSessionState.NotReady) { + Log.d(TAG, "Gemini Live already initialized or initializing") + return + } hostActivityRef = WeakReference(activity) requestAudioPermissionIfNeeded(activity) - viewModelScope.launch { - todoRepository.todos.collect @androidx.annotation.RequiresPermission(android.Manifest.permission.RECORD_AUDIO) { todos -> - val isMicOnInUI = todos.find { it.id == MIC_TODO_ID } - ?.let { it as? MicControl }?.isMicOn ?: false - - val currentLiveStatus = liveSessionState.value is LiveSessionState.Running - - if (isMicOnInUI != currentLiveStatus) { - if (isMicOnInUI) { - startLiveSession() - } else { - stopLiveSession() - } - } - } - } - viewModelScope.launch { Log.d(TAG, "Start Gemini Live initialization") @@ -220,6 +172,8 @@ class TodoScreenViewModel @Inject constructor(private val todoRepository: TodoRe **Never share the id with the user:** you don't need to share the id with the user. It is just here to help you perform the check/uncheck and remove operations to the list. + **Voice Feedback Commands:** If the user sends a message like 'Turning Microphone On', you MUST acknowledge it immediately by saying that exact phrase clearly. + **If Unsure:** If you can't determine the update from the request, politely ask the user to rephrase or try something else. """.trimIndent(), ) @@ -239,8 +193,8 @@ class TodoScreenViewModel @Inject constructor(private val todoRepository: TodoRe val toggleTodoStatus = FunctionDeclaration( "toggleTodoStatus", - "Change the status of the task", - mapOf("todoId" to Schema.integer("The id of the task to remove from the todo list")), + "Toggle the completion status of a task (e.g. check off or uncheck)", + mapOf("todoId" to Schema.integer("The id of the task to mark as completed or incomplete")), ) val getTodoList = FunctionDeclaration( @@ -249,8 +203,9 @@ class TodoScreenViewModel @Inject constructor(private val todoRepository: TodoRe emptyMap(), ) - val generativeModel = Firebase.ai(backend = GenerativeBackend.vertexAI()).liveModel( - "gemini-live-2.5-flash-preview-native-audio-09-2025", + // See https://firebase.google.com/docs/ai-logic/live-api for an overview of available models + val generativeModel = Firebase.ai(backend = GenerativeBackend.googleAI()).liveModel( + "gemini-2.5-flash-native-audio-preview-12-2025", generationConfig = liveGenerationConfig, systemInstruction = systemInstruction, tools = listOf( @@ -265,19 +220,10 @@ class TodoScreenViewModel @Inject constructor(private val todoRepository: TodoRe try { session = generativeModel.connect() liveSessionState.update { LiveSessionState.Ready } - - todoRepository.updateMicStatus(micIsOn = false) Log.i(TAG, "MIC STATE UPDATE: Session connected (LiveSessionState.Ready).") - } - // Change: Rethrow CancellationException so the coroutine cancels properly - catch (e: CancellationException) { - throw e - } - catch (e: Exception) { + } catch (e: Exception) { Log.e(TAG, "Error connecting to the model", e) liveSessionState.update { LiveSessionState.Error } - todoRepository.updateMicStatus(micIsOn = false) - Log.i(TAG, "MIC STATE UPDATE: Connection Error (LiveSessionState.Error).") } } } @@ -285,47 +231,83 @@ class TodoScreenViewModel @Inject constructor(private val todoRepository: TodoRe private fun handleFunctionCall(functionCall: FunctionCallPart): FunctionResponsePart { return when (functionCall.name) { "getTodoList" -> { - val todoList = todoRepository.getTodoList().filterNot { it.id == MIC_STATUS_TODO_ID }.reversed() + Log.d(TAG, "Tool Call: getTodoList") + val todoList = todoRepository.getTodoList().filterIsInstance() + + // Return structured JSON for reliable tool use + val jsonList = todoList.reversed().map { item -> + JsonObject( + mapOf( + "id" to JsonPrimitive(item.id), + "task" to JsonPrimitive(item.task), + "isCompleted" to JsonPrimitive(item.isCompleted), + ), + ) + } val response = JsonObject( mapOf( "success" to JsonPrimitive(true), - "message" to JsonPrimitive("List of tasks in the todo list: $todoList"), + "tasks" to kotlinx.serialization.json.JsonArray(jsonList), ), ) FunctionResponsePart(functionCall.name, response, functionCall.id) } "addTodo" -> { val taskDescription = functionCall.args["taskDescription"]!!.jsonPrimitive.content - todoRepository.addTodo(taskDescription) - val response = JsonObject( - mapOf( - "success" to JsonPrimitive(true), - "message" to JsonPrimitive("Task $taskDescription added to the todo list"), - ), - ) - FunctionResponsePart(functionCall.name, response, functionCall.id) + val id = todoRepository.addTodo(taskDescription) + if (id != null) { + val response = JsonObject( + mapOf( + "success" to JsonPrimitive(true), + "message" to JsonPrimitive("Task $taskDescription added to the todo list (id: $id)"), + ), + ) + FunctionResponsePart(functionCall.name, response, functionCall.id) + } else { + val response = JsonObject( + mapOf( + "success" to JsonPrimitive(false), + "message" to JsonPrimitive("Task $taskDescription wasn't properly added to the list"), + ), + ) + FunctionResponsePart(functionCall.name, response, functionCall.id) + } } "removeTodo" -> { - val taskId = functionCall.args["todoId"]!!.jsonPrimitive.int - todoRepository.removeTodo(taskId) - val response = JsonObject( - mapOf( - "success" to JsonPrimitive(true), - "message" to JsonPrimitive("Task was removed from the todo list"), - ), - ) - FunctionResponsePart(functionCall.name, response, functionCall.id) + try { + val taskId = functionCall.args["todoId"]!!.jsonPrimitive.int + todoRepository.removeTodo(taskId) + val response = JsonObject( + mapOf( + "success" to JsonPrimitive(true), + "message" to JsonPrimitive("Task removed successfully."), + ), + ) + FunctionResponsePart(functionCall.name, response, functionCall.id) + } catch (e: Exception) { + val response = JsonObject( + mapOf("success" to JsonPrimitive(false), "error" to JsonPrimitive(e.message)) + ) + FunctionResponsePart(functionCall.name, response, functionCall.id) + } } "toggleTodoStatus" -> { - val taskId = functionCall.args["todoId"]!!.jsonPrimitive.int - todoRepository.toggleTodoStatus(taskId) - val response = JsonObject( - mapOf( - "success" to JsonPrimitive(true), - "message" to JsonPrimitive("Task was toggled in the todo list"), - ), - ) - FunctionResponsePart(functionCall.name, response, functionCall.id) + try { + val taskId = functionCall.args["todoId"]!!.jsonPrimitive.int + todoRepository.toggleTodoStatus(taskId) + val response = JsonObject( + mapOf( + "success" to JsonPrimitive(true), + "message" to JsonPrimitive("Task status toggled successfully."), + ), + ) + FunctionResponsePart(functionCall.name, response, functionCall.id) + } catch (e: Exception) { + val response = JsonObject( + mapOf("success" to JsonPrimitive(false), "error" to JsonPrimitive(e.message)) + ) + FunctionResponsePart(functionCall.name, response, functionCall.id) + } } else -> { val response = JsonObject( @@ -345,4 +327,4 @@ class TodoScreenViewModel @Inject constructor(private val todoRepository: TodoRe ActivityCompat.requestPermissions(activity, arrayOf(Manifest.permission.RECORD_AUDIO), 1) } } -} \ No newline at end of file +} diff --git a/samples/gemini-live-todo/src/main/res/values/strings.xml b/samples/gemini-live-todo/src/main/res/values/strings.xml index edbd7dfd..5211dcfa 100644 --- a/samples/gemini-live-todo/src/main/res/values/strings.xml +++ b/samples/gemini-live-todo/src/main/res/values/strings.xml @@ -14,9 +14,10 @@ ~ limitations under the License. ~ --> + - Gemini Live API Todo - Simple ToDo app using the Gemini Live API to interact with the items in the list. + Gemini Live API to-do + Simple to-do app using the Gemini Live API to interact with the items in the list. New Task Add Error @@ -33,7 +34,6 @@ Exit app We need microphone access to continue to the main experience. Permissions Denied. Please grant Audio access on the host phone to proceed. - Loading Todo List... Error loading list. Please try again. Completed diff --git a/settings.gradle.kts b/settings.gradle.kts index 977e9376..14bffd8c 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -43,11 +43,11 @@ include(":samples:gemini-chatbot") include(":samples:genai-summarization") include(":samples:genai-writing-assistance") include(":samples:genai-image-description") -include(":samples:imagen") -include(":samples:imagen-editing") include(":samples:magic-selfie") include(":samples:gemini-video-summarization") include(":samples:gemini-live-todo") include(":samples:gemini-video-metadata-creation") include(":samples:gemini-image-chat") +include(":samples:gemini-hybrid") +include(":samples:nanobanana") include(":ui-component")