From a1fa415bb670cbb5f33939ee28b21b575e6db7ba Mon Sep 17 00:00:00 2001 From: Swayam Date: Tue, 11 Aug 2026 00:34:49 +0200 Subject: [PATCH 1/3] feat: add stateful Gemini Live copilot --- .gitignore | 9 + README.md | 86 ++- browser-extension/content.js | 311 ++++++++++ browser-extension/manifest.json | 20 + chat.html | 32 +- env.example | 45 +- index.html | 74 ++- llm-response.html | 122 ++-- main.js | 742 ++++++++++++++++++------ package-lock.json | 194 ++----- package.json | 29 +- preload.js | 19 +- settings.html | 49 +- setup.sh | 137 +---- src/core/config.js | 7 +- src/core/first-run.js | 57 +- src/managers/window.manager.js | 181 ++++-- src/services/antigravity.service.js | 197 +++++++ src/services/browser-bridge.service.js | 120 ++++ src/services/capture.service.js | 30 + src/services/disabled-speech.service.js | 45 ++ src/services/gemini-live.service.js | 418 +++++++++++++ src/services/llm.service.js | 561 ++++++++---------- src/services/system-audio.service.js | 255 ++++++++ src/ui/main-window.js | 133 +++-- src/ui/settings-window.js | 53 +- 26 files changed, 2883 insertions(+), 1043 deletions(-) create mode 100644 browser-extension/content.js create mode 100644 browser-extension/manifest.json create mode 100644 src/services/antigravity.service.js create mode 100644 src/services/browser-bridge.service.js create mode 100644 src/services/disabled-speech.service.js create mode 100644 src/services/gemini-live.service.js create mode 100644 src/services/system-audio.service.js diff --git a/.gitignore b/.gitignore index 57cbcfa0..a7a5e72f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ node_modules/ +.opencluely-cache/ .env .venv-whisper/ .whisper-models/ @@ -7,3 +8,11 @@ dist/ bin/ .DS_Store *.log + +# Local coaching materials (not part of the application release) +/MISSION.md +/RESOURCES.md +/learning-records/ +/lessons/ +/reference/ +/assets/course.css diff --git a/README.md b/README.md index a2ed045f..d77bcfa1 100644 --- a/README.md +++ b/README.md @@ -75,11 +75,17 @@ If you would rather build from source, three steps are all it takes. ./setup.sh ``` - The script installs Node dependencies, creates your `.env` from the example, sets up a local Whisper virtual environment, points the config at it, and launches the app. + The script installs Node dependencies, creates your `.env` from the example, and launches the app. -3. Add your Gemini key. +3. Configure Google Cloud authentication. - On first launch the Settings window opens automatically. Get a free key from [Google AI Studio](https://aistudio.google.com/) and paste it in, or edit `.env` directly. Both work, and changes are picked up without a restart. + Set `GOOGLE_CLOUD_PROJECT` in `.env`, then create local Application Default Credentials: + + ```bash + gcloud auth application-default login + ``` + + The selected project needs Vertex AI access and billing. Credentials remain in your normal local Google Cloud configuration and are not committed to this repository. ### Platform notes @@ -94,65 +100,50 @@ If you would rather build from source, three steps are all it takes. ./setup.sh --build # Build a distributable for your OS ./setup.sh --ci # Use npm ci instead of npm install ./setup.sh --no-run # Set up only, do not launch -./setup.sh --install-system-deps # Install sox for the microphone (optional) -./setup.sh --skip-whisper # Skip the local Whisper bootstrap +./setup.sh --install-system-deps # Install optional platform audio tools ``` ## Configuration -The setup script writes sensible defaults. The only required value is a Gemini API key. +The setup script writes sensible defaults. Gemini Live requires a Google Cloud project and local ADC authentication. ```bash -# Required -GEMINI_API_KEY=your_gemini_api_key_here - -# Optional speech provider. Pick one. -SPEECH_PROVIDER=whisper - -# Azure option -AZURE_SPEECH_KEY=your_azure_speech_key -AZURE_SPEECH_REGION=your_region - -# Local Whisper option -WHISPER_COMMAND=whisper -WHISPER_MODEL_DIR=.whisper-models -WHISPER_MODEL=small -WHISPER_LANGUAGE=auto -WHISPER_DEVICE=auto -WHISPER_PYTHON= -WHISPER_CAPTURE_MODE=vad -WHISPER_RESPONSE_TARGET=both -WHISPER_MANUAL_MAX_MS=90000 -WHISPER_GPU_IDLE_MS=60000 +# Required for Vertex AI and Gemini Live +GOOGLE_CLOUD_PROJECT=your-billed-gcp-project +GOOGLE_CLOUD_LOCATION=europe-west4 +GEMINI_LIVE_MODEL=gemini-live-2.5-flash-native-audio + +# macOS Call Copilot system audio +SYSTEM_AUDIO_COMMAND=bin/SystemAudioDump +FFMPEG_COMMAND=ffmpeg +CALL_COPILOT_SILENCE_MS=800 ``` -Speech is optional. If no provider is configured, the microphone button hides itself across the app. - -## Optional voice setup - -You can use local Whisper for offline transcription or Azure Speech for a cloud option. - -For local Whisper, `./setup.sh` handles the full setup. It creates `.venv-whisper`, installs `openai-whisper`, points `.env` at the virtual environment, creates `.whisper-models`, and runs a quick speech test. The app reads its own PCM WAV recordings directly; ffmpeg is only needed when transcribing other audio formats through the CLI fallback. - -For Azure Speech, create a Speech resource in the [Azure Portal](https://portal.azure.com/), then add the key and region to `.env` with `SPEECH_PROVIDER=azure`. +Live audio is optional. On macOS it requires the bundled `SystemAudioDump` helper, `ffmpeg`, Screen Recording permission, and an authenticated Gemini Live project. ## How it works -1. **Ask.** Use automatic pause detection, choose manual start/stop capture in Settings, or use the screenshot shortcut. -2. **Reason.** Gemini reads the audio or image with full conversation context and works toward a precise answer. -3. **Answer.** Voice responses stream to chat, the overlay, or both, according to Settings. +1. **Observe.** Start Live Screen, Live Audio, or use the one-shot screenshot shortcut. +2. **Reason.** Gemini maintains the active task while combining recent screen and conversation context. +3. **Assist.** Responses stream to the native overlay, chat window, or optional CoderPad browser rail. ## Keyboard shortcuts | Action | Shortcut | Description | |---|---|---| | Screenshot capture | `Cmd/Ctrl + Shift + S` | Capture the screen and analyze it with Gemini | -| Toggle speech | `Alt + R` | Start or stop voice recognition, if configured | +| Toggle speech | `Alt + R` | Legacy shortcut; live call assistance uses Call Copilot | | Toggle visibility | `Cmd/Ctrl + Shift + V` | Show or hide all windows | | Toggle interaction | `Cmd/Ctrl + Shift + I` or `Alt + A` | Enable or disable click through | | Open chat | `Cmd/Ctrl + Shift + C` | Open the interactive chat window | +| Toggle Call Copilot | `Cmd/Ctrl + Shift + L` | Listen to macOS system audio and suggest concise responses | | Settings | `Cmd/Ctrl + ,` | Open the settings panel | +Call Copilot captures the remote/system-audio channel on macOS and streams it +to Gemini Live for real-time transcription and assistance. It starts only when +explicitly toggled and requires Screen Recording permission. Use it only on +calls where recording and AI assistance are permitted. + ## Project status OpenCluely is under active development. The core is stable and improvements ship regularly. @@ -162,15 +153,14 @@ OpenCluely is under active development. The core is stable and improvements ship - Stealth overlay with a draggable command bar and a click through toggle - Hidden during screen share, with automatic hiding when a share begins - Screenshot capture with direct Gemini analysis, no OCR step -- Configurable manual or VAD-driven voice capture -- Persistent local Whisper worker with optional CUDA acceleration and idle GPU release -- Configurable chat/overlay routing for streamed voice answers -- Whisper hallucination filter that drops phantom phrases on silence +- Stateful Gemini Live audio and screen context +- Intent anchoring across short conversational detours +- Configurable chat/overlay routing for streamed answers - AI response window with markdown and syntax highlighting - Global shortcuts for capture, visibility, interaction, chat, and settings - Session memory and a full chat UI - Language picker and a DSA skill prompt -- Optional Azure Speech and local Whisper, with an auto hiding mic button +- Optional browser-native response rail for CoderPad - Multi-monitor and area capture support - Window binding and positioning - Settings management with disguise and stealth modes @@ -199,7 +189,7 @@ OpenCluely is under active development. The core is stable and improvements ship - **Electron will not start or shows a blank window on Linux.** Try `npm run dev`, and make sure X11 or XWayland is available in headless setups. - **macOS screen capture does not work.** Grant Screen Recording permission under System Settings, Privacy and Security, then relaunch the app. - **Windows SmartScreen blocks the app.** Click More info, then Run anyway, or use `npm start` during development. -- **Microphone or voice not working.** Voice is optional. For Azure, add valid keys to `.env`. For Whisper, install `openai-whisper`, `ffmpeg`, and `sox`, then set `SPEECH_PROVIDER=whisper`. +- **Gemini Live is not connecting.** Run `gcloud auth application-default login`, set `GOOGLE_CLOUD_PROJECT`, confirm Vertex AI access and billing, then restart OpenCluely. @@ -221,12 +211,12 @@ The app is built for learning and practice. You are responsible for following th ## License -Released under the MIT License. See [LICENSE](LICENSE) for details. +Released under the Apache License 2.0. See [LICENSE](LICENSE) for details. ## Acknowledgments - Google Gemini for the AI reasoning -- Azure Speech and OpenAI Whisper for optional voice input +- Google Cloud Vertex AI and Gemini Live for real-time assistance - Electron for the cross platform desktop runtime - [Vysper by varun-singhh](https://github.com/varun-singhh/Vysper) for UI and structure inspiration diff --git a/browser-extension/content.js b/browser-extension/content.js new file mode 100644 index 00000000..faacbcf3 --- /dev/null +++ b/browser-extension/content.js @@ -0,0 +1,311 @@ +(() => { + if (document.getElementById('opencluely-browser-rail-host')) return; + + const WS_URL = 'ws://127.0.0.1:17321/?token=oc_local_7f86c1d293b74b5eb8ae3fd4'; + const host = document.createElement('div'); + host.id = 'opencluely-browser-rail-host'; + host.style.cssText = 'all:initial;position:fixed;inset:0 0 auto auto;z-index:2147483647;pointer-events:none;'; + document.documentElement.appendChild(host); + const root = host.attachShadow({ mode: 'open' }); + + root.innerHTML = ` + + `; + + const rail = root.querySelector('.rail'); + const mark = root.querySelector('.mark'); + const state = root.querySelector('.state'); + const answer = root.querySelector('.answer'); + const intent = root.querySelector('.intent-text'); + const collapse = root.querySelector('.collapse'); + const liveScreen = root.querySelector('.live-screen'); + const liveAudio = root.querySelector('.live-audio'); + const input = root.querySelector('textarea'); + const send = root.querySelector('.send'); + + let socket; + let reconnectTimer; + let currentText = ''; + let requestSequence = 0; + + const setConnection = connected => { + mark.classList.toggle('connected', connected); + state.textContent = connected ? 'connected' : 'offline'; + }; + + const setAnswer = text => { + currentText = String(text || ''); + answer.textContent = currentText || 'Waiting for an answer…'; + answer.classList.toggle('empty', !currentText); + }; + + const deriveIntent = text => { + const clean = String(text || '').replace(/\*\*/g, ''); + const match = clean.match(/(?:^|\n)Intent\s*[—:-]\s*([^\n]+)/i); + if (match) intent.textContent = match[1].trim(); + }; + + const command = (name, payload = {}) => { + if (!socket || socket.readyState !== WebSocket.OPEN) return; + socket.send(JSON.stringify({ + type: 'command', + command: name, + payload, + requestId: String(++requestSequence) + })); + }; + + const handleEvent = ({ channel, data = {} }) => { + if (channel === 'transcription-llm-response-start') { + setAnswer(''); + rail.classList.remove('collapsed'); + collapse.textContent = '›'; + } else if (channel === 'transcription-llm-response-chunk') { + setAnswer(currentText + String(data.delta || '')); + deriveIntent(currentText); + answer.scrollTop = answer.scrollHeight; + } else if (channel === 'transcription-llm-response') { + setAnswer(data.response || data.content || currentText); + deriveIntent(currentText); + } else if (channel === 'llm-response') { + setAnswer(data.response || data.content || ''); + deriveIntent(currentText); + } else if (channel === 'live-screen-status') { + liveScreen.classList.toggle('active', !!data.active); + liveScreen.textContent = data.active ? 'Screen active' : 'Live Screen'; + } else if (channel === 'call-copilot-status') { + const active = !!(data.isCapturing || data.liveConnected); + liveAudio.classList.toggle('active', active); + liveAudio.textContent = active ? 'Audio active' : 'Live Audio'; + } else if (channel === 'llm-error' || channel === 'ocr-error') { + setAnswer(`OpenCluely error: ${data.error || 'Unknown error'}`); + } else if (channel === 'session-cleared') { + intent.textContent = 'Waiting for screen context…'; + setAnswer(''); + } + }; + + const connect = () => { + clearTimeout(reconnectTimer); + try { socket = new WebSocket(WS_URL); } catch (_) { return scheduleReconnect(); } + socket.addEventListener('open', () => { + setConnection(true); + command('get-status'); + }); + socket.addEventListener('message', event => { + try { + const message = JSON.parse(event.data); + if (message.type === 'event') handleEvent(message); + if (message.type === 'command-result' && message.result?.status) { + const statusResult = message.result.status; + handleEvent({ channel: 'live-screen-status', data: statusResult.liveScreen || {} }); + handleEvent({ channel: 'call-copilot-status', data: statusResult.callCopilot || {} }); + } + } catch (_) {} + }); + socket.addEventListener('close', () => { + setConnection(false); + scheduleReconnect(); + }); + socket.addEventListener('error', () => socket.close()); + }; + + const scheduleReconnect = () => { + clearTimeout(reconnectTimer); + reconnectTimer = setTimeout(connect, 1500); + }; + + const setCollapsed = collapsed => { + rail.classList.toggle('collapsed', collapsed); + collapse.textContent = collapsed ? '‹' : '›'; + collapse.setAttribute('aria-expanded', String(!collapsed)); + collapse.setAttribute('aria-label', collapsed ? 'Expand OpenCluely' : 'Collapse OpenCluely'); + collapse.title = collapsed ? 'Expand' : 'Collapse'; + }; + collapse.addEventListener('click', () => setCollapsed(!rail.classList.contains('collapsed'))); + liveScreen.addEventListener('click', () => command('toggle-live-screen')); + liveAudio.addEventListener('click', () => command('toggle-call-copilot')); + const sendMessage = () => { + const text = input.value.trim(); + if (!text) return; + command('send-chat-message', { text }); + input.value = ''; + }; + send.addEventListener('click', sendMessage); + input.addEventListener('keydown', event => { + if (event.key === 'Enter' && !event.shiftKey) { + event.preventDefault(); + sendMessage(); + } + }); + root.addEventListener('click', event => event.stopPropagation()); + root.addEventListener('keydown', event => event.stopPropagation()); + connect(); +})(); diff --git a/browser-extension/manifest.json b/browser-extension/manifest.json new file mode 100644 index 00000000..c253e059 --- /dev/null +++ b/browser-extension/manifest.json @@ -0,0 +1,20 @@ +{ + "manifest_version": 3, + "name": "OpenCluely Browser Rail", + "version": "1.0.1", + "description": "Shows local OpenCluely responses inside supported browser tabs.", + "permissions": [], + "host_permissions": [ + "http://127.0.0.1/*" + ], + "content_scripts": [ + { + "matches": [ + "https://coderpad.io/*", + "https://*.coderpad.io/*" + ], + "js": ["content.js"], + "run_at": "document_idle" + } + ] +} diff --git a/chat.html b/chat.html index b0f68c4f..4e52ad2a 100644 --- a/chat.html +++ b/chat.html @@ -14,8 +14,9 @@ margin: 0; padding: 0; overflow: hidden; - font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, - sans-serif; + font-family: "Avenir Next", "Helvetica Neue", sans-serif; + -webkit-font-smoothing: antialiased; + text-rendering: optimizeLegibility; } .chat-container { @@ -26,7 +27,7 @@ rgba(0, 0, 0, 0.3) 0%, rgba(20, 20, 20, 0.4) 100% ); - backdrop-filter: blur(25px); + backdrop-filter: none; border-radius: 12px; border: 1px solid rgba(255, 255, 255, 0.1); box-shadow: 0 4px 25px rgba(0, 0, 0, 0.15); @@ -42,7 +43,7 @@ justify-content: space-between; -webkit-app-region: drag; background: rgba(0, 0, 0, 0.2); - backdrop-filter: blur(10px); + backdrop-filter: none; cursor: move; flex-shrink: 0; } @@ -80,7 +81,7 @@ height: 8px; border-radius: 50%; background: #ff4757; - animation: pulse 2s infinite; + animation: none; display: none; box-shadow: 0 0 10px rgba(255, 71, 87, 0.5); } @@ -161,8 +162,15 @@ background: rgba(255, 152, 0, 0.1); } .message.assistant { - background: rgba(156, 39, 176, 0.1); - border-left: 3px solid #9c27b0; + background: linear-gradient(145deg, rgba(31, 44, 38, 0.92), rgba(18, 23, 21, 0.92)); + border: 1px solid rgba(142, 227, 179, 0.16); + border-left: 4px solid #8ee3b3; + border-radius: 10px; + padding: 16px 18px; + color: #f6f7f5; + font-size: 17px; + line-height: 1.55; + letter-spacing: -0.01em; /* Removed all height restrictions and overflow hidden */ display: block; } @@ -173,6 +181,8 @@ word-wrap: break-word; word-break: break-word; overflow-wrap: break-word; + font-size: 17px; + line-height: 1.55; } /* Ensure all content in assistant messages is fully visible */ @@ -323,12 +333,12 @@ color: rgba(255, 255, 255, 1); } - .message.assistant h1 { font-size: 16px; } - .message.assistant h2 { font-size: 15px; } - .message.assistant h3 { font-size: 14px; } + .message.assistant h1 { font-size: 24px; } + .message.assistant h2 { font-size: 21px; } + .message.assistant h3 { font-size: 19px; } .message.assistant h4, .message.assistant h5, - .message.assistant h6 { font-size: 13px; } + .message.assistant h6 { font-size: 17px; } /* Ensure paragraphs are fully visible */ .message.assistant p { diff --git a/env.example b/env.example index d770c1e7..590865c7 100644 --- a/env.example +++ b/env.example @@ -1,31 +1,18 @@ -# Google Gemini API Configuration -# Get your API key from: https://makersuite.google.com/app/apikey -GEMINI_API_KEY=your_gemini_api_key_here +# Text LLM provider: antigravity (Google OAuth via agy CLI) or gemini (Vertex AI) +LLM_PROVIDER=gemini +ANTIGRAVITY_COMMAND= +ANTIGRAVITY_MODEL=Gemini 3.1 Pro (Low) +ANTIGRAVITY_TIMEOUT_MS=120000 -# Speech Recognition Configuration -# Choose one provider: azure or whisper -SPEECH_PROVIDER=whisper +# Vertex AI and Gemini Live. Authenticate once with: +# gcloud auth application-default login +GOOGLE_CLOUD_PROJECT= +GOOGLE_CLOUD_LOCATION=europe-west4 +GEMINI_LIVE_MODEL=gemini-live-2.5-flash-native-audio -# Optional: Azure Speech Services Configuration -AZURE_SPEECH_KEY=your_azure_speech_key_here -AZURE_SPEECH_REGION=your_azure_region_here - -# Optional: Local OpenAI Whisper Configuration -# Requires a local Whisper CLI installation, for example: -# pip install openai-whisper -# brew install ffmpeg sox -# Use `whisper`, `python3 -m whisper`, or on Windows `.venv-whisper/Scripts/whisper.exe` -WHISPER_COMMAND=whisper -# Optional: where Whisper model weights are stored. Leave unset to use a stable -# app-data folder (recommended). Set an ABSOLUTE path to override; a relative -# path is ignored because it cannot be resolved reliably in packaged builds. -# WHISPER_MODEL_DIR= -WHISPER_MODEL=small -WHISPER_LANGUAGE=auto -WHISPER_SEGMENT_MS=4000 -WHISPER_DEVICE=auto -WHISPER_PYTHON= -WHISPER_CAPTURE_MODE=vad -WHISPER_RESPONSE_TARGET=both -WHISPER_MANUAL_MAX_MS=90000 -WHISPER_GPU_IDLE_MS=60000 +# macOS Call Copilot (system audio -> Gemini Live) +SYSTEM_AUDIO_COMMAND= +FFMPEG_COMMAND=ffmpeg +CALL_COPILOT_SILENCE_MS=800 +CALL_COPILOT_MAX_UTTERANCE_MS=20000 +CALL_COPILOT_VAD_FLOOR=0.006 diff --git a/index.html b/index.html index 31e37f5d..a8d28440 100644 --- a/index.html +++ b/index.html @@ -22,7 +22,7 @@ rgba(0, 0, 0, 0.4) 0%, rgba(20, 20, 20, 0.5) 100% ); - backdrop-filter: blur(20px); + backdrop-filter: none; border-radius: 8px; display: flex; align-items: center; @@ -73,7 +73,7 @@ .command-item.recording i { color: #ff4757; text-shadow: 0 0 10px rgba(255, 71, 87, 0.5); - animation: pulse 2s infinite; + animation: none; } .command-item.active { @@ -103,6 +103,32 @@ box-shadow: 0 0 0 1px rgba(96, 165, 250, 0.25) inset; } + #exitButton { + width: 26px; + height: 26px; + border-radius: 6px; + display: inline-flex; + align-items: center; + justify-content: center; + padding: 0; + color: rgba(248, 113, 113, 0.82); + } + + #exitButton i { + font-size: 13px; + } + + #exitButton:hover { + background: rgba(239, 68, 68, 0.18); + color: #fca5a5; + box-shadow: 0 0 0 1px rgba(248, 113, 113, 0.28) inset; + } + + #exitButton:focus-visible { + outline: 2px solid rgba(248, 113, 113, 0.72); + outline-offset: 2px; + } + .command-separator { width: 1px; height: 16px; @@ -127,7 +153,7 @@ .status-dot.interactive { background-color: #10b981; /* Green for active/interactive */ box-shadow: 0 0 10px rgba(16, 185, 129, 0.6); - animation: pulse-green 2s infinite; + animation: none; } .status-dot.non-interactive { @@ -263,7 +289,7 @@ rgba(20, 20, 20, 0.7) 0%, rgba(10, 10, 10, 0.6) 100% ); - backdrop-filter: blur(18px); + backdrop-filter: none; border: 1px solid rgba(255, 255, 255, 0.12); border-radius: 10px; color: rgba(255, 255, 255, 0.95); @@ -358,30 +384,34 @@ ⌘⇧S
-
- +
+ + ⌘⇧G
-
- - DSA +
+
-
- - +
+
+
+ +
+
@@ -433,6 +463,14 @@ Open chat + + + Ctrl/Cmd + + Shift + + L + + Toggle Call Copilot (system audio) + Alt diff --git a/llm-response.html b/llm-response.html index 3a4276f4..32d3e426 100644 --- a/llm-response.html +++ b/llm-response.html @@ -7,6 +7,15 @@