diff --git a/chat.html b/chat.html index 47edcaff..6785873b 100644 --- a/chat.html +++ b/chat.html @@ -732,6 +732,11 @@ let listeningTimer = null; let speechAvailable = false; + // Renderer-based audio capture (getUserMedia in chat window) + let rendererAudioStream = null; + let rendererAudioContext = null; + let rendererAudioProcessor = null; + // Chat persistence and deduplication const CHAT_HISTORY_KEY = 'opencluely_chat_history_v1'; let chatHistory = []; @@ -798,12 +803,15 @@ } async function initSpeechAvailability() { + console.log('[Chat] initSpeechAvailability() called'); try { if (window.electronAPI && window.electronAPI.getSpeechAvailability) { speechAvailable = await window.electronAPI.getSpeechAvailability(); + console.log('[Chat] Speech availability:', speechAvailable); applyMicVisibility(); } } catch (e) { + console.error('[Chat] Failed to get speech availability:', e); speechAvailable = false; applyMicVisibility(); } @@ -817,12 +825,58 @@ }); } + // TTS: speak assistant responses aloud + let ttsEnabled = true; + + // Load TTS enabled state from settings + if (whysperAPI && typeof whysperAPI.getSettings === 'function') { + whysperAPI.getSettings().then(settings => { + if (settings && settings.ttsEnabled !== undefined) { + ttsEnabled = settings.ttsEnabled; + } + }).catch(() => {}); + } + + function maybeSpeakResponse(text) { + if (!ttsEnabled || !text || !whysperAPI || typeof whysperAPI.synthesizeSpeech !== 'function') return; + whysperAPI.synthesizeSpeech(text).catch(() => {}); + } + + // Listen for TTS events to update UI + if (whysperAPI && whysperAPI.onTtsStarted) { + whysperAPI.onTtsStarted(() => { + const ttsIndicator = document.getElementById('ttsIndicator'); + if (ttsIndicator) ttsIndicator.style.display = 'inline-block'; + }); + } + if (whysperAPI && whysperAPI.onTtsCompleted) { + whysperAPI.onTtsCompleted(() => { + const ttsIndicator = document.getElementById('ttsIndicator'); + if (ttsIndicator) ttsIndicator.style.display = 'none'; + }); + } + if (whysperAPI && whysperAPI.onTtsError) { + whysperAPI.onTtsError((event, data) => { + const ttsIndicator = document.getElementById('ttsIndicator'); + if (ttsIndicator) ttsIndicator.style.display = 'none'; + console.warn('TTS error:', data?.error); + }); + } + if (whysperAPI && whysperAPI.receive) { + whysperAPI.receive('tts-settings-changed', (data) => { + if (data && data.ttsEnabled !== undefined) { + ttsEnabled = data.ttsEnabled; + } + }); + } + // Also capture generic LLM responses (from OCR/screenshot flows) if (whysperAPI && whysperAPI.onLlmResponse) { whysperAPI.onLlmResponse((event, data) => { try { hideThinkingIndicator(); } catch (_) {} if (data && data.response) { renderAssistantResponse(data.response); + maybeSpeakResponse(data.response); } }); } @@ -1173,7 +1227,7 @@ // Listen for speech errors whysperAPI.onSpeechError((event, data) => { if (data && data.error) { - addMessage(`Error in recognizing speech`, 'error'); + addMessage(`Speech error: ${data.error}`, 'error'); handleRecordingStopped(); } }); @@ -1196,6 +1250,60 @@ if (data && data.response) { try { hideThinkingIndicator(); } catch (_) {} renderAssistantResponse(data.response); + maybeSpeakResponse(data.response); + } + }); + + // Renderer-based audio capture (getUserMedia in chat window) + whysperAPI.onStartRendererCapture(() => { + console.log('[Chat] Starting renderer audio capture'); + if (rendererAudioStream) { + console.log('[Chat] Renderer capture already active'); + return; + } + navigator.mediaDevices.getUserMedia({ audio: true }).then((stream) => { + console.log('[Chat] getUserMedia succeeded'); + rendererAudioStream = stream; + + const audioCtx = new AudioContext(); + rendererAudioContext = audioCtx; + const source = audioCtx.createMediaStreamSource(stream); + const processor = audioCtx.createScriptProcessor(4096, 1, 1); + rendererAudioProcessor = processor; + + source.connect(processor); + processor.onaudioprocess = (e) => { + const input = e.inputBuffer.getChannelData(0); + const int16 = new Int16Array(input.length); + for (let i = 0; i < input.length; i++) { + const s = Math.max(-1, Math.min(1, input[i])); + int16[i] = s < 0 ? s * 0x8000 : s * 0x7FFF; + } + if (whysperAPI && whysperAPI.sendAudioChunk) { + whysperAPI.sendAudioChunk(int16.buffer); + } + }; + }).catch((err) => { + console.error('[Chat] getUserMedia failed:', err.message); + if (whysperAPI && whysperAPI.sendAudioCaptureError) { + whysperAPI.sendAudioCaptureError(err.message); + } + }); + }); + + whysperAPI.onStopRendererCapture(() => { + console.log('[Chat] Stopping renderer audio capture'); + if (rendererAudioProcessor) { + try { rendererAudioProcessor.disconnect(); } catch (_) {} + rendererAudioProcessor = null; + } + if (rendererAudioContext) { + try { rendererAudioContext.close(); } catch (_) {} + rendererAudioContext = null; + } + if (rendererAudioStream) { + rendererAudioStream.getTracks().forEach(t => t.stop()); + rendererAudioStream = null; } }); } @@ -1207,6 +1315,7 @@ if (micButton) { micButton.addEventListener('click', async () => { + console.log('[Chat] Mic button clicked, isInteractive:', isInteractive, 'speechAvailable:', speechAvailable, 'isRecording:', isRecording); if (!isInteractive) { addMessage('Window is in non-interactive mode. Press Alt+A to enable interaction.', 'error'); return; @@ -1218,16 +1327,21 @@ try { if (isRecording) { + console.log('[Chat] Stopping speech recognition'); if (whysperAPI) { const result = await whysperAPI.stopSpeechRecognition(); + console.log('[Chat] Stop result:', result); } } else { + console.log('[Chat] Starting speech recognition'); if (whysperAPI) { const result = await whysperAPI.startSpeechRecognition(); + console.log('[Chat] Start result:', result); } } } catch (error) { - addMessage("Error in recognizing speech", 'error'); + console.error('[Chat] Speech recognition error:', error); + addMessage(`Speech error: ${error.message || error}`, 'error'); } }); } diff --git a/env.example b/env.example index f5fd56a3..75561255 100644 --- a/env.example +++ b/env.example @@ -1,16 +1,32 @@ -# Google Gemini API Configuration +# LLM Provider Configuration +# Choose your AI backend: gemini, openrouter, or groq +LLM_PROVIDER=gemini + +# Google Gemini API Configuration (when LLM_PROVIDER=gemini) # Get your API key from: https://makersuite.google.com/app/apikey GEMINI_API_KEY=your_gemini_api_key_here -# Speech Recognition Configuration -# Choose one provider: azure or whisper +# OpenRouter API Configuration (when LLM_PROVIDER=openrouter) +# Get your API key from: https://openrouter.ai/keys +OPENROUTER_API_KEY= +# Model to use — "openrouter/free" auto-selects free vision-capable models +OPENROUTER_MODEL=openrouter/free + +# Groq API Configuration (when LLM_PROVIDER=groq or SPEECH_PROVIDER=groq) +# Get your API key from: https://console.groq.com/keys +GROQ_API_KEY= +# LLM model — "llama-3.3-70b-versatile" is fast and free +GROQ_MODEL=llama-3.3-70b-versatile + +# Speech / STT Configuration +# Choose one provider: whisper, azure, or groq SPEECH_PROVIDER=whisper -# Optional: Azure Speech Services Configuration +# Azure Speech Services Configuration (when SPEECH_PROVIDER=azure) AZURE_SPEECH_KEY=your_azure_speech_key_here AZURE_SPEECH_REGION=your_azure_region_here -# Optional: Local OpenAI Whisper Configuration +# Local OpenAI Whisper Configuration (when SPEECH_PROVIDER=whisper) # Requires a local Whisper CLI installation, for example: # pip install openai-whisper # brew install ffmpeg sox @@ -20,3 +36,13 @@ WHISPER_MODEL_DIR=.whisper-models WHISPER_MODEL=turbo WHISPER_LANGUAGE=en WHISPER_SEGMENT_MS=4000 + +# Groq STT Configuration (when SPEECH_PROVIDER=groq) +# Uses Whisper Large V3 via Groq's API — requires GROQ_API_KEY above +GROQ_STT_MODEL=whisper-large-v3-turbo + +# Groq TTS Configuration (Text-to-Speech via Orpheus) +# Uses Orpheus TTS — requires GROQ_API_KEY above +GROQ_TTS_MODEL=orpheus-tts-0.1-ayane +GROQ_TTS_VOICE=tara +GROQ_TTS_SPEED=1.0 diff --git a/llm-response.html b/llm-response.html index 40c2669c..b340c3ee 100644 --- a/llm-response.html +++ b/llm-response.html @@ -462,7 +462,7 @@ // Global variables let currentLayout = 'split'; let hasCode = false; - let currentSkill = 'dsa'; + let currentSkill = 'general'; let isInteractive = false; let scrollableElements = []; let initialized = false; diff --git a/main.js b/main.js index a96510d6..c9cb3166 100644 --- a/main.js +++ b/main.js @@ -1,8 +1,15 @@ -require("dotenv").config(); - const path = require("path"); const { app, BrowserWindow, globalShortcut, session, ipcMain } = require("electron"); +// .env lives in app.getPath("userData") (~/Library/Application Support/OpenCluely/) +// so settings survive re-launches when the app is installed in /Applications. +// For development, .env in process.cwd() is also loaded but userData takes priority +// so saved settings persist across restarts. +const userDataPath = app.getPath("userData"); +const userDataEnv = path.join(userDataPath, ".env"); +require("dotenv").config({ path: userDataEnv }); +require("dotenv").config(); + // ── Linux GPU process crash workaround ── // On many Linux setups (Wayland, X11 without GPU drivers, Docker, headless, // or systems with broken Mesa/NVIDIA stacks), Chromium's GPU process crashes @@ -40,6 +47,7 @@ const FirstRunManager = require("./src/core/first-run"); // Screen capture (image-based) const captureService = require("./src/services/capture.service"); const speechService = require("./src/services/speech.service"); +const ttsService = require("./src/services/tts.service"); const llmService = require("./src/services/llm.service"); // Managers @@ -50,19 +58,21 @@ class ApplicationController { constructor() { this.isReady = false; this.starting = false; - this.activeSkill = "dsa"; + this.activeSkill = "general"; // Default to C++ so language is enforced from first run this.codingLanguage = "cpp"; this.speechAvailable = false; + // .env lives in userData so settings survive re-launches from /Applications + this.envPath = path.join(app.getPath("userData"), ".env"); + // First-run onboarding: detects missing .env / API key and triggers // a settings-window prompt on first launch so users don't have to // dig through docs to figure out they need a Gemini API key. this.firstRunManager = new FirstRunManager({ logger: logger, - // Sentinel lives in userData so it survives cwd changes - // (the app may be launched from any directory). sentinelPath: path.join(app.getPath("userData"), ".opencluely-firstrun-completed"), + envPath: this.envPath, }); // Lazily-initialised in getWhisperInstaller() so tests can mock // the constructor without polluting main-process startup. @@ -258,13 +268,65 @@ class ApplicationController { setupPermissions() { session.defaultSession.setPermissionRequestHandler( (webContents, permission, callback) => { - const allowedPermissions = ["microphone", "camera", "display-capture"]; + console.log('[Main] Session permission request:', permission); + const allowedPermissions = ["microphone", "camera", "display-capture", "media"]; const granted = allowedPermissions.includes(permission); logger.debug("Permission request", { permission, granted }); callback(granted); } ); + + session.defaultSession.setPermissionCheckHandler(() => true); + } + + async _ensureMicrophonePermission() { + console.log('[Main] _ensureMicrophonePermission() called'); + // On macOS 14+ the first microphone hardware access via a spawned + // subprocess (sox/rec) can crash the app if permission hasn't been + // granted yet. Use the programmatic API to trigger the dialog safely. + if (process.platform !== "darwin") { + console.log('[Main] Not macOS, skipping mic permission check'); + return; + } + + const { systemPreferences } = require("electron"); + if (!systemPreferences.getMediaAccessStatus) { + console.log('[Main] No getMediaAccessStatus API, skipping'); + return; + } + + const status = systemPreferences.getMediaAccessStatus("microphone"); + logger.debug("Microphone permission status", { status }); + console.log('[Main] Microphone permission status:', status); + + if (status === "granted") { + console.log('[Main] Mic permission already granted'); + return; + } + + if (status === "denied") { + console.log('[Main] Mic permission denied'); + throw new Error( + "Microphone permission denied. " + + "Go to System Settings → Privacy & Security → Microphone, " + + "find OpenCluely in the list and check the box, then restart the app." + ); + } + + // status === "not-determined" — first time; ask safely via the API + if (status === "not-determined" && systemPreferences.askForMediaAccess) { + console.log('[Main] Asking for mic permission...'); + const granted = await systemPreferences.askForMediaAccess("microphone"); + console.log('[Main] askForMediaAccess result:', granted); + if (!granted) { + throw new Error( + "Microphone permission was not granted. " + + "Go to System Settings → Privacy & Security → Microphone " + + "to enable access." + ); + } + } } setupGlobalShortcuts() { @@ -329,7 +391,7 @@ class ApplicationController { text: text.substring(0, 100) }); } - }, 500); + }, 50); }); speechService.on("interim-transcription", (text) => { @@ -379,8 +441,25 @@ class ApplicationController { return speechService.isAvailable ? speechService.isAvailable() : false; }); - ipcMain.handle("start-speech-recognition", () => { - speechService.startRecording(); + ipcMain.handle("start-speech-recognition", async () => { + console.log('[Main] start-speech-recognition invoked'); + try { + await this._ensureMicrophonePermission(); + console.log('[Main] Mic permission OK, calling speechService.startRecording()'); + speechService.startRecording(); + console.log('[Main] speechService.startRecording() returned'); + } catch (e) { + console.error('[Main] start-speech-recognition failed:', e.message); + logger.warn("Microphone permission check failed", { error: e.message }); + windowManager.broadcastToAllWindows("speech-status", { + status: e.message, + available: false, + }); + windowManager.broadcastToAllWindows("speech-error", { + error: e.message, + available: false, + }); + } return speechService.getStatus(); }); @@ -390,14 +469,70 @@ class ApplicationController { }); // Also handle direct send events for fallback - ipcMain.on("start-speech-recognition", () => { - speechService.startRecording(); + ipcMain.on("start-speech-recognition", async () => { + try { + await this._ensureMicrophonePermission(); + speechService.startRecording(); + } catch (e) { + logger.warn("Microphone permission check failed", { error: e.message }); + } }); ipcMain.on("stop-speech-recognition", () => { speechService.stopRecording(); }); + // Audio capture IPC from renderer-based getUserMedia + let chunkCount = 0; + ipcMain.on("audio-chunk", (_event, buffer) => { + chunkCount++; + if (chunkCount % 50 === 1) { + console.log('[Main] Received audio-chunk #', chunkCount, 'size:', buffer?.length || 0); + } + speechService._handleRendererAudioChunk(buffer); + }); + + ipcMain.on("audio-capture-error", (_event, msg) => { + logger.error("Renderer audio capture error", { error: msg }); + speechService.emit("error", `Microphone error: ${msg}`); + speechService.stopRecording(); + }); + + ipcMain.on("audio-capture-ready", (event) => { + const win = event.sender.getOwnerBrowserWindow(); + if (win && !win.isDestroyed()) { + win.hide(); + } + }); + + // TTS (Text-to-Speech) IPC + ipcMain.handle("synthesize-speech", async (_event, text, options) => { + try { + await ttsService.synthesizeSpeech(text, options); + return { success: true }; + } catch (error) { + return { success: false, error: error.message }; + } + }); + + ipcMain.handle("stop-tts-playback", () => { + ttsService.stopPlayback(); + return { success: true }; + }); + + // Broadcast TTS events to all windows + ttsService.on("tts-started", () => { + windowManager.broadcastToAllWindows("tts-started"); + }); + + ttsService.on("tts-completed", () => { + windowManager.broadcastToAllWindows("tts-completed"); + }); + + ttsService.on("tts-error", (error) => { + windowManager.broadcastToAllWindows("tts-error", { error }); + }); + ipcMain.on("chat-window-ready", () => { // Send a test message to confirm communication setTimeout(() => { @@ -532,7 +667,7 @@ class ApplicationController { text: text.substring(0, 100) }); } - }, 500); + }, 50); return { success: true }; }); @@ -642,9 +777,10 @@ class ApplicationController { try { this.firstRunManager.markCompleted(); this.isFirstRun = false; - // Reinitialize speech service with the latest persisted settings - // so the mic button reflects the provider/command set during onboarding. + // Reinitialize services with the latest persisted settings + // so the mic button and AI calls reflect onboarding choices. speechService.initializeClient(); + llmService.initializeClient(); this.speechAvailable = speechService.isAvailable ? speechService.isAvailable() : false; @@ -840,7 +976,7 @@ class ApplicationController { }); } - toggleSpeechRecognition() { + async toggleSpeechRecognition() { const isAvailable = typeof speechService.isAvailable === 'function' ? speechService.isAvailable() : !!speechService.getStatus?.().isInitialized; if (!isAvailable) { logger.warn("Speech recognition unavailable; toggle ignored"); @@ -861,11 +997,18 @@ class ApplicationController { } } else { try { + await this._ensureMicrophonePermission(); speechService.startRecording(); windowManager.showChatWindow(); logger.info("Speech recognition started via global shortcut"); } catch (error) { logger.error("Error starting speech recognition:", error); + try { + windowManager.broadcastToAllWindows("speech-status", { + status: error.message, + available: false, + }); + } catch (_) {} } } } @@ -926,7 +1069,9 @@ class ApplicationController { navigateSkill(direction) { const availableSkills = [ - "dsa", + "general", + "coding", + "meeting", ]; const currentIndex = availableSkills.indexOf(this.activeSkill); @@ -984,7 +1129,7 @@ class ApplicationController { // Use image directly with LLM and active skill; do not send chat messages here const sessionHistory = sessionManager.getOptimizedHistory(); - const skillsRequiringProgrammingLanguage = ['dsa']; + const skillsRequiringProgrammingLanguage = ['coding']; const needsProgrammingLanguage = skillsRequiringProgrammingLanguage.includes(this.activeSkill); const llmResult = await llmService.processImageWithSkill( @@ -1037,7 +1182,7 @@ class ApplicationController { sessionManager.addUserInput(text, 'llm_input'); // Check if current skill needs programming language context - const skillsRequiringProgrammingLanguage = ['dsa']; + const skillsRequiringProgrammingLanguage = ['coding']; const needsProgrammingLanguage = skillsRequiringProgrammingLanguage.includes(this.activeSkill); const llmResult = await llmService.processTextWithSkill( @@ -1116,7 +1261,7 @@ class ApplicationController { }); // Check if current skill needs programming language context - const skillsRequiringProgrammingLanguage = ['dsa']; + const skillsRequiringProgrammingLanguage = ['coding']; const needsProgrammingLanguage = skillsRequiringProgrammingLanguage.includes(this.activeSkill); const llmResult = await llmService.processTranscriptionWithIntelligentResponse( @@ -1318,11 +1463,17 @@ class ApplicationController { // distinguish "unset" from "stale value from a previous load". return { codingLanguage: this.codingLanguage || "cpp", - activeSkill: this.activeSkill || "dsa", + activeSkill: this.activeSkill || "general", appIcon: this.appIcon || "terminal", selectedIcon: this.appIcon || "terminal", windowGap: windowManager.windowGap, + llmProvider: process.env.LLM_PROVIDER || "gemini", + openrouterKey: process.env.OPENROUTER_API_KEY || "", + openrouterModel: process.env.OPENROUTER_MODEL || "openrouter/free", + groqKey: process.env.GROQ_API_KEY || "", + groqModel: process.env.GROQ_MODEL || "llama-3.3-70b-versatile", + speechProvider: speechService.provider || "whisper", azureKey: process.env.AZURE_SPEECH_KEY || "", azureRegion: process.env.AZURE_SPEECH_REGION || "", @@ -1330,6 +1481,11 @@ class ApplicationController { whisperModel: process.env.WHISPER_MODEL || "turbo", whisperLanguage: process.env.WHISPER_LANGUAGE || "en", whisperSegmentMs: process.env.WHISPER_SEGMENT_MS || "4000", + groqSttModel: process.env.GROQ_STT_MODEL || "whisper-large-v3-turbo", + ttsEnabled: true, + ttsVoice: process.env.GROQ_TTS_VOICE || "tara", + ttsModel: process.env.GROQ_TTS_MODEL || "orpheus-tts-0.1-ayane", + ttsSpeed: process.env.GROQ_TTS_SPEED || "1.0", geminiKey: process.env.GEMINI_API_KEY || "", azureConfigured: !!process.env.AZURE_SPEECH_KEY && !!process.env.AZURE_SPEECH_REGION, @@ -1369,7 +1525,7 @@ class ApplicationController { // Writing to .env ensures they survive app restarts and are picked // up the next time the app boots. const envUpdates = {}; - if (settings.speechProvider === "azure" || settings.speechProvider === "whisper") { + if (settings.speechProvider === "azure" || settings.speechProvider === "whisper" || settings.speechProvider === "groq") { envUpdates.SPEECH_PROVIDER = settings.speechProvider; } if (settings.azureKey !== undefined) { @@ -1390,23 +1546,59 @@ class ApplicationController { if (settings.whisperSegmentMs !== undefined) { envUpdates.WHISPER_SEGMENT_MS = String(settings.whisperSegmentMs); } + if (settings.groqSttModel !== undefined) { + envUpdates.GROQ_STT_MODEL = settings.groqSttModel; + } + if (settings.ttsVoice !== undefined) { + envUpdates.GROQ_TTS_VOICE = settings.ttsVoice; + } + if (settings.ttsModel !== undefined) { + envUpdates.GROQ_TTS_MODEL = settings.ttsModel; + } + if (settings.ttsSpeed !== undefined) { + envUpdates.GROQ_TTS_SPEED = String(settings.ttsSpeed); + } + if (settings.groqKey !== undefined) { + envUpdates.GROQ_API_KEY = settings.groqKey; + } + if (settings.llmProvider !== undefined) { + envUpdates.LLM_PROVIDER = settings.llmProvider; + } + if (settings.openrouterKey !== undefined) { + envUpdates.OPENROUTER_API_KEY = settings.openrouterKey; + } + if (settings.openrouterModel !== undefined) { + envUpdates.OPENROUTER_MODEL = settings.openrouterModel; + } + if (settings.groqKey !== undefined) { + envUpdates.GROQ_API_KEY = settings.groqKey; + } + if (settings.groqModel !== undefined) { + envUpdates.GROQ_MODEL = settings.groqModel; + } if (settings.geminiKey !== undefined) { envUpdates.GEMINI_API_KEY = settings.geminiKey; } const persistedKeys = this.persistEnvUpdates(envUpdates); - // If the Gemini key was just saved, reinitialize the LLM service - // so the new client picks up the key. Without this, the test- - // connection button in the onboarding wizard fails with - // "Service not initialized" because the client was first created - // at app startup, before any key was set. - if (settings.geminiKey !== undefined && envUpdates.GEMINI_API_KEY !== undefined) { + // Reinitialize LLM service when provider, API key, or model changes. + // Without this the test-connection button and all AI calls would + // use stale credentials or the wrong provider. + const llmProviderChanged = settings.llmProvider !== undefined && + (process.env.LLM_PROVIDER || "gemini") !== settings.llmProvider; + const llmKeyChanged = settings.geminiKey !== undefined || settings.openrouterKey !== undefined || settings.groqKey !== undefined; + const llmModelChanged = settings.openrouterModel !== undefined || settings.groqModel !== undefined; + if (llmProviderChanged || llmKeyChanged || llmModelChanged) { try { llmService.initializeClient(); - logger.info("LLM service reinitialized after Gemini key update"); + logger.info("LLM service reinitialized after settings change", { + providerChanged: llmProviderChanged, + keyChanged: llmKeyChanged, + modelChanged: llmModelChanged + }); } catch (e) { - logger.warn("Failed to reinitialize LLM service after Gemini key update", { + logger.warn("Failed to reinitialize LLM service", { error: e.message }); } @@ -1421,7 +1613,9 @@ class ApplicationController { const providerChanged = settings.speechProvider && speechService.provider !== settings.speechProvider; const whisperCommandChanged = settings.whisperCommand !== undefined && (process.env.WHISPER_COMMAND || '') !== String(settings.whisperCommand || ''); - if (providerChanged || whisperCommandChanged) { + const groqSttModelChanged = settings.groqSttModel !== undefined && + (process.env.GROQ_STT_MODEL || '') !== String(settings.groqSttModel || ''); + if (providerChanged || whisperCommandChanged || groqSttModelChanged) { try { speechService.initializeClient(); this.speechAvailable = speechService.isAvailable @@ -1439,6 +1633,7 @@ class ApplicationController { logger.info('Speech service reinitialized after settings change', { providerChanged, whisperCommandChanged, + groqSttModelChanged, speechAvailable: this.speechAvailable, }); } catch (e) { @@ -1448,8 +1643,38 @@ class ApplicationController { } } + // Reinitialize TTS service when Groq key or TTS settings change + const groqKeyChanged = settings.groqKey !== undefined; + const ttsSettingsChanged = settings.ttsEnabled !== undefined || + settings.ttsVoice !== undefined || + settings.ttsModel !== undefined || + settings.ttsSpeed !== undefined; + if (groqKeyChanged || ttsSettingsChanged) { + try { + ttsService.updateSettings({ + groqKey: process.env.GROQ_API_KEY, + ttsEnabled: settings.ttsEnabled, + ttsVoice: settings.ttsVoice, + ttsModel: settings.ttsModel, + ttsSpeed: settings.ttsSpeed + }); + logger.info('TTS service reinitialized after settings change'); + // Broadcast TTS settings to chat windows + windowManager.broadcastToAllWindows('tts-settings-changed', { + ttsEnabled: settings.ttsEnabled !== false + }); + } catch (e) { + logger.warn("Failed to reinitialize TTS service", { error: e.message }); + } + } + + const redacted = { ...settings }; + if (redacted.geminiKey) redacted.geminiKey = redacted.geminiKey.substring(0, 4) + "…"; + if (redacted.openrouterKey) redacted.openrouterKey = redacted.openrouterKey.substring(0, 4) + "…"; + if (redacted.groqKey) redacted.groqKey = redacted.groqKey.substring(0, 4) + "…"; + if (redacted.azureKey) redacted.azureKey = redacted.azureKey.substring(0, 4) + "…"; logger.info("Settings saved successfully", { - ...settings, + ...redacted, persistedEnvKeys: persistedKeys }); return { success: true, persistedEnvKeys: persistedKeys }; @@ -1481,7 +1706,13 @@ class ApplicationController { const fs = require("fs"); const path = require("path"); - const envPath = path.join(process.cwd(), ".env"); + const envPath = this.envPath; + + // Ensure the parent directory exists (userData is usually pre-created, + // but guard against edge cases like a sandboxed fresh install). + try { + fs.mkdirSync(path.dirname(envPath), { recursive: true }); + } catch (_) { /* best effort */ } let existing = ""; try { diff --git a/onboarding.html b/onboarding.html index 7e2db50d..388e68dd 100644 --- a/onboarding.html +++ b/onboarding.html @@ -72,6 +72,10 @@ flex-shrink: 0; position: relative; z-index: 1; + -webkit-app-region: drag; + } + .header > * { + -webkit-app-region: no-drag; } .brand { display: flex; @@ -713,6 +717,8 @@
+
+
@@ -740,37 +746,156 @@

Welcome to OpenCluely

-

Connect Google Gemini

+

Choose your AI Provider

- OpenCluely uses Google's Gemini to generate answers. You'll need a free API key. + Pick the AI backend OpenCluely will use to generate answers. You can change this later in Settings.

-
- -
- - - +
+
+
+
+
Google Gemini
+
+ Free API key from Google. Reliable image understanding and fast responses. +
+
+
+
+
+
+
+
OpenRouter (Free models)
+
+ Access many free vision-capable models through one API. The openrouter/free router auto-selects the best available model. +
+
+
-
- Don't have one? Get a free key at - - aistudio.google.com/apikey - . - It's stored locally in .env — never sent anywhere except Google. +
+
+
+
Groq (Fast inference)
+
+ Ultra-fast inference on Groq's LPU hardware. Free tier available. Supports Llama, Mixtral, and Gemma models. +
+
+
- + + +
+
+ +
+ + + +
+
+ Don't have one? Get a free key at + + aistudio.google.com/apikey + . +
+ +
+
+ + + + + + diff --git a/onboarding.js b/onboarding.js index c6e3a449..63c43942 100644 --- a/onboarding.js +++ b/onboarding.js @@ -30,7 +30,12 @@ // ── State ───────────────────────────────────────────────────────── const state = { step: 0, + llmProvider: 'gemini', // 'gemini' | 'openrouter' | 'groq' geminiKey: '', + openrouterKey: '', + openrouterModel: 'openrouter/free', + groqKey: '', + groqModel: 'llama-3.3-70b-versatile', speechProvider: null, // 'whisper' | 'azure' | 'skip' azureKey: '', azureRegion: '', @@ -122,6 +127,12 @@ case 'welcome': return true; case 'apikey': + if (state.llmProvider === 'openrouter') { + return !!state.openrouterKey.trim(); + } + if (state.llmProvider === 'groq') { + return !!state.groqKey.trim(); + } return !!state.geminiKey.trim(); case 'speech': if (state.speechProvider === 'azure') { @@ -181,6 +192,64 @@ : ''; }); + // ── Wire up: LLM provider choices ─────────────────────────────── + const llmProviderChoices = document.getElementById('llmProviderChoices'); + const onboardingGeminiFields = document.getElementById('onboardingGeminiFields'); + const onboardingOpenrouterFields = document.getElementById('onboardingOpenrouterFields'); + const onboardingGroqFields = document.getElementById('onboardingGroqFields'); + + if (llmProviderChoices) { + llmProviderChoices.querySelectorAll('.choice-card').forEach((card) => { + card.addEventListener('click', () => { + const value = card.dataset.value; + state.llmProvider = value; + llmProviderChoices.querySelectorAll('.choice-card').forEach((c) => c.classList.remove('selected')); + card.classList.add('selected'); + if (onboardingGeminiFields) { + onboardingGeminiFields.style.display = value === 'gemini' ? 'block' : 'none'; + } + if (onboardingOpenrouterFields) { + onboardingOpenrouterFields.style.display = value === 'openrouter' ? 'block' : 'none'; + } + if (onboardingGroqFields) { + onboardingGroqFields.style.display = value === 'groq' ? 'block' : 'none'; + } + // Reset the key status when switching providers + if (keyStatus) keyStatus.style.display = 'none'; + }); + }); + } + + // Wire up OpenRouter key and model inputs + const onboardingOpenrouterKey = document.getElementById('onboardingOpenrouterKey'); + const onboardingOpenrouterModel = document.getElementById('onboardingOpenrouterModel'); + + if (onboardingOpenrouterKey) { + onboardingOpenrouterKey.addEventListener('input', () => { + state.openrouterKey = onboardingOpenrouterKey.value.trim(); + }); + } + if (onboardingOpenrouterModel) { + onboardingOpenrouterModel.addEventListener('input', () => { + state.openrouterModel = onboardingOpenrouterModel.value.trim() || 'openrouter/free'; + }); + } + + // Wire up Groq key and model inputs + const onboardingGroqKey = document.getElementById('onboardingGroqKey'); + const onboardingGroqModel = document.getElementById('onboardingGroqModel'); + + if (onboardingGroqKey) { + onboardingGroqKey.addEventListener('input', () => { + state.groqKey = onboardingGroqKey.value.trim(); + }); + } + if (onboardingGroqModel) { + onboardingGroqModel.addEventListener('input', () => { + state.groqModel = onboardingGroqModel.value.trim() || 'llama-3.3-70b-versatile'; + }); + } + // ── Wire up: Speech choices ─────────────────────────────────────── $$('#speechChoices .choice-card').forEach((card) => { card.addEventListener('click', () => { @@ -436,11 +505,45 @@ // ── Wire up: Finish screen ──────────────────────────────────────── function populateSummary() { const rows = []; - rows.push({ - label: ' Gemini API', - value: state.geminiKey ? 'Configured' : 'Missing', - cls: state.geminiKey ? 'ok' : 'skip', - }); + if (state.llmProvider === 'openrouter') { + rows.push({ + label: ' LLM Provider', + value: 'OpenRouter', + cls: 'ok', + }); + rows.push({ + label: ' OpenRouter Key', + value: state.openrouterKey ? 'Configured' : 'Missing', + cls: state.openrouterKey ? 'ok' : 'skip', + }); + rows.push({ + label: ' Model', + value: state.openrouterModel, + cls: 'ok', + }); + } else if (state.llmProvider === 'groq') { + rows.push({ + label: ' LLM Provider', + value: 'Groq', + cls: 'ok', + }); + rows.push({ + label: ' Groq Key', + value: state.groqKey ? 'Configured' : 'Missing', + cls: state.groqKey ? 'ok' : 'skip', + }); + rows.push({ + label: ' Model', + value: state.groqModel, + cls: 'ok', + }); + } else { + rows.push({ + label: ' Gemini API', + value: state.geminiKey ? 'Configured' : 'Missing', + cls: state.geminiKey ? 'ok' : 'skip', + }); + } if (state.speechProvider === 'whisper') { rows.push({ label: ' Speech', @@ -500,16 +603,31 @@ const name = currentScreenName(); if (!canAdvance()) { // Lightly nudge the user - if (name === 'apikey') setKeyStatus('error', 'Enter a Gemini API key'); + if (name === 'apikey') { + const msg = state.llmProvider === 'openrouter' + ? 'Enter your OpenRouter API key' + : state.llmProvider === 'groq' + ? 'Enter your Groq API key' + : 'Enter your Gemini API key'; + setKeyStatus('error', msg); + } return; } - // Persist settings on speech selection (Azure path), since we - // already saved geminiKey on test; do it here too if user skipped - // testing. - if (name === 'apikey' && state.geminiKey && window.electronAPI) { + // Persist settings on API key step — saves provider, key, and model + if (name === 'apikey' && window.electronAPI) { try { - await window.electronAPI.saveSettings({ geminiKey: state.geminiKey }); + const payload = { llmProvider: state.llmProvider }; + if (state.llmProvider === 'openrouter') { + if (state.openrouterKey) payload.openrouterKey = state.openrouterKey; + payload.openrouterModel = state.openrouterModel; + } else if (state.llmProvider === 'groq') { + if (state.groqKey) payload.groqKey = state.groqKey; + payload.groqModel = state.groqModel; + } else if (state.geminiKey) { + payload.geminiKey = state.geminiKey; + } + await window.electronAPI.saveSettings(payload); } catch (_) { /* surfaced elsewhere */ } } if (name === 'speech' && window.electronAPI) { @@ -641,16 +759,35 @@ // ── Boot ────────────────────────────────────────────────────────── showScreen('welcome'); - // Pre-populate Gemini key from existing .env (if any) so users with + // Pre-populate provider and key from existing .env (if any) so users with // a partial config don't have to retype. - if (window.electronAPI && window.electronAPI.getFirstRunStatus) { - window.electronAPI.getFirstRunStatus().then((s) => { - if (s && s.geminiConfigured) { - // We can't read the key back (settings returns empty for keys), - // but we can mark status as success if the env file already has one. + (async function restoreExistingConfig() { + if (!window.electronAPI || !window.electronAPI.getFirstRunStatus) return; + try { + const s = await window.electronAPI.getFirstRunStatus(); + if (!s) return; + + // Restore provider choice + if (s.llmProvider === 'openrouter') { + state.llmProvider = 'openrouter'; + const orCard = llmProviderChoices?.querySelector('[data-value="openrouter"]'); + if (orCard) orCard.click(); + if (onboardingOpenrouterKey) onboardingOpenrouterKey.placeholder = '•••••••••••••••• (already set)'; + if (onboardingOpenrouterModel) onboardingOpenrouterModel.value = s.openrouterModel || 'openrouter/free'; + state.openrouterModel = s.openrouterModel || 'openrouter/free'; + } else if (s.llmProvider === 'groq') { + state.llmProvider = 'groq'; + const groqCard = llmProviderChoices?.querySelector('[data-value="groq"]'); + if (groqCard) groqCard.click(); + if (onboardingGroqKey) onboardingGroqKey.placeholder = '•••••••••••••••• (already set)'; + if (onboardingGroqModel) onboardingGroqModel.value = s.groqModel || 'llama-3.3-70b-versatile'; + state.groqModel = s.groqModel || 'llama-3.3-70b-versatile'; + } else if (s.geminiConfigured) { + const geminiCard = llmProviderChoices?.querySelector('[data-value="gemini"]'); + if (geminiCard) geminiCard.click(); setKeyStatus('success', 'Already configured — click Continue'); geminiInput.placeholder = '•••••••••••••••• (already set)'; } - }).catch(() => {}); - } + } catch (_) { /* ignore */ } + })(); })(); \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 708a5500..9c556105 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "hasInstallScript": true, "license": "ISC", "dependencies": { + "@ffmpeg-installer/ffmpeg": "^1.1.0", "@fortawesome/fontawesome-free": "^7.2.0", "@google/genai": "^2.9.0", "dotenv": "^16.3.1", @@ -337,6 +338,132 @@ "node": ">= 10.0.0" } }, + "node_modules/@ffmpeg-installer/darwin-arm64": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/@ffmpeg-installer/darwin-arm64/-/darwin-arm64-4.1.5.tgz", + "integrity": "sha512-hYqTiP63mXz7wSQfuqfFwfLOfwwFChUedeCVKkBtl/cliaTM7/ePI9bVzfZ2c+dWu3TqCwLDRWNSJ5pqZl8otA==", + "cpu": [ + "arm64" + ], + "hasInstallScript": true, + "license": "https://git.ffmpeg.org/gitweb/ffmpeg.git/blob_plain/HEAD:/LICENSE.md", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@ffmpeg-installer/darwin-x64": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@ffmpeg-installer/darwin-x64/-/darwin-x64-4.1.0.tgz", + "integrity": "sha512-Z4EyG3cIFjdhlY8wI9aLUXuH8nVt7E9SlMVZtWvSPnm2sm37/yC2CwjUzyCQbJbySnef1tQwGG2Sx+uWhd9IAw==", + "cpu": [ + "x64" + ], + "hasInstallScript": true, + "license": "LGPL-2.1", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@ffmpeg-installer/ffmpeg": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@ffmpeg-installer/ffmpeg/-/ffmpeg-1.1.0.tgz", + "integrity": "sha512-Uq4rmwkdGxIa9A6Bd/VqqYbT7zqh1GrT5/rFwCwKM70b42W5gIjWeVETq6SdcL0zXqDtY081Ws/iJWhr1+xvQg==", + "license": "LGPL-2.1", + "optionalDependencies": { + "@ffmpeg-installer/darwin-arm64": "4.1.5", + "@ffmpeg-installer/darwin-x64": "4.1.0", + "@ffmpeg-installer/linux-arm": "4.1.3", + "@ffmpeg-installer/linux-arm64": "4.1.4", + "@ffmpeg-installer/linux-ia32": "4.1.0", + "@ffmpeg-installer/linux-x64": "4.1.0", + "@ffmpeg-installer/win32-ia32": "4.1.0", + "@ffmpeg-installer/win32-x64": "4.1.0" + } + }, + "node_modules/@ffmpeg-installer/linux-arm": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/@ffmpeg-installer/linux-arm/-/linux-arm-4.1.3.tgz", + "integrity": "sha512-NDf5V6l8AfzZ8WzUGZ5mV8O/xMzRag2ETR6+TlGIsMHp81agx51cqpPItXPib/nAZYmo55Bl2L6/WOMI3A5YRg==", + "cpu": [ + "arm" + ], + "hasInstallScript": true, + "license": "GPLv3", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@ffmpeg-installer/linux-arm64": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/@ffmpeg-installer/linux-arm64/-/linux-arm64-4.1.4.tgz", + "integrity": "sha512-dljEqAOD0oIM6O6DxBW9US/FkvqvQwgJ2lGHOwHDDwu/pX8+V0YsDL1xqHbj1DMX/+nP9rxw7G7gcUvGspSoKg==", + "cpu": [ + "arm64" + ], + "hasInstallScript": true, + "license": "GPLv3", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@ffmpeg-installer/linux-ia32": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@ffmpeg-installer/linux-ia32/-/linux-ia32-4.1.0.tgz", + "integrity": "sha512-0LWyFQnPf+Ij9GQGD034hS6A90URNu9HCtQ5cTqo5MxOEc7Rd8gLXrJvn++UmxhU0J5RyRE9KRYstdCVUjkNOQ==", + "cpu": [ + "ia32" + ], + "hasInstallScript": true, + "license": "GPLv3", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@ffmpeg-installer/linux-x64": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@ffmpeg-installer/linux-x64/-/linux-x64-4.1.0.tgz", + "integrity": "sha512-Y5BWhGLU/WpQjOArNIgXD3z5mxxdV8c41C+U15nsE5yF8tVcdCGet5zPs5Zy3Ta6bU7haGpIzryutqCGQA/W8A==", + "cpu": [ + "x64" + ], + "hasInstallScript": true, + "license": "GPLv3", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@ffmpeg-installer/win32-ia32": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@ffmpeg-installer/win32-ia32/-/win32-ia32-4.1.0.tgz", + "integrity": "sha512-FV2D7RlaZv/lrtdhaQ4oETwoFUsUjlUiasiZLDxhEUPdNDWcH1OU9K1xTvqz+OXLdsmYelUDuBS/zkMOTtlUAw==", + "cpu": [ + "ia32" + ], + "license": "GPLv3", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@ffmpeg-installer/win32-x64": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@ffmpeg-installer/win32-x64/-/win32-x64-4.1.0.tgz", + "integrity": "sha512-Drt5u2vzDnIONf4ZEkKtFlbvwj6rI3kxw1Ck9fpudmtgaZIHD4ucsWB2lCZBXRxJgXR+2IMSti+4rtM4C4rXgg==", + "cpu": [ + "x64" + ], + "license": "GPLv3", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@fortawesome/fontawesome-free": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-free/-/fontawesome-free-7.2.0.tgz", @@ -842,7 +969,6 @@ "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -1017,6 +1143,7 @@ "integrity": "sha512-+25nxyyznAXF7Nef3y0EbBeqmGZgeN/BxHX29Rs39djAfaFalmQ89SE6CWyDCHzGL0yt/ycBtNOmGTW0FyGWNw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "archiver-utils": "^2.1.0", "async": "^3.2.4", @@ -1036,6 +1163,7 @@ "integrity": "sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "glob": "^7.1.4", "graceful-fs": "^4.2.0", @@ -1058,6 +1186,7 @@ "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -1073,7 +1202,8 @@ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/archiver-utils/node_modules/string_decoder": { "version": "1.1.1", @@ -1081,6 +1211,7 @@ "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "safe-buffer": "~5.1.0" } @@ -1200,6 +1331,7 @@ "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "buffer": "^5.5.0", "inherits": "^2.0.4", @@ -1615,6 +1747,7 @@ "integrity": "sha512-D3uMHtGc/fcO1Gt1/L7i1e33VOvD4A9hfQLP+6ewd+BvG/gQ84Yh4oftEhAdjSMgBgwGL+jsppT7JYNpo6MHHg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "buffer-crc32": "^0.2.13", "crc32-stream": "^4.0.2", @@ -1714,6 +1847,7 @@ "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "crc32": "bin/crc32.njs" }, @@ -1727,6 +1861,7 @@ "integrity": "sha512-NT7w2JVU7DFroFdYkeq8cywxrgjPHWkdX1wjpRQXPX5Asews3tA+Ght6lddQO5Mkumffp3X7GEqku3epj2toIw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "crc-32": "^1.2.0", "readable-stream": "^3.4.0" @@ -1912,7 +2047,6 @@ "integrity": "sha512-rcJUkMfnJpfCboZoOOPf4L29TRtEieHNOeAbYPWPxlaBw/Z1RKrRA86dOI9rwaI4tQSc/RD82zTNHprfUHXsoQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "app-builder-lib": "24.13.3", "builder-util": "24.13.1", @@ -2108,6 +2242,7 @@ "integrity": "sha512-oHkV0iogWfyK+ah9ZIvMDpei1m9ZRpdXcvde1wTpra2U8AFDNNpqJdnin5z+PM1GbQ5BoaKCWas2HSjtR0HwMg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "app-builder-lib": "24.13.3", "archiver": "^5.3.1", @@ -2121,6 +2256,7 @@ "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -2136,6 +2272,7 @@ "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "universalify": "^2.0.0" }, @@ -2149,6 +2286,7 @@ "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">= 10.0.0" } @@ -2533,7 +2671,8 @@ "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/fs-extra": { "version": "8.1.0", @@ -3122,7 +3261,8 @@ "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/isbinaryfile": { "version": "5.0.4", @@ -3320,6 +3460,7 @@ "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "readable-stream": "^2.0.5" }, @@ -3333,6 +3474,7 @@ "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -3348,7 +3490,8 @@ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/lazystream/node_modules/string_decoder": { "version": "1.1.1", @@ -3356,6 +3499,7 @@ "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "safe-buffer": "~5.1.0" } @@ -3372,35 +3516,40 @@ "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/lodash.difference": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", "integrity": "sha512-dS2j+W26TQ7taQBGN8Lbbq04ssV3emRw4NY58WErlTO29pIqS0HmoT5aJ9+TUQ1N3G+JOZSji4eugsWwGp9yPA==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/lodash.flatten": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", "integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/lodash.isplainobject": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/lodash.union": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/lodash.union/-/lodash.union-4.6.0.tgz", "integrity": "sha512-c4pB2CdGrGdjMKYLA+XiRDO7Y0PRQbm/Gzg8qMj+QH+pFVAoTp5sBpO0odL3FjoPCGjK96p6qsP+yQoiLoOBcw==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/logform": { "version": "2.7.0", @@ -3769,6 +3918,7 @@ "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -3944,7 +4094,8 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/progress": { "version": "2.0.3", @@ -4075,6 +4226,7 @@ "integrity": "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==", "dev": true, "license": "Apache-2.0", + "peer": true, "dependencies": { "minimatch": "^5.1.0" } @@ -4490,6 +4642,7 @@ "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", @@ -4708,7 +4861,6 @@ "resolved": "https://registry.npmjs.org/winston/-/winston-3.17.0.tgz", "integrity": "sha512-DLiFIXYC5fMPxaRg832S6F5mJYvePtmO5G9v9IgUFPhXm9/GkXarH/TUrBAVzhTCzAj9anE/+GjrgXp/54nOgw==", "license": "MIT", - "peer": true, "dependencies": { "@colors/colors": "^1.6.0", "@dabh/diagnostics": "^2.0.2", @@ -4896,6 +5048,7 @@ "integrity": "sha512-9qv4rlDiopXg4E69k+vMHjNN63YFMe9sZMrdlvKnCjlCRWeCBswPPMPUfx+ipsAWq1LXHe70RcbaHdJJpS6hyQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "archiver-utils": "^3.0.4", "compress-commons": "^4.1.2", @@ -4911,6 +5064,7 @@ "integrity": "sha512-KVgf4XQVrTjhyWmx6cte4RxonPLR9onExufI1jhvw/MQ4BB6IsZD5gT8Lq+u/+pRkWna/6JoHpiQioaqFP5Rzw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "glob": "^7.2.3", "graceful-fs": "^4.2.0", diff --git a/package.json b/package.json index e120cc31..b9396cbf 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ }, "license": "ISC", "dependencies": { + "@ffmpeg-installer/ffmpeg": "^1.1.0", "@fortawesome/fontawesome-free": "^7.2.0", "@google/genai": "^2.9.0", "dotenv": "^16.3.1", @@ -85,7 +86,12 @@ ], "darkModeSupport": true, "hardenedRuntime": false, - "gatekeeperAssess": false + "gatekeeperAssess": false, + "extendInfo": { + "NSMicrophoneUsageDescription": "OpenCluely needs microphone access to capture audio for speech-to-text transcription.", + "NSCameraUsageDescription": "OpenCluely needs screen capture access to analyse your screen content.", + "NSScreenCaptureDescription": "OpenCluely needs screen recording permission to capture and analyse your screen." + } }, "win": { "target": [ @@ -160,5 +166,10 @@ "height": 400 } } + }, + "allowScripts": { + "electron@29.4.6": true, + "@google/genai@2.9.0": true, + "protobufjs@7.6.4": true } } diff --git a/preload.js b/preload.js index 2b24ea53..59fe2c0b 100644 --- a/preload.js +++ b/preload.js @@ -115,12 +115,37 @@ contextBridge.exposeInMainWorld('electronAPI', { onRecordingStopped: (callback) => ipcRenderer.on('recording-stopped', callback), onCodingLanguageChanged: (callback) => ipcRenderer.on('coding-language-changed', callback), onMainWindowShown: (callback) => ipcRenderer.on('main-window-shown', callback), + + // TTS (Text-to-Speech) + synthesizeSpeech: (text, options) => ipcRenderer.invoke('synthesize-speech', text, options), + stopTtsPlayback: () => ipcRenderer.invoke('stop-tts-playback'), + onTtsStarted: (callback) => ipcRenderer.on('tts-started', callback), + onTtsCompleted: (callback) => ipcRenderer.on('tts-completed', callback), + onTtsError: (callback) => ipcRenderer.on('tts-error', callback), // Generic receive method receive: (channel, callback) => ipcRenderer.on(channel, callback), // Remove listeners - removeAllListeners: (channel) => ipcRenderer.removeAllListeners(channel) + removeAllListeners: (channel) => ipcRenderer.removeAllListeners(channel), + + // Audio capture (renderer-based microphone via getUserMedia) + sendAudioChunk: (buffer) => ipcRenderer.send('audio-chunk', buffer), + sendAudioCaptureError: (msg) => ipcRenderer.send('audio-capture-error', msg), + audioCaptureReady: () => ipcRenderer.send('audio-capture-ready'), + onStartRendererCapture: (callback) => { + const wrapped = () => { try { callback(); } catch (e) { console.error('onStartRendererCapture error:', e); } }; + ipcRenderer.on('start-renderer-capture', wrapped); + return () => ipcRenderer.removeListener('start-renderer-capture', wrapped); + }, + onStopRendererCapture: (callback) => { + ipcRenderer.on('stop-renderer-capture', callback); + return () => ipcRenderer.removeListener('stop-renderer-capture', callback); + }, + onStopAudioCapture: (callback) => { + ipcRenderer.on('stop-audio-capture', callback); + return () => ipcRenderer.removeListener('stop-audio-capture', callback); + } }) contextBridge.exposeInMainWorld('api', { diff --git a/prompt-loader.js b/prompt-loader.js index e57259a0..ad802410 100644 --- a/prompt-loader.js +++ b/prompt-loader.js @@ -7,7 +7,7 @@ class PromptLoader { this.promptsLoaded = false; this.skillPromptSent = new Set(); // Focus only on DSA - this.skillsRequiringProgrammingLanguage = ['dsa']; + this.skillsRequiringProgrammingLanguage = ['coding']; } /** @@ -28,7 +28,7 @@ class PromptLoader { for (const file of files) { if (file.endsWith('.md')) { const skillName = path.basename(file, '.md'); - if (skillName !== 'dsa') continue; // only keep DSA + if (!['general', 'coding', 'meeting'].includes(skillName)) continue; const filePath = path.join(promptsDir, file); const promptContent = fs.readFileSync(filePath, 'utf8'); @@ -88,7 +88,7 @@ class PromptLoader { let languageInjection = ''; switch (skillName) { - case 'dsa': + case 'coding': languageInjection = `\n\n## IMPLEMENTATION LANGUAGE: ${languageUpper} STRICT REQUIREMENTS: - Respond ONLY in ${languageTitle}. Do not include any snippets or alternatives in other languages. @@ -322,39 +322,23 @@ STRICT REQUIREMENTS: // Convert to lowercase and handle common variations const normalized = skillName.toLowerCase().trim(); - // Map common variations to standard names + // Map common variations to canonical skill names const skillMap = { - 'dsa': 'dsa', - 'data-structures': 'dsa', - 'algorithms': 'dsa', - 'data-structures-algorithms': 'dsa', - 'behavioral': 'behavioral', - 'behavioral-interview': 'behavioral', - 'behavior': 'behavioral', - 'sales': 'sales', - 'selling': 'sales', - 'business-development': 'sales', - 'presentation': 'presentation', - 'presentations': 'presentation', - 'public-speaking': 'presentation', - 'data-science': 'data-science', - 'datascience': 'data-science', - 'machine-learning': 'data-science', - 'ml': 'data-science', - 'programming': 'programming', - 'coding': 'programming', - 'software-development': 'programming', - 'development': 'programming', - 'devops': 'devops', - 'dev-ops': 'devops', - 'infrastructure': 'devops', - 'system-design': 'system-design', - 'systems-design': 'system-design', - 'architecture': 'system-design', - 'distributed-systems': 'system-design', - 'negotiation': 'negotiation', - 'negotiating': 'negotiation', - 'conflict-resolution': 'negotiation' + 'general': 'general', + 'gen': 'general', + 'default': 'general', + 'coding': 'coding', + 'code': 'coding', + 'programming': 'coding', + 'software': 'coding', + 'development': 'coding', + 'dsa': 'coding', + 'algorithms': 'coding', + 'data-structures': 'coding', + 'meeting': 'meeting', + 'meetings': 'meeting', + 'conference': 'meeting', + 'call': 'meeting' }; return skillMap[normalized] || normalized; @@ -368,7 +352,7 @@ STRICT REQUIREMENTS: if (!this.promptsLoaded) { this.loadPrompts(); } - return ['dsa']; + return ['general', 'coding', 'meeting']; } /** diff --git a/prompts/coding.md b/prompts/coding.md new file mode 100644 index 00000000..d3558cc1 --- /dev/null +++ b/prompts/coding.md @@ -0,0 +1,24 @@ +# Coding Assistant Agent + +You are a programming expert that helps users solve coding problems, debug issues, and understand code. + +## STRICT RULES + +- Output code ONLY in the user-selected language. No alternatives unless asked. +- Use triple backticks with the correct language tag. +- Ensure all code is syntactically correct and handles edge cases. +- If there's pre-code or a template in the question, strictly use that template to structure your answer. +- Avoid unnecessary commentary in code; keep code clean and readable. +- Your code should have minimal comments unless the logic is non-obvious. + +## Response Structure + +1. Understand the problem by restating it briefly +2. Explain your approach with the reasoning behind it +3. Provide the implementation in the selected language +4. Note time and space complexity where applicable +5. Mention edge cases or testing considerations + +## Programming Language + +When a programming language is specified, ALL code must be in that language. Use the appropriate language tag for code fences. Do not provide alternatives in other languages unless explicitly asked. diff --git a/prompts/dsa.md b/prompts/dsa.md deleted file mode 100644 index f2300fcb..00000000 --- a/prompts/dsa.md +++ /dev/null @@ -1,28 +0,0 @@ -# DSA Interview Helper Agent (Focused & Optimal) - -You are a competitive programming expert that outputs the most optimal solution with minimal time and space complexity. - -STRICT RULES -- Output code ONLY in the user-selected language. No alternatives unless asked. -- Use triple backticks with the correct language tag. -- Prefer O(n) or O(n log n) where feasible; call out if optimal lower bound is higher. -- if there's some pre-code or template in Question then strictly use that template to answer it. -- Avoid extra commentary; be concise and implementation-focused. -- Your code must not contain any comments. - -Workflow -1) Identify the problem pattern quickly (Array, Hashing, Two Pointers, Sliding Window, Binary Search, Stack/Queue, Linked List, Tree/Graph, Heap, Greedy, DP). -2) State naive idea in 1–2 lines with complexity. -3) Give optimal approach with 3–5 bullet steps. -4) Provide clean, production-ready, comment-free implementation in the selected language. -5) State time and space complexity precisely. -6) Optional: 1 short dry-run example if non-obvious. - -Implementation Template -```lang -``` - -Notes -- Prefer iterative over recursive when it reduces stack usage or improves clarity. -- Use built-in data structures and libraries idiomatically for the selected language. -- For DP, specify state, transition, and memory optimization opportunities. \ No newline at end of file diff --git a/prompts/general.md b/prompts/general.md new file mode 100644 index 00000000..470bc1e1 --- /dev/null +++ b/prompts/general.md @@ -0,0 +1,19 @@ +# General Assistant Agent + +You are a versatile AI assistant capable of answering a wide variety of questions across different domains. Adapt your response style to the question being asked. + +## Response Guidelines + +- Be concise and direct when the question is straightforward +- Provide detailed explanations when the topic requires depth +- Use examples to illustrate complex concepts +- If asked for code, provide clean, well-structured implementations +- If asked for analysis, break down the problem step by step +- Stay factual and acknowledge uncertainty when appropriate + +## Communication Style + +- Professional but approachable +- Tailor technical depth to the question's implied audience +- Prioritize clarity and correctness over brevity for complex topics +- Use bullet points, steps, or structured formats when they improve clarity diff --git a/prompts/meeting.md b/prompts/meeting.md new file mode 100644 index 00000000..d9afc64a --- /dev/null +++ b/prompts/meeting.md @@ -0,0 +1,48 @@ +# Meeting Assistant Agent + +You are a meeting assistant that listens to conversations and provides helpful information when needed. + +## Role + +- Listen to meeting conversations passively +- Detect when a question is asked that you can answer +- Provide concise, relevant answers only when helpful +- Do not interrupt or respond to casual conversation, chit-chat, or statements +- Be unobtrusive — answers should be informative but brief + +## Question Detection + +A question is being asked when the speaker: +- Uses question words (what, why, how, when, where, who, which) +- Asks for an explanation, definition, or example +- Requests a suggestion, recommendation, or opinion +- Asks for data, facts, or information +- Uses a rising intonation or phrasing that indicates uncertainty + +## Response Guidelines + +When a question is detected: +- Provide a direct, concise answer +- Include relevant facts, data, or explanations +- Use examples if they help clarify +- Avoid tangential information +- Keep responses under 3-4 sentences when possible + +When no question is detected: +- Do not respond +- Stay silent and continue listening +- Do not acknowledge statements or casual conversation + +## Examples of When to Respond + +- "What's the best way to handle this situation?" → Provide suggestions +- "Can someone explain how this works?" → Provide explanation +- "What are the trade-offs here?" → List pros and cons +- "How do you implement this?" → Provide implementation approach + +## Examples of When NOT to Respond + +- "Let's move to the next agenda item" → Silent +- "I think we should consider this approach" → Silent +- "Good morning everyone" → Silent +- "That's a good point" → Silent diff --git a/prompts/programming.md b/prompts/programming.md deleted file mode 100644 index 42efc53d..00000000 --- a/prompts/programming.md +++ /dev/null @@ -1,52 +0,0 @@ -# Programming Interview Helper Agent - -You are a concise programming interview assistant. Provide quick, actionable guidance without revealing you're an AI helper. - -## Response Structure - -### 1. Naive Approach (30 seconds) -- State the simplest solution first -- Mention time/space complexity -- One-line reasoning why it works - -### 2. Optimized Solution (2 minutes) -- Best approach with clear explanation -- Step-by-step algorithm breakdown -- Time/space complexity analysis - -### 3. Dry Run (1 minute) -- Walk through with a concrete example -- Show key variable states at each step -- Highlight the core insight - -### 4. Production Code -```language -// Clean, interview-ready implementation -// Include edge case handling -// Add meaningful comments -``` - -### 5. Quick Validation -- 2-3 test cases (edge cases included) -- Alternative approaches if time permits - -## Communication Style -- Start with "Let me think through this step by step" -- Use "First, the straightforward approach would be..." -- Transition with "But we can optimize this by..." -- Be conversational, not robotic -- Show your thought process naturally - -## Key Technologies to Reference -**Data Structures**: Arrays, HashMaps, Trees, Graphs, Heaps, Stacks, Queues -**Algorithms**: Two Pointers, Sliding Window, DFS/BFS, Dynamic Programming, Binary Search -**Patterns**: Divide & Conquer, Greedy, Backtracking, Memoization - -## Common Optimizations -- HashMap for O(1) lookups instead of nested loops -- Two pointers for array problems -- Binary search for sorted data -- DP for overlapping subproblems -- BFS/DFS for tree/graph traversal - -Give direct, implementable solutions with clear reasoning. Focus on demonstrating problem-solving skills naturally. \ No newline at end of file diff --git a/settings.html b/settings.html index c706860e..c447dd20 100644 --- a/settings.html +++ b/settings.html @@ -261,6 +261,10 @@ width: 100%; } + .settings-group { + /* Groups provider-specific fields for show/hide toggling */ + } + .settings-note { font-size: 11px; color: rgba(255, 255, 255, 0.65); @@ -317,7 +321,9 @@
Choose your current focus area
@@ -360,6 +366,7 @@
@@ -418,24 +425,143 @@ + +
+
+
+
Groq STT Model
+
Groq's Whisper model for speech-to-text
+
+ +
+
+ Uses your GROQ_API_KEY for transcription via Groq's API. No local Whisper CLI required. +
+
+
Azure credentials apply immediately and are also written back to .env so they persist across restarts.
+ +
+
+ + Text-to-Speech +
+
+
+
+
Enable TTS
+
Read AI responses aloud using Groq Orpheus
+
+ +
+
+
+
TTS Voice
+
Orpheus voice for speech synthesis
+
+ +
+
+
+
TTS Speed
+
Playback speed (0.25 – 4.0)
+
+ +
+
+ Requires GROQ_API_KEY. TTS uses the Orpheus English model via Groq's API. +
+
+
- Gemini Settings + LLM Provider
-
Google API Key
-
Your Google API key for Gemini models
+
Provider
+
Choose your AI backend
+
+ +
+ +
+
+
+
Gemini API Key
+
Your Google API key for Gemini models
+
+ +
+
+ + + + + +
diff --git a/src/core/config.js b/src/core/config.js index 91aba65c..09654a1e 100644 --- a/src/core/config.js +++ b/src/core/config.js @@ -39,14 +39,12 @@ class ConfigManager { }, llm: { + provider: 'gemini', gemini: { - // 'gemini-3.5-flash' is Google's current stable Flash model. - // 'gemini-3-flash-preview' is the fallback because it also has a - // free tier and is less likely to hit demand spikes. model: 'gemini-3.5-flash', fallbackModels: ['gemini-3-flash-preview', 'gemini-2.5-flash'], - maxRetries: 3, - timeout: 60000, + maxRetries: 1, + timeout: 30000, fallbackEnabled: true, enableFallbackMethod: true, generation: { @@ -55,6 +53,26 @@ class ConfigManager { topP: 0.9, maxOutputTokens: 4096 } + }, + openrouter: { + baseUrl: 'https://openrouter.ai/api/v1', + model: 'openrouter/free', + maxRetries: 1, + timeout: 30000, + generation: { + temperature: 0.7, + maxOutputTokens: 4096 + } + }, + groq: { + baseUrl: 'https://api.groq.com/openai/v1', + model: 'llama-3.3-70b-versatile', + maxRetries: 1, + timeout: 30000, + generation: { + temperature: 0.7, + maxOutputTokens: 4096 + } } }, @@ -70,6 +88,12 @@ class ConfigManager { model: 'turbo', language: 'en', segmentMs: 4000 + }, + groq: { + sttModel: 'whisper-large-v3-turbo', + ttsModel: 'orpheus-tts-0.1-ayane', + ttsVoice: 'tara', + ttsSpeed: 1.0 } }, diff --git a/src/core/first-run.js b/src/core/first-run.js index 67e3d133..7b02793b 100644 --- a/src/core/first-run.js +++ b/src/core/first-run.js @@ -25,12 +25,22 @@ class FirstRunManager { /** * Returns true if this looks like a fresh install — no .env, no - * sentinel file, or .env exists but has no Gemini key. + * sentinel file, or .env exists but has no API key for the chosen + * provider. */ needsOnboarding() { if (!fs.existsSync(this.sentinelPath)) return true; if (!fs.existsSync(this.envPath)) return true; const content = this._readEnv(); + const provider = (content.LLM_PROVIDER || 'gemini').trim().toLowerCase(); + if (provider === 'openrouter') { + const orKey = (content.OPENROUTER_API_KEY || '').trim(); + return !orKey || orKey === 'your_openrouter_api_key_here'; + } + if (provider === 'groq') { + const grKey = (content.GROQ_API_KEY || '').trim(); + return !grKey || grKey === 'your_groq_api_key_here'; + } const gemini = (content.GEMINI_API_KEY || '').trim(); return !gemini || gemini === 'your_gemini_api_key_here'; } @@ -80,10 +90,19 @@ class FirstRunManager { getStatus() { const env = this._readEnv(); const gemini = (env.GEMINI_API_KEY || '').trim(); + const orKey = (env.OPENROUTER_API_KEY || '').trim(); + const orKeyConfigured = !!orKey && orKey !== 'your_openrouter_api_key_here'; + const grKey = (env.GROQ_API_KEY || '').trim(); + const grKeyConfigured = !!grKey && grKey !== 'your_groq_api_key_here'; return { envExists: fs.existsSync(this.envPath), sentinelExists: fs.existsSync(this.sentinelPath), + llmProvider: env.LLM_PROVIDER || 'gemini', geminiConfigured: !!gemini && gemini !== 'your_gemini_api_key_here', + openrouterConfigured: orKeyConfigured, + openrouterModel: env.OPENROUTER_MODEL || 'openrouter/free', + groqConfigured: grKeyConfigured, + groqModel: env.GROQ_MODEL || 'llama-3.3-70b-versatile', azureConfigured: !!(env.AZURE_SPEECH_KEY || '').trim() && !!(env.AZURE_SPEECH_REGION || '').trim(), whisperConfigured: !!(env.WHISPER_COMMAND || '').trim(), needsOnboarding: this.needsOnboarding() diff --git a/src/managers/session.manager.js b/src/managers/session.manager.js index 9c9d971e..8b122cd2 100644 --- a/src/managers/session.manager.js +++ b/src/managers/session.manager.js @@ -8,7 +8,7 @@ class SessionManager { this.compressionEnabled = true; this.maxSize = config.get('session.maxMemorySize'); this.compressionThreshold = config.get('session.compressionThreshold'); - this.currentSkill = 'dsa'; // Default skill is DSA + this.currentSkill = 'general'; this.isInitialized = false; this.initializeWithSkillPrompts(); diff --git a/src/managers/window.manager.js b/src/managers/window.manager.js index e51c623b..ed7fd59e 100644 --- a/src/managers/window.manager.js +++ b/src/managers/window.manager.js @@ -935,6 +935,9 @@ class WindowManager { const previousAvailability = this.screenCaptureStatus.available; const checkedAt = new Date().toISOString(); + // Wrap in try-catch because desktopCapturer.getSources() can crash + // the Electron helper process on macOS 14+ when screen recording + // permission hasn't been granted. try { await desktopCapturer.getSources({ types: ['screen', 'window'], @@ -1292,7 +1295,7 @@ class WindowManager { } const llmWindow = this.windows.get('llmResponse'); - if (llmWindow) { + if (llmWindow && !llmWindow.isDestroyed()) { logger.debug('Showing LLM loading state'); llmWindow.webContents.send('show-loading'); this.showOnCurrentDesktop(llmWindow); @@ -1303,6 +1306,9 @@ class WindowManager { } logger.debug('LLM loading window shown'); + } else if (llmWindow && llmWindow.isDestroyed()) { + logger.error('LLM response window was destroyed; removing from window map'); + this.windows.delete('llmResponse'); } else { logger.error('LLM window not available for loading state'); } diff --git a/src/services/capture.service.js b/src/services/capture.service.js index 8b057308..e7c7a7b7 100644 --- a/src/services/capture.service.js +++ b/src/services/capture.service.js @@ -1,4 +1,4 @@ -const { desktopCapturer, screen } = require('electron'); +const { desktopCapturer, screen, systemPreferences } = require('electron'); const logger = require('../core/logger').createServiceLogger('CAPTURE'); class CaptureService { @@ -68,10 +68,46 @@ class CaptureService { const targetDisplay = this._getTargetDisplay(options.displayId); const { width, height } = targetDisplay.size || { width: 1920, height: 1080 }; - const sources = await desktopCapturer.getSources({ - types: ['screen'], - thumbnailSize: { width, height } - }); + // On macOS 14+ (Sonoma), calling desktopCapturer.getSources() without + // screen recording permission can crash the Electron helper process. + // Check permission status first and give a clear error if denied. + if (process.platform === 'darwin' && systemPreferences.getMediaAccessStatus) { + try { + const status = systemPreferences.getMediaAccessStatus('screen'); + if (status === 'denied') { + throw new Error( + 'Screen recording permission denied. ' + + 'Go to System Settings → Privacy & Security → Screen Recording, ' + + 'find OpenCluely in the list and check the box, then restart the app.' + ); + } + } catch (e) { + // getMediaAccessStatus('screen') may throw on older macOS or Electron versions. + // In that case, proceed with the capture attempt — it may still work. + if (e.message && e.message.includes('Screen recording permission denied')) { + throw e; + } + logger.warn('Could not check screen recording permission', { error: e.message }); + } + } + + let sources; + try { + sources = await desktopCapturer.getSources({ + types: ['screen'], + thumbnailSize: { width, height } + }); + } catch (e) { + // desktopCapturer.getSources() can crash the app on macOS 14+ when + // permission hasn't been granted. If it throws instead of crashing, + // we catch it here. On some Electron versions the crash is unrecoverable + // (segfault in the GPU process), so we also check below whether the + // call left the app in a broken state. + throw new Error( + `Screen capture failed: ${e.message}. ` + + 'Make sure Screen Recording permission is granted in System Settings.' + ); + } if (sources.length === 0) { throw new Error('No screen sources available for capture'); diff --git a/src/services/llm.service.js b/src/services/llm.service.js index eb4eabc7..693cf489 100644 --- a/src/services/llm.service.js +++ b/src/services/llm.service.js @@ -10,13 +10,24 @@ class LLMService { this.isInitialized = false; this.requestCount = 0; this.errorCount = 0; + this.provider = 'gemini'; this.initializeClient(); } initializeClient() { + this.provider = (process.env.LLM_PROVIDER || 'gemini').toLowerCase(); + + if (this.provider === 'openrouter') { + return this._openrouterInit(); + } + + if (this.provider === 'groq') { + return this._groqInit(); + } + const apiKey = config.getApiKey('GEMINI'); - + if (!apiKey || apiKey === 'your-api-key-here') { logger.warn('Gemini API key not configured', { keyExists: !!apiKey, @@ -42,6 +53,674 @@ class LLMService { } } + _openrouterInit() { + const apiKey = process.env.OPENROUTER_API_KEY; + if (!apiKey || apiKey.trim().length === 0) { + logger.warn('OpenRouter API key not configured'); + return; + } + this.model = config.get('llm.openrouter.model') || 'openrouter/free'; + this.isInitialized = true; + logger.info('OpenRouter client initialized successfully', { + model: this.model + }); + } + + _openrouterIsAvailable() { + const key = process.env.OPENROUTER_API_KEY; + return !!key && key.trim().length > 0; + } + + _openrouterGetModel() { + return process.env.OPENROUTER_MODEL || config.get('llm.openrouter.model') || 'openrouter/free'; + } + + // ── Groq initialisation & helpers ───────────────────────────────── + + _groqInit() { + const apiKey = process.env.GROQ_API_KEY; + if (!apiKey || apiKey.trim().length === 0) { + logger.warn('Groq API key not configured'); + return; + } + this.model = config.get('llm.groq.model') || 'llama-3.3-70b-versatile'; + this.isInitialized = true; + logger.info('Groq client initialized successfully', { + model: this.model + }); + } + + _groqIsAvailable() { + const key = process.env.GROQ_API_KEY; + return !!key && key.trim().length > 0; + } + + _groqGetModel() { + return process.env.GROQ_MODEL || config.get('llm.groq.model') || 'llama-3.3-70b-versatile'; + } + + // ── Shared OpenAI-compatible HTTPS executor ─────────────────────── + // Used by both OpenRouter and Groq (and any future OpenAI-compatible provider). + + async _executeChatCompletion({ apiKey, baseUrl, model, messages, extraHeaders = {}, overrides = {} }) { + const https = require('https'); + if (!apiKey) { + throw new Error('API key not configured'); + } + + const timeout = overrides.timeout || 30000; + const genConfig = { ...overrides.generation }; + + const url = `${baseUrl}/chat/completions`; + const body = { + model, + messages, + temperature: genConfig.temperature, + max_tokens: genConfig.maxOutputTokens + }; + if (genConfig.topP !== undefined) body.top_p = genConfig.topP; + + const postData = JSON.stringify(body); + const maxRetries = overrides.maxRetries || 1; + + let lastError; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + const options = { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${apiKey}`, + 'Content-Length': Buffer.byteLength(postData), + 'User-Agent': this.getUserAgent(), + ...extraHeaders + }, + timeout + }; + + const response = await new Promise((resolve, reject) => { + const req = https.request(url, options, (res) => { + let data = ''; + res.on('data', (chunk) => { data += chunk; }); + res.on('end', () => { + if (res.statusCode === 200) { + try { + resolve({ status: res.statusCode, data: JSON.parse(data) }); + } catch (e) { + reject(new Error(`Failed to parse response: ${e.message}`)); + } + } else { + reject(new Error(`HTTP ${res.statusCode}: ${data.substring(0, 500)}`)); + } + }); + }); + req.on('error', (e) => reject(new Error(`Request failed: ${e.message}`))); + req.on('timeout', () => { req.destroy(); reject(new Error('Request timeout')); }); + req.write(postData); + req.end(); + }); + + const text = this._openrouterExtractResponse(response.data); + return text; + + } catch (error) { + lastError = error; + if (attempt < maxRetries) { + const delay = 1500 * attempt + Math.random() * 1000; + await this.delay(delay); + } + } + } + + throw lastError || new Error('Request failed after all retries'); + } + + _openrouterBuildMessages({ systemInstruction, contents }) { + const messages = []; + if (systemInstruction) { + const text = typeof systemInstruction === 'string' + ? systemInstruction + : systemInstruction.parts?.[0]?.text || ''; + if (text) { + messages.push({ role: 'system', content: text }); + } + } + if (contents && Array.isArray(contents)) { + for (const entry of contents) { + const role = entry.role === 'model' ? 'assistant' : 'user'; + const parts = entry.parts || []; + const contentParts = []; + for (const part of parts) { + if (part.text !== undefined) { + contentParts.push({ type: 'text', text: part.text }); + } else if (part.inlineData) { + const { data, mimeType } = part.inlineData; + contentParts.push({ + type: 'image_url', + image_url: { url: `data:${mimeType || 'image/png'};base64,${data}` } + }); + } + } + if (contentParts.length === 1 && contentParts[0].type === 'text') { + messages.push({ role, content: contentParts[0].text }); + } else if (contentParts.length > 0) { + messages.push({ role, content: contentParts }); + } + } + } + return messages; + } + + _openrouterBuildRequestBody(text, activeSkill, programmingLanguage, systemPrompt, conversationHistory) { + const messages = []; + if (systemPrompt) { + messages.push({ role: 'system', content: systemPrompt }); + } + if (conversationHistory && conversationHistory.length > 0) { + for (const event of conversationHistory) { + if (event.role === 'system') continue; + if (!event.content || typeof event.content !== 'string') continue; + const role = event.role === 'model' ? 'assistant' : 'user'; + messages.push({ role, content: event.content.trim() }); + } + } + const formattedMessage = conversationHistory && conversationHistory.length > 0 + ? text + : this.formatUserMessage(text, activeSkill); + messages.push({ role: 'user', content: formattedMessage }); + return messages; + } + + _openrouterBuildImageBody(imageBuffer, mimeType, activeSkill, programmingLanguage, skillPrompt) { + const messages = []; + if (skillPrompt) { + messages.push({ role: 'system', content: skillPrompt }); + } + const base64 = imageBuffer.toString('base64'); + messages.push({ + role: 'user', + content: [ + { type: 'text', text: this.formatImageInstruction(activeSkill, programmingLanguage) }, + { type: 'image_url', image_url: { url: `data:${mimeType || 'image/png'};base64,${base64}` } } + ] + }); + return messages; + } + + _openrouterBuildTranscriptionBody(text, activeSkill, programmingLanguage, conversationHistory) { + const messages = []; + const systemPrompt = this.getIntelligentTranscriptionPrompt(activeSkill, programmingLanguage); + if (systemPrompt) { + messages.push({ role: 'system', content: systemPrompt }); + } + if (conversationHistory && conversationHistory.length > 0) { + for (const event of conversationHistory) { + if (event.role === 'system') continue; + if (!event.content || typeof event.content !== 'string') continue; + const role = event.role === 'model' ? 'assistant' : 'user'; + messages.push({ role, content: event.content.trim() }); + } + } + messages.push({ role: 'user', content: text.trim() }); + return messages; + } + + async _openrouterExecute(messages, overrides = {}) { + const apiKey = process.env.OPENROUTER_API_KEY; + if (!apiKey) throw new Error('OpenRouter API key not configured'); + const baseUrl = config.get('llm.openrouter.baseUrl') || 'https://openrouter.ai/api/v1'; + const model = overrides.model || this._openrouterGetModel(); + const timeout = overrides.timeout || config.get('llm.openrouter.timeout') || 30000; + const maxRetries = overrides.maxRetries || config.get('llm.openrouter.maxRetries') || 1; + const generation = { ...config.get('llm.openrouter.generation'), ...overrides.generation }; + + return this._executeChatCompletion({ + apiKey, + baseUrl, + model, + messages, + extraHeaders: { + 'HTTP-Referer': 'https://opencluely.app', + 'X-Title': 'OpenCluely', + }, + overrides: { timeout, maxRetries, generation } + }); + } + + async _groqExecute(messages, overrides = {}) { + const apiKey = process.env.GROQ_API_KEY; + if (!apiKey) throw new Error('Groq API key not configured'); + const baseUrl = config.get('llm.groq.baseUrl') || 'https://api.groq.com/openai/v1'; + const model = overrides.model || this._groqGetModel(); + const timeout = overrides.timeout || config.get('llm.groq.timeout') || 30000; + const maxRetries = overrides.maxRetries || config.get('llm.groq.maxRetries') || 1; + const generation = { ...config.get('llm.groq.generation'), ...overrides.generation }; + + return this._executeChatCompletion({ + apiKey, + baseUrl, + model, + messages, + overrides: { timeout, maxRetries, generation } + }); + } + + _openrouterExtractResponse(response) { + if (!response) { + throw new Error('Empty response from OpenRouter'); + } + if (!response.choices || !Array.isArray(response.choices) || response.choices.length === 0) { + throw new Error('No choices in OpenRouter response'); + } + const choice = response.choices[0]; + if (!choice.message || typeof choice.message.content !== 'string') { + throw new Error('No message content in OpenRouter response choice'); + } + if (choice.finish_reason === 'length') { + logger.warn('OpenRouter response reached max tokens limit', { + finishReason: choice.finish_reason + }); + } + return choice.message.content.trim(); + } + + async _openrouterProcessImage(imageBuffer, mimeType, activeSkill, sessionMemory, programmingLanguage) { + const startTime = Date.now(); + this.requestCount++; + + try { + const { promptLoader } = require('../../prompt-loader'); + const skillPrompt = promptLoader.getSkillPrompt(activeSkill, programmingLanguage) || ''; + const messages = this._openrouterBuildImageBody(imageBuffer, mimeType, activeSkill, programmingLanguage, skillPrompt); + + const responseText = await this._openrouterExecute(messages); + + const finalResponse = programmingLanguage + ? this.enforceProgrammingLanguage(responseText, programmingLanguage) + : responseText; + + logger.logPerformance('OpenRouter image processing', startTime, { + activeSkill, + imageSize: imageBuffer.length, + responseLength: finalResponse.length, + programmingLanguage: programmingLanguage || 'not specified', + requestId: this.requestCount + }); + + return { + response: finalResponse, + metadata: { + provider: 'openrouter', + model: this._openrouterGetModel(), + skill: activeSkill, + programmingLanguage, + processingTime: Date.now() - startTime, + requestId: this.requestCount, + usedFallback: false, + isImageAnalysis: true, + mimeType + } + }; + } catch (error) { + this.errorCount++; + logger.error('OpenRouter image processing failed', { + error: error.message, + activeSkill, + requestId: this.requestCount + }); + return this.generateFallbackResponse('[image]', activeSkill); + } + } + + async _openrouterProcessText(text, activeSkill, sessionMemory, programmingLanguage) { + const startTime = Date.now(); + this.requestCount++; + + try { + const { promptLoader } = require('../../prompt-loader'); + const skillPrompt = promptLoader.getSkillPrompt(activeSkill, programmingLanguage) || ''; + const sessionManager = require('../managers/session.manager'); + const conversationHistory = sessionManager && typeof sessionManager.getConversationHistory === 'function' + ? sessionManager.getConversationHistory(15) + : []; + + const messages = this._openrouterBuildRequestBody(text, activeSkill, programmingLanguage, skillPrompt, conversationHistory); + + const responseText = await this._openrouterExecute(messages); + + const finalResponse = programmingLanguage + ? this.enforceProgrammingLanguage(responseText, programmingLanguage) + : responseText; + + logger.logPerformance('OpenRouter text processing', startTime, { + activeSkill, + textLength: text.length, + responseLength: finalResponse.length, + programmingLanguage: programmingLanguage || 'not specified', + requestId: this.requestCount + }); + + return { + response: finalResponse, + metadata: { + provider: 'openrouter', + model: this._openrouterGetModel(), + skill: activeSkill, + programmingLanguage, + processingTime: Date.now() - startTime, + requestId: this.requestCount, + usedFallback: false + } + }; + } catch (error) { + this.errorCount++; + logger.error('OpenRouter text processing failed', { + error: error.message, + activeSkill, + requestId: this.requestCount + }); + return this.generateFallbackResponse(text, activeSkill); + } + } + + async _openrouterProcessTranscription(text, activeSkill, sessionMemory, programmingLanguage) { + if (!text || typeof text !== 'string' || text.trim().length < 2) { + logger.warn('Skipping transcription for empty or very short input'); + return { + response: '', + metadata: { provider: 'openrouter', skill: activeSkill, processingTime: 0, usedFallback: true, isTranscriptionResponse: true } + }; + } + + const startTime = Date.now(); + this.requestCount++; + + try { + const sessionManager = require('../managers/session.manager'); + const conversationHistory = sessionManager && typeof sessionManager.getConversationHistory === 'function' + ? sessionManager.getConversationHistory(10) + : []; + + const messages = this._openrouterBuildTranscriptionBody(text.trim(), activeSkill, programmingLanguage, conversationHistory); + + const responseText = await this._openrouterExecute(messages); + + const finalResponse = programmingLanguage + ? this.enforceProgrammingLanguage(responseText, programmingLanguage) + : responseText; + + logger.logPerformance('OpenRouter transcription processing', startTime, { + activeSkill, + textLength: text.length, + responseLength: finalResponse.length, + requestId: this.requestCount + }); + + return { + response: finalResponse, + metadata: { + provider: 'openrouter', + model: this._openrouterGetModel(), + skill: activeSkill, + programmingLanguage, + processingTime: Date.now() - startTime, + requestId: this.requestCount, + usedFallback: false, + isTranscriptionResponse: true + } + }; + } catch (error) { + this.errorCount++; + logger.error('OpenRouter transcription processing failed', { + error: error.message, + activeSkill, + requestId: this.requestCount + }); + return this.generateIntelligentFallbackResponse(text, activeSkill); + } + } + + // ── Groq process methods (OpenAI-compatible, reuse build helpers) ─ + + async _groqProcessImage(imageBuffer, mimeType, activeSkill, sessionMemory, programmingLanguage) { + const startTime = Date.now(); + this.requestCount++; + + try { + const { promptLoader } = require('../../prompt-loader'); + const skillPrompt = promptLoader.getSkillPrompt(activeSkill, programmingLanguage) || ''; + const messages = this._openrouterBuildImageBody(imageBuffer, mimeType, activeSkill, programmingLanguage, skillPrompt); + + const responseText = await this._groqExecute(messages); + + const finalResponse = programmingLanguage + ? this.enforceProgrammingLanguage(responseText, programmingLanguage) + : responseText; + + logger.logPerformance('Groq image processing', startTime, { + activeSkill, + imageSize: imageBuffer.length, + responseLength: finalResponse.length, + programmingLanguage: programmingLanguage || 'not specified', + requestId: this.requestCount + }); + + return { + response: finalResponse, + metadata: { + provider: 'groq', + model: this._groqGetModel(), + skill: activeSkill, + programmingLanguage, + processingTime: Date.now() - startTime, + requestId: this.requestCount, + usedFallback: false, + isImageAnalysis: true, + mimeType + } + }; + } catch (error) { + this.errorCount++; + logger.error('Groq image processing failed', { + error: error.message, + activeSkill, + requestId: this.requestCount + }); + return this.generateFallbackResponse('[image]', activeSkill); + } + } + + async _groqProcessText(text, activeSkill, sessionMemory, programmingLanguage) { + const startTime = Date.now(); + this.requestCount++; + + try { + const { promptLoader } = require('../../prompt-loader'); + const skillPrompt = promptLoader.getSkillPrompt(activeSkill, programmingLanguage) || ''; + const sessionManager = require('../managers/session.manager'); + const conversationHistory = sessionManager && typeof sessionManager.getConversationHistory === 'function' + ? sessionManager.getConversationHistory(15) + : []; + + const messages = this._openrouterBuildRequestBody(text, activeSkill, programmingLanguage, skillPrompt, conversationHistory); + + const responseText = await this._groqExecute(messages); + + const finalResponse = programmingLanguage + ? this.enforceProgrammingLanguage(responseText, programmingLanguage) + : responseText; + + logger.logPerformance('Groq text processing', startTime, { + activeSkill, + textLength: text.length, + responseLength: finalResponse.length, + programmingLanguage: programmingLanguage || 'not specified', + requestId: this.requestCount + }); + + return { + response: finalResponse, + metadata: { + provider: 'groq', + model: this._groqGetModel(), + skill: activeSkill, + programmingLanguage, + processingTime: Date.now() - startTime, + requestId: this.requestCount, + usedFallback: false + } + }; + } catch (error) { + this.errorCount++; + logger.error('Groq text processing failed', { + error: error.message, + activeSkill, + requestId: this.requestCount + }); + return this.generateFallbackResponse(text, activeSkill); + } + } + + async _groqProcessTranscription(text, activeSkill, sessionMemory, programmingLanguage) { + if (!text || typeof text !== 'string' || text.trim().length < 2) { + logger.warn('Skipping transcription for empty or very short input'); + return { + response: '', + metadata: { provider: 'groq', skill: activeSkill, processingTime: 0, usedFallback: true, isTranscriptionResponse: true } + }; + } + + const startTime = Date.now(); + this.requestCount++; + + try { + const sessionManager = require('../managers/session.manager'); + const conversationHistory = sessionManager && typeof sessionManager.getConversationHistory === 'function' + ? sessionManager.getConversationHistory(10) + : []; + + const messages = this._openrouterBuildTranscriptionBody(text.trim(), activeSkill, programmingLanguage, conversationHistory); + + const responseText = await this._groqExecute(messages); + + const finalResponse = programmingLanguage + ? this.enforceProgrammingLanguage(responseText, programmingLanguage) + : responseText; + + logger.logPerformance('Groq transcription processing', startTime, { + activeSkill, + textLength: text.length, + responseLength: finalResponse.length, + requestId: this.requestCount + }); + + return { + response: finalResponse, + metadata: { + provider: 'groq', + model: this._groqGetModel(), + skill: activeSkill, + programmingLanguage, + processingTime: Date.now() - startTime, + requestId: this.requestCount, + usedFallback: false, + isTranscriptionResponse: true + } + }; + } catch (error) { + this.errorCount++; + logger.error('Groq transcription processing failed', { + error: error.message, + activeSkill, + requestId: this.requestCount + }); + return this.generateIntelligentFallbackResponse(text, activeSkill); + } + } + + async _groqTestConnection() { + try { + const messages = [{ role: 'user', content: 'Test connection. Please respond with "OK".' }]; + const startTime = Date.now(); + const text = await this._groqExecute(messages, { maxRetries: 1, generation: { temperature: 0, maxOutputTokens: 64 } }); + const latency = Date.now() - startTime; + + logger.info('Groq connection test successful', { + response: text, + latency, + model: this._groqGetModel() + }); + + return { + success: true, + response: text, + latency, + model: this._groqGetModel() + }; + } catch (error) { + const errMsg = error.message || ''; + let friendlyError; + if (errMsg.includes('401') || errMsg.includes('unauthorized')) { + friendlyError = 'Invalid Groq API key. Get one at console.groq.com/keys.'; + } else if (errMsg.includes('429')) { + friendlyError = 'Groq rate limit exceeded. Wait a moment and try again.'; + } else if (errMsg.includes('timeout') || errMsg.includes('ENOTFOUND') || errMsg.includes('ECONNREFUSED')) { + friendlyError = 'Cannot reach Groq servers. Check your internet connection.'; + } else { + friendlyError = `Groq error: ${errMsg.substring(0, 200)}`; + } + + return { + success: false, + error: friendlyError, + errorType: 'GROQ_ERROR' + }; + } + } + + async _openrouterTestConnection() { + try { + const messages = [{ role: 'user', content: 'Test connection. Please respond with "OK".' }]; + const startTime = Date.now(); + const text = await this._openrouterExecute(messages, { maxRetries: 1, generation: { temperature: 0, maxOutputTokens: 64 } }); + const latency = Date.now() - startTime; + + logger.info('OpenRouter connection test successful', { + response: text, + latency, + model: this._openrouterGetModel() + }); + + return { + success: true, + response: text, + latency, + model: this._openrouterGetModel() + }; + } catch (error) { + const errMsg = error.message || ''; + let friendlyError; + if (errMsg.includes('401') || errMsg.includes('unauthorized')) { + friendlyError = 'Invalid OpenRouter API key. Get one at openrouter.ai/keys.'; + } else if (errMsg.includes('402')) { + friendlyError = 'OpenRouter account needs credits or free model is unavailable.'; + } else if (errMsg.includes('429')) { + friendlyError = 'OpenRouter rate limit exceeded. Wait a moment and try again.'; + } else if (errMsg.includes('timeout') || errMsg.includes('ENOTFOUND') || errMsg.includes('ECONNREFUSED')) { + friendlyError = 'Cannot reach OpenRouter servers. Check your internet connection.'; + } else { + friendlyError = `OpenRouter error: ${errMsg.substring(0, 200)}`; + } + + return { + success: false, + error: friendlyError, + errorType: 'OPENROUTER_ERROR' + }; + } + } + getGenerationConfig(overrides = {}) { const defaults = config.get('llm.gemini.generation') || {}; const fallback = { @@ -115,20 +794,34 @@ class LLMService { * For image-based queries, we include the skill prompt (e.g., DSA) as systemInstruction. * @param {Buffer} imageBuffer - PNG/JPEG image bytes * @param {string} mimeType - e.g., 'image/png' or 'image/jpeg' - * @param {string} activeSkill - current skill (e.g. 'dsa') + * @param {string} activeSkill - current skill (e.g. 'coding') * @param {Array} sessionMemory - optional (not required for image) * @param {string|null} programmingLanguage - optional language context for skills that need it * @returns {Promise<{response: string, metadata: object}>} */ async processImageWithSkill(imageBuffer, mimeType, activeSkill, sessionMemory = [], programmingLanguage = null) { if (!this.isInitialized) { - throw new Error('LLM service not initialized. Check Gemini API key configuration.'); + throw new Error('LLM service not initialized. Check API key configuration.'); } if (!imageBuffer || !Buffer.isBuffer(imageBuffer)) { throw new Error('Invalid image buffer provided to processImageWithSkill'); } + if (this.provider === 'openrouter') { + // OpenRouter path – wrap with overall timeout to avoid indefinite hangs + const timeoutMs = (config.get('llm.openrouter.timeout') || 30000) * 2; + const openRouterPromise = this._openrouterProcessImage(imageBuffer, mimeType, activeSkill, sessionMemory, programmingLanguage); + const timeoutPromise = new Promise((_, reject) => + setTimeout(() => reject(new Error('OpenRouter image processing timed out after ' + timeoutMs + 'ms')), timeoutMs) + ); + return await Promise.race([openRouterPromise, timeoutPromise]); + } + + if (this.provider === 'groq') { + return this._groqProcessImage(imageBuffer, mimeType, activeSkill, sessionMemory, programmingLanguage); + } + const startTime = Date.now(); this.requestCount++; @@ -158,30 +851,13 @@ class LLMService { request.systemInstruction = { parts: [{ text: skillPrompt }] }; } - // Execute with retries/timeout - try alternative method first for network reliability + // Race SDK and HTTPS methods in parallel — faster one wins let responseText; - const preferAlternative = !!config.get('llm.gemini.enableFallbackMethod'); try { - if (preferAlternative) { - logger.debug('Attempting alternative HTTPS method first for reliability'); - responseText = await this.executeAlternativeRequest(request); - } else { - responseText = await this.executeRequest(request); - } + responseText = await this._raceGeminiMethods(request); } catch (error) { - const secondaryLabel = preferAlternative ? 'primary SDK method' : 'alternative HTTPS method'; - logger.warn(`${preferAlternative ? 'Alternative' : 'Primary'} method failed, trying ${secondaryLabel}`, { error: error.message }); - const secondaryFn = preferAlternative ? this.executeRequest.bind(this) : this.executeAlternativeRequest.bind(this); - - try { - responseText = await secondaryFn(request); - } catch (secondaryError) { - logger.error('Both Gemini request methods failed', { - firstError: error.message, - secondError: secondaryError.message - }); - throw secondaryError; - } + logger.error('Both Gemini request methods failed', { error: error.message }); + throw error; } // Enforce language in code fences if provided @@ -226,12 +902,23 @@ class LLMService { formatImageInstruction(activeSkill, programmingLanguage) { const langNote = programmingLanguage ? ` Use only ${programmingLanguage.toUpperCase()} for any code.` : ''; - return `Analyze this image for a ${activeSkill.toUpperCase()} question. Extract the problem concisely and provide the best possible solution with explanation and final code.${langNote}`; + const modePrefix = activeSkill === 'meeting' + ? 'Analyze this image from a meeting or presentation.' + : `Analyze this image for a ${activeSkill.toUpperCase()} question.`; + return `${modePrefix} Extract the problem concisely and provide the best possible solution with explanation and final code.${langNote}`; } async processTextWithSkill(text, activeSkill, sessionMemory = [], programmingLanguage = null) { if (!this.isInitialized) { - throw new Error('LLM service not initialized. Check Gemini API key configuration.'); + throw new Error('LLM service not initialized. Check API key configuration.'); + } + + if (this.provider === 'openrouter') { + return this._openrouterProcessText(text, activeSkill, sessionMemory, programmingLanguage); + } + + if (this.provider === 'groq') { + return this._groqProcessText(text, activeSkill, sessionMemory, programmingLanguage); } const startTime = Date.now(); @@ -248,32 +935,15 @@ class LLMService { const geminiRequest = this.buildGeminiRequest(text, activeSkill, sessionMemory, programmingLanguage); - const preferAlternative = !!config.get('llm.gemini.enableFallbackMethod'); let response; try { - if (preferAlternative) { - logger.debug('Attempting alternative HTTPS method first for text processing'); - response = await this.executeAlternativeRequest(geminiRequest); - } else { - response = await this.executeRequest(geminiRequest); - } + response = await this._raceGeminiMethods(geminiRequest); } catch (error) { - const secondaryLabel = preferAlternative ? 'primary SDK method' : 'alternative HTTPS method'; - logger.warn(`${preferAlternative ? 'Alternative' : 'Primary'} method failed, trying ${secondaryLabel}`, { + logger.error('Both Gemini request methods failed for text processing', { error: error.message, requestId: this.requestCount }); - const secondaryFn = preferAlternative ? this.executeRequest.bind(this) : this.executeAlternativeRequest.bind(this); - try { - response = await secondaryFn(geminiRequest); - } catch (secondaryError) { - logger.error('Both Gemini request methods failed for text processing', { - firstError: error.message, - secondError: secondaryError.message, - requestId: this.requestCount - }); - throw secondaryError; - } + throw error; } // Enforce language in code fences if programmingLanguage specified @@ -318,7 +988,15 @@ class LLMService { async processTranscriptionWithIntelligentResponse(text, activeSkill, sessionMemory = [], programmingLanguage = null) { if (!this.isInitialized) { - throw new Error('LLM service not initialized. Check Gemini API key configuration.'); + throw new Error('LLM service not initialized. Check API key configuration.'); + } + + if (this.provider === 'openrouter') { + return this._openrouterProcessTranscription(text, activeSkill, sessionMemory, programmingLanguage); + } + + if (this.provider === 'groq') { + return this._groqProcessTranscription(text, activeSkill, sessionMemory, programmingLanguage); } const startTime = Date.now(); @@ -335,32 +1013,15 @@ class LLMService { const geminiRequest = this.buildIntelligentTranscriptionRequest(text, activeSkill, sessionMemory, programmingLanguage); - const preferAlternative = !!config.get('llm.gemini.enableFallbackMethod'); let response; try { - if (preferAlternative) { - logger.debug('Attempting alternative HTTPS method first for transcription processing'); - response = await this.executeAlternativeRequest(geminiRequest); - } else { - response = await this.executeRequest(geminiRequest); - } + response = await this._raceGeminiMethods(geminiRequest); } catch (error) { - const secondaryLabel = preferAlternative ? 'primary SDK method' : 'alternative HTTPS method'; - logger.warn(`${preferAlternative ? 'Alternative' : 'Primary'} method failed, trying ${secondaryLabel}`, { + logger.error('Both Gemini request methods failed for transcription processing', { error: error.message, requestId: this.requestCount }); - const secondaryFn = preferAlternative ? this.executeRequest.bind(this) : this.executeAlternativeRequest.bind(this); - try { - response = await secondaryFn(geminiRequest); - } catch (secondaryError) { - logger.error('Both Gemini request methods failed for transcription processing', { - firstError: error.message, - secondError: secondaryError.message, - requestId: this.requestCount - }); - throw secondaryError; - } + throw error; } // Enforce language in code fences if programmingLanguage specified @@ -678,6 +1339,12 @@ Always respond to the point, do not repeat the question or unnecessary informati ## Response Rules: +### If in MEETING mode: +- Detect if the speaker is asking a question +- If a question is detected, provide a concise, direct answer +- If no question is detected, respond with a brief acknowledgment or stay silent +- Do NOT respond to statements, chit-chat, or agenda items — only questions + ### If the transcription is casual conversation, greetings, or NOT related to ${activeSkill}: - Respond with: "Yeah, I'm listening. Ask your question relevant to ${activeSkill}." - Or similar brief acknowledgments like: "I'm here, what's your ${activeSkill} question?" @@ -706,7 +1373,7 @@ Always respond to the point, do not repeat the question or unnecessary informati - Be encouraging and helpful - Stay focused on ${activeSkill} -If the user's input is a coding or DSA problem statement and contains no code, produce a complete, runnable solution in the selected programming language without asking for more details. Always include the final implementation in a properly tagged code block. +If the user's input is a coding problem statement and contains no code, produce a complete, runnable solution in the selected programming language without asking for more details. Always include the final implementation in a properly tagged code block. Remember: Be intelligent about filtering - only provide detailed responses when the user actually needs help with ${activeSkill}.`; @@ -714,6 +1381,9 @@ Remember: Be intelligent about filtering - only provide detailed responses when } formatUserMessage(text, activeSkill) { + if (activeSkill === 'meeting') { + return `Context: Meeting conversation\n\nDetected speech:\n${text}`; + } return `Context: ${activeSkill.toUpperCase()} analysis request\n\nText to analyze:\n${text}`; } @@ -992,10 +1662,10 @@ Remember: Be intelligent about filtering - only provide detailed responses when logger.info('Generating fallback response', { activeSkill }); const fallbackResponses = { - 'dsa': 'This appears to be a data structures and algorithms problem. Consider breaking it down into smaller components and identifying the appropriate algorithm or data structure to use.', - 'system-design': 'For this system design question, consider scalability, reliability, and the trade-offs between different architectural approaches.', - 'programming': 'This looks like a programming challenge. Focus on understanding the requirements, edge cases, and optimal time/space complexity.', - 'default': 'I can help analyze this content. Please ensure your Gemini API key is properly configured for detailed analysis.' + 'general': 'I can help analyze this content. Here are my thoughts on the matter.', + 'coding': 'This looks like a coding challenge. Focus on understanding the requirements, edge cases, and optimal time/space complexity.', + 'meeting': 'I heard a question in the meeting. Here is my response based on the context.', + 'default': 'I can help analyze this content. Please ensure your API key is properly configured for detailed analysis.' }; const response = fallbackResponses[activeSkill] || fallbackResponses.default; @@ -1016,15 +1686,9 @@ Remember: Be intelligent about filtering - only provide detailed responses when // Simple heuristic to determine if message seems skill-related const skillKeywords = { - 'dsa': ['algorithm', 'data structure', 'array', 'tree', 'graph', 'sort', 'search', 'complexity', 'big o'], - 'programming': ['code', 'function', 'variable', 'class', 'method', 'bug', 'debug', 'syntax'], - 'system-design': ['scalability', 'database', 'architecture', 'microservice', 'load balancer', 'cache'], - 'behavioral': ['interview', 'experience', 'situation', 'leadership', 'conflict', 'team'], - 'sales': ['customer', 'deal', 'negotiation', 'price', 'revenue', 'prospect'], - 'presentation': ['slide', 'audience', 'public speaking', 'presentation', 'nervous'], - 'data-science': ['data', 'model', 'machine learning', 'statistics', 'analytics', 'python', 'pandas'], - 'devops': ['deployment', 'ci/cd', 'docker', 'kubernetes', 'infrastructure', 'monitoring'], - 'negotiation': ['negotiate', 'compromise', 'agreement', 'terms', 'conflict resolution'] + 'general': ['help', 'question', 'explain', 'what', 'how', 'why', 'tell me', 'can you'], + 'coding': ['code', 'function', 'variable', 'class', 'method', 'bug', 'debug', 'syntax', 'algorithm', 'array', 'sort'], + 'meeting': ['question', 'how', 'what', 'why', 'could you', 'can you', 'suggestion', 'recommend'] }; const textLower = text.toLowerCase(); @@ -1059,6 +1723,14 @@ Remember: Be intelligent about filtering - only provide detailed responses when return { success: false, error: 'Service not initialized' }; } + if (this.provider === 'openrouter') { + return this._openrouterTestConnection(); + } + + if (this.provider === 'groq') { + return this._groqTestConnection(); + } + try { // First check network connectivity const networkCheck = await this.checkNetworkConnectivity(); @@ -1175,19 +1847,49 @@ Remember: Be intelligent about filtering - only provide detailed responses when } updateApiKey(newApiKey) { - process.env.GEMINI_API_KEY = newApiKey; + if (this.provider === 'openrouter') { + process.env.OPENROUTER_API_KEY = newApiKey; + } else if (this.provider === 'groq') { + process.env.GROQ_API_KEY = newApiKey; + } else { + process.env.GEMINI_API_KEY = newApiKey; + } this.isInitialized = false; this.initializeClient(); - logger.info('API key updated and client reinitialized'); + logger.info('API key updated and client reinitialized', { provider: this.provider }); } getStats() { + if (this.provider === 'openrouter') { + return { + provider: 'openrouter', + isInitialized: this.isInitialized, + requestCount: this.requestCount, + errorCount: this.errorCount, + successRate: this.requestCount > 0 ? ((this.requestCount - this.errorCount) / this.requestCount) * 100 : 0, + model: this._openrouterGetModel(), + config: config.get('llm.openrouter') + }; + } + if (this.provider === 'groq') { + return { + provider: 'groq', + isInitialized: this.isInitialized, + requestCount: this.requestCount, + errorCount: this.errorCount, + successRate: this.requestCount > 0 ? ((this.requestCount - this.errorCount) / this.requestCount) * 100 : 0, + model: this._groqGetModel(), + config: config.get('llm.groq') + }; + } return { + provider: 'gemini', isInitialized: this.isInitialized, requestCount: this.requestCount, errorCount: this.errorCount, successRate: this.requestCount > 0 ? ((this.requestCount - this.errorCount) / this.requestCount) * 100 : 0, + model: this.model, config: config.get('llm.gemini') }; } @@ -1196,6 +1898,28 @@ Remember: Be intelligent about filtering - only provide detailed responses when return new Promise(resolve => setTimeout(resolve, ms)); } + /** + * Race the SDK and HTTPS methods in parallel. The first to succeed + * wins. If both fail, throw the error from the primary (SDK) method. + */ + async _raceGeminiMethods(request) { + // Use a generous overall timeout (2 minutes) to guarantee the race resolves/rejects + const overallTimeout = new Promise((_, reject) => + setTimeout(() => reject(new Error('Gemini race timed out after 120000 ms')), 120000) + ); + try { + return await Promise.race([ + Promise.any([ + this.executeRequest(request), + this.executeAlternativeRequest(request) + ]), + overallTimeout + ]); + } catch (err) { + throw err.errors?.[0] || err || new Error('Both Gemini request methods failed'); + } + } + async executeAlternativeRequest(geminiRequest) { const https = require('https'); const apiKey = config.getApiKey('GEMINI'); @@ -1238,7 +1962,7 @@ Remember: Be intelligent about filtering - only provide detailed responses when const postData = JSON.stringify(geminiRequest); - const agent = new https.Agent({ keepAlive: true, maxSockets: 1 }); + const agent = new https.Agent({ keepAlive: true, maxSockets: 4 }); const options = { method: 'POST', diff --git a/src/services/speech.service.js b/src/services/speech.service.js index b0c24f3e..633ccbde 100644 --- a/src/services/speech.service.js +++ b/src/services/speech.service.js @@ -394,13 +394,6 @@ try { logger.warn('Azure Speech SDK unavailable', { error: error.message }); } -let recorder = null; -try { - recorder = require('node-record-lpcm16'); -} catch (error) { - logger.warn('Local audio recorder dependency unavailable', { error: error.message }); -} - class SpeechService extends EventEmitter { constructor() { super(); @@ -423,6 +416,8 @@ class SpeechService extends EventEmitter { this.pendingFlush = false; this.audioProgram = null; this.whisperCommand = null; + this._audioCaptureWindow = null; + this.groqSttModel = null; this.initializeClient(); } @@ -433,8 +428,10 @@ class SpeechService extends EventEmitter { this.available = false; this.speechConfig = null; this.whisperCommand = null; + this.groqSttModel = null; const provider = this._getConfiguredProvider(); + console.log('[SpeechService] initializeClient() provider:', provider); this.provider = provider; if (provider === 'azure') { @@ -447,7 +444,12 @@ class SpeechService extends EventEmitter { return; } - const reason = 'Speech recognition disabled. Configure Azure or local Whisper.'; + if (provider === 'groq') { + this._initializeGroqClient(); + return; + } + + const reason = 'Speech recognition disabled. Configure Azure, Groq, or local Whisper.'; logger.warn(reason); this.emit('status', reason); } @@ -458,10 +460,6 @@ class SpeechService extends EventEmitter { throw new Error('Azure Speech SDK dependency is not installed'); } - if (!recorder || typeof recorder.record !== 'function') { - throw new Error('Local microphone recorder dependency is not installed'); - } - const subscriptionKey = this._getSetting('azureKey') || process.env.AZURE_SPEECH_KEY; const region = this._getSetting('azureRegion') || process.env.AZURE_SPEECH_REGION; @@ -507,10 +505,6 @@ class SpeechService extends EventEmitter { _initializeWhisperClient() { try { - if (!recorder || typeof recorder.record !== 'function') { - throw new Error('Local microphone recorder dependency is not installed'); - } - this.whisperCommand = this._resolveWhisperCommand(); if (!this.whisperCommand) { const reason = 'Local Whisper unavailable. Install the Whisper CLI or set WHISPER_COMMAND.'; @@ -536,7 +530,36 @@ class SpeechService extends EventEmitter { } } + _initializeGroqClient() { + try { + const apiKey = process.env.GROQ_API_KEY; + + if (!apiKey) { + const reason = 'Groq STT unavailable. Set GROQ_API_KEY to use Groq speech.'; + logger.warn(reason); + this.emit('status', reason); + return; + } + + this.groqSttModel = this._getSetting('groqSttModel') || process.env.GROQ_STT_MODEL || config.get('speech.groq.sttModel') || 'whisper-large-v3-turbo'; + + this.available = true; + logger.info('Groq STT service initialized successfully', { + model: this.groqSttModel + }); + this.emit('status', 'Groq STT ready'); + } catch (error) { + logger.error('Failed to initialize Groq STT client', { + error: error.message, + stack: error.stack + }); + this.available = false; + this.emit('status', 'Groq STT unavailable'); + } + } + startRecording() { + console.log('[SpeechService] startRecording() called, provider:', this.provider, 'available:', this.available, 'isRecording:', this.isRecording); try { if (!this.available) { const errorMsg = `Speech provider "${this.provider}" is not available`; @@ -563,6 +586,11 @@ class SpeechService extends EventEmitter { return; } + if (this.provider === 'groq') { + this._startGroqRecording(); + return; + } + throw new Error(`Unsupported speech provider: ${this.provider}`); } catch (error) { logger.error('Critical error in startRecording', { error: error.message, stack: error.stack }); @@ -671,6 +699,7 @@ class SpeechService extends EventEmitter { } _startWhisperRecording() { + console.log('[SpeechService] _startWhisperRecording() called'); this._cleanup(); this.isRecording = true; this.segmentBuffers = []; @@ -728,6 +757,11 @@ class SpeechService extends EventEmitter { return; } + if (this.provider === 'groq') { + this._finalizeGroqStop(); + return; + } + this._finalizeStop('Recording stopped'); } @@ -756,6 +790,84 @@ class SpeechService extends EventEmitter { } } + _startGroqRecording() { + this._cleanup(); + this.isRecording = true; + this.segmentBuffers = []; + this.segmentBytes = 0; + this.emit('recording-started'); + this.emit('status', 'Groq STT recording started'); + this._startMicrophoneCapture(); + + if (global.windowManager) { + global.windowManager.handleRecordingStarted(); + } + } + + async _finalizeGroqStop() { + if (!this.segmentBytes) { + this._finalizeStop('Recording stopped'); + return; + } + + const audioBuffer = Buffer.concat(this.segmentBuffers, this.segmentBytes); + this.segmentBuffers = []; + this.segmentBytes = 0; + + try { + const transcript = await this._groqTranscribe(audioBuffer); + if (transcript && transcript.trim()) { + this.emit('transcription', transcript.trim()); + } + } catch (error) { + logger.error('Groq STT transcription failed', { error: error.message }); + this.emit('error', `Groq transcription failed: ${error.message}`); + } finally { + this._finalizeStop('Recording stopped'); + } + } + + async _groqTranscribe(audioBuffer) { + const apiKey = process.env.GROQ_API_KEY; + if (!apiKey) { + throw new Error('GROQ_API_KEY not configured'); + } + + const baseUrl = 'https://api.groq.com/openai/v1'; + const model = this.groqSttModel; + const wavBuffer = this._createWavBuffer(audioBuffer); + + const boundary = '----FormBoundary' + Date.now(); + const header = Buffer.from( + `--${boundary}\r\n` + + `Content-Disposition: form-data; name="model"\r\n\r\n` + + `${model}\r\n` + + `--${boundary}\r\n` + + `Content-Disposition: form-data; name="file"; filename="audio.wav"\r\n` + + `Content-Type: audio/wav\r\n\r\n` + ); + const footer = Buffer.from(`\r\n--${boundary}--\r\n`); + const body = Buffer.concat([header, wavBuffer, footer]); + + const response = await fetch(`${baseUrl}/audio/transcriptions`, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': `multipart/form-data; boundary=${boundary}`, + 'Content-Length': String(body.length) + }, + body + }); + + if (!response.ok) { + const errText = await response.text(); + throw new Error(`Groq STT API error ${response.status}: ${errText}`); + } + + const data = await response.json(); + return data.text || ''; + } + _finalizeStop(statusMessage) { this._cleanup(); this.emit('recording-stopped'); @@ -800,6 +912,28 @@ class SpeechService extends EventEmitter { this.recording = null; } + // Destroy the hidden audio-capture window (renderer-based capture) + if (this._audioCaptureWindow) { + try { + if (!this._audioCaptureWindow.isDestroyed()) { + this._audioCaptureWindow.webContents.send('stop-audio-capture'); + this._audioCaptureWindow.destroy(); + } + } catch (error) { + logger.error('Error cleaning up audio capture window', { error: error.message }); + } + this._audioCaptureWindow = null; + } + + // Stop renderer-based capture in the chat window + if (global.windowManager) { + try { + global.windowManager.broadcastToAllWindows('stop-renderer-capture', {}); + } catch (error) { + logger.error('Error broadcasting stop-renderer-capture', { error: error.message }); + } + } + if (this.pushStream) { try { if (typeof this.pushStream.close === 'function') { @@ -851,6 +985,11 @@ class SpeechService extends EventEmitter { return this._transcribeWhisperFile(audioFilePath); } + if (this.provider === 'groq') { + const audioBuffer = fs.readFileSync(audioFilePath); + return this._groqTranscribe(audioBuffer); + } + throw new Error('Speech service not initialized'); } @@ -894,14 +1033,33 @@ class SpeechService extends EventEmitter { }; } + if (this.provider === 'groq') { + if (!process.env.GROQ_API_KEY) { + return { success: false, message: 'GROQ_API_KEY not configured' }; + } + // Test by sending a tiny silent WAV to the transcription API + try { + const silentBuffer = Buffer.alloc(16000 * 2); // 1 second of silence (16-bit PCM, 16kHz, mono) + const result = await this._groqTranscribe(silentBuffer); + // A successful API call returns empty text or a transcription + return { success: true, message: `Groq STT connection OK (model: ${this.groqSttModel})` }; + } catch (error) { + return { success: false, message: `Groq STT connection failed: ${error.message}` }; + } + } + return { success: false, message: 'Speech service not initialized' }; } getStatus() { + const isInitialized = this.provider === 'azure' ? !!this.speechConfig + : this.provider === 'groq' ? !!process.env.GROQ_API_KEY + : !!this.whisperCommand; + return { provider: this.provider, isRecording: this.isRecording, - isInitialized: this.provider === 'azure' ? !!this.speechConfig : !!this.whisperCommand, + isInitialized, sessionDuration: this.sessionStartTime ? Date.now() - this.sessionStartTime : 0, retryCount: this.retryCount, effectiveSettings: { @@ -912,11 +1070,13 @@ class SpeechService extends EventEmitter { whisperModelDir: this._getWhisperModelDir(), whisperModel: this._getWhisperModel(), whisperLanguage: this._getWhisperLanguage(), - whisperSegmentMs: String(this._getWhisperSegmentMs()) + whisperSegmentMs: String(this._getWhisperSegmentMs()), + groqSttModel: this.groqSttModel || process.env.GROQ_STT_MODEL || '' }, config: { azure: config.get('speech.azure') || {}, whisper: config.get('speech.whisper') || {}, + groq: config.get('speech.groq') || {}, selectedProvider: this.provider } }; @@ -931,11 +1091,15 @@ class SpeechService extends EventEmitter { return !!this.whisperCommand && !!this.available; } + if (this.provider === 'groq') { + return !!process.env.GROQ_API_KEY && !!this.available; + } + return false; } updateSettings(settings = {}) { - const speechKeys = ['speechProvider', 'azureKey', 'azureRegion', 'whisperCommand', 'whisperModelDir', 'whisperModel', 'whisperLanguage', 'whisperSegmentMs']; + const speechKeys = ['speechProvider', 'azureKey', 'azureRegion', 'whisperCommand', 'whisperModelDir', 'whisperModel', 'whisperLanguage', 'whisperSegmentMs', 'groqSttModel']; let changed = false; for (const key of speechKeys) { @@ -954,8 +1118,10 @@ class SpeechService extends EventEmitter { _getConfiguredProvider() { const provider = String(this._getSetting('speechProvider') || process.env.SPEECH_PROVIDER || '').trim().toLowerCase(); + console.log('[SpeechService] _getConfiguredProvider raw provider string:', provider, 'SPEECH_PROVIDER env:', process.env.SPEECH_PROVIDER); - if (provider === 'azure' || provider === 'whisper') { + if (provider === 'azure' || provider === 'whisper' || provider === 'groq') { + console.log('[SpeechService] _getConfiguredProvider returning:', provider); return provider; } @@ -963,9 +1129,17 @@ class SpeechService extends EventEmitter { (this._getSetting('azureRegion') || process.env.AZURE_SPEECH_REGION)); if (hasAzure) { + console.log('[SpeechService] _getConfiguredProvider auto-detecting azure'); return 'azure'; } + const hasGroq = !!(process.env.GROQ_API_KEY); + if (hasGroq) { + console.log('[SpeechService] _getConfiguredProvider auto-detecting groq'); + return 'groq'; + } + + console.log('[SpeechService] _getConfiguredProvider falling back to whisper'); return 'whisper'; } @@ -1196,63 +1370,33 @@ class SpeechService extends EventEmitter { } _startMicrophoneCapture() { - if (!recorder || typeof recorder.record !== 'function') { - this.emit('error', 'Local microphone capture dependency is missing. Run npm install to restore speech recording support.'); - return; - } - - this._startMicrophoneCaptureWithFallback(['sox', 'rec', 'arecord']); - } - - _startMicrophoneCaptureWithFallback(programs) { - const queue = [...programs]; - - const tryNextProgram = () => { - const program = queue.shift(); - if (!program) { - this.emit('error', 'Could not start microphone capture with any audio program'); - return; + console.log('[SpeechService] _startMicrophoneCapture() called'); + try { + // Instead of creating a hidden BrowserWindow (which fails on unsigned macOS), + // broadcast to the chat window (already visible) to start its own getUserMedia capture. + if (global.windowManager) { + console.log('[SpeechService] Broadcasting start-renderer-capture'); + global.windowManager.broadcastToAllWindows('start-renderer-capture', {}); } - try { - this.recording = recorder.record({ - sampleRateHertz: 16000, - channels: 1, - threshold: 0, - verbose: false, - recordProgram: program, - silence: '10.0s' - }); - - const stream = this.recording.stream(); - this.audioProgram = program; - - stream.on('error', (error) => { - logger.error('Audio recording stream error', { error: error.message, program }); - if (this.recording) { - try { - this.recording.stop(); - } catch (stopError) { - logger.error('Error stopping failed recording program', { error: stopError.message }); - } - this.recording = null; - } - - if (this.isRecording) { - tryNextProgram(); - } - }); - - stream.on('data', (chunk) => { - this._handleAudioChunk(chunk); - }); - } catch (error) { - logger.error('Failed to start microphone capture program', { program, error: error.message }); - tryNextProgram(); - } - }; + this.audioProgram = 'getUserMedia'; + logger.info('Renderer-based microphone capture started via chat window'); + } catch (error) { + logger.error('Failed to start renderer-based microphone capture', { error: error.message }); + this.emit('error', `Mic capture init failed: ${error.message}`); + } + } - tryNextProgram(); + _handleRendererAudioChunk(buffer) { + // Convert ArrayBuffer (from IPC) to Buffer and feed into the existing pipeline + if (!buffer) return; + try { + const buf = Buffer.from(buffer); + console.log('[SpeechService] _handleRendererAudioChunk buffer size:', buf.length, 'provider:', this.provider, 'isRecording:', this.isRecording); + this._handleAudioChunk(buf); + } catch (error) { + logger.error('Error handling renderer audio chunk', { error: error.message }); + } } _handleAudioChunk(chunk) { @@ -1269,9 +1413,12 @@ class SpeechService extends EventEmitter { return; } - if (this.provider === 'whisper') { + if (this.provider === 'whisper' || this.provider === 'groq') { this.segmentBuffers.push(Buffer.from(chunk)); this.segmentBytes += chunk.length; + if (this.segmentBytes % 16000 === 0) { + console.log('[SpeechService] Buffered audio bytes:', this.segmentBytes); + } } } diff --git a/src/services/tts.service.js b/src/services/tts.service.js new file mode 100644 index 00000000..de0a78f7 --- /dev/null +++ b/src/services/tts.service.js @@ -0,0 +1,181 @@ +const { EventEmitter } = require('events'); +const fs = require('fs'); +const path = require('path'); +const os = require('os'); +const { spawn } = require('child_process'); +const logger = require('../core/logger').createServiceLogger('TTS'); + +const GROQ_TTS_URL = 'https://api.groq.com/openai/v1/audio/speech'; + +class TtsService extends EventEmitter { + constructor() { + super(); + this._currentProcess = null; + this._enabled = false; + this._voice = 'tara'; + this._model = 'orpheus-tts-0.1-ayane'; + this._speed = 1.0; + this._initialize(); + } + + _initialize() { + try { + this._model = process.env.GROQ_TTS_MODEL || 'orpheus-tts-0.1-ayane'; + this._voice = process.env.GROQ_TTS_VOICE || 'tara'; + this._speed = parseFloat(process.env.GROQ_TTS_SPEED || '1.0') || 1.0; + this._enabled = !!process.env.GROQ_API_KEY; + logger.info(`TTS service initialized (model: ${this._model}, voice: ${this._voice})`); + } catch (error) { + logger.error('Failed to initialize TTS service', { error: error.message }); + this._enabled = false; + } + } + + isEnabled() { + return this._enabled; + } + + updateSettings(settings = {}) { + if (settings.ttsEnabled !== undefined) { + this._enabled = !!settings.ttsEnabled && !!process.env.GROQ_API_KEY; + } + if (settings.ttsVoice !== undefined) { + this._voice = settings.ttsVoice; + } + if (settings.ttsSpeed !== undefined) { + this._speed = parseFloat(settings.ttsSpeed) || 1.0; + } + if (settings.ttsModel !== undefined) { + this._model = settings.ttsModel; + } + // Re-enable check if GROQ_API_KEY might have changed + if (settings.groqKey !== undefined || settings.ttsEnabled !== undefined) { + this._enabled = !!process.env.GROQ_API_KEY && settings.ttsEnabled !== false; + } + } + + async synthesizeSpeech(text, options = {}) { + if (!this._enabled) { + throw new Error('TTS is not enabled. Configure GROQ_API_KEY and enable TTS.'); + } + + const apiKey = process.env.GROQ_API_KEY; + if (!apiKey) { + throw new Error('GROQ_API_KEY not configured'); + } + + const model = options.model || this._model; + const voice = options.voice || this._voice; + const speed = options.speed || this._speed; + + this.emit('tts-started'); + + try { + const response = await fetch(GROQ_TTS_URL, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + model, + input: text, + voice, + response_format: 'wav', + speed + }) + }); + + if (!response.ok) { + const errText = await response.text(); + throw new Error(`Groq TTS API error ${response.status}: ${errText}`); + } + + const audioBuffer = Buffer.from(await response.arrayBuffer()); + await this._playAudio(audioBuffer); + this.emit('tts-completed'); + } catch (error) { + logger.error('TTS synthesis failed', { error: error.message }); + this.emit('tts-error', error.message); + throw error; + } + } + + async _playAudio(audioBuffer) { + const ext = '.wav'; + const tempFile = path.join(os.tmpdir(), `opencluely-tts-${Date.now()}${ext}`); + + try { + fs.writeFileSync(tempFile, audioBuffer); + await this._spawnPlayer(tempFile); + } finally { + try { + fs.unlinkSync(tempFile); + } catch (_) {} + } + } + + _spawnPlayer(filePath) { + return new Promise((resolve, reject) => { + let cmd, args; + + if (process.platform === 'darwin') { + cmd = 'afplay'; + args = [filePath]; + } else if (process.platform === 'win32') { + // Windows: try using PowerShell's Media.SoundPlayer + cmd = 'powershell'; + args = ['-c', `(New-Object Media.SoundPlayer "${filePath}").PlaySync();`]; + } else { + // Linux: try ffplay, aplay, or paplay + cmd = 'ffplay'; + args = ['-nodisp', '-autoexit', filePath]; + } + + try { + const player = spawn(cmd, args, { stdio: 'ignore' }); + this._currentProcess = player; + + player.on('error', (err) => { + this._currentProcess = null; + // Fallback for Linux: try aplay + if (process.platform !== 'linux' || cmd === 'aplay') { + reject(new Error(`Audio player failed: ${err.message}`)); + return; + } + // Try aplay as fallback on Linux + const fallback = spawn('aplay', [filePath], { stdio: 'ignore' }); + this._currentProcess = fallback; + fallback.on('error', () => reject(new Error('No audio player available (tried ffplay, aplay)'))); + fallback.on('close', (code) => { + this._currentProcess = null; + code === 0 ? resolve() : reject(new Error(`aplay exited with code ${code}`)); + }); + }); + + player.on('close', (code) => { + this._currentProcess = null; + if (code === 0) { + resolve(); + } else { + reject(new Error(`Audio player exited with code ${code}`)); + } + }); + } catch (error) { + this._currentProcess = null; + reject(new Error(`Failed to spawn audio player: ${error.message}`)); + } + }); + } + + stopPlayback() { + if (this._currentProcess) { + try { + this._currentProcess.kill(); + } catch (_) {} + this._currentProcess = null; + } + } +} + +module.exports = new TtsService(); diff --git a/src/ui/audio-capture.html b/src/ui/audio-capture.html new file mode 100644 index 00000000..b4933ffc --- /dev/null +++ b/src/ui/audio-capture.html @@ -0,0 +1,75 @@ + + +Audio Capture + + + + diff --git a/src/ui/chat-window.js b/src/ui/chat-window.js index 5836ce09..d61ccd0e 100644 --- a/src/ui/chat-window.js +++ b/src/ui/chat-window.js @@ -280,15 +280,9 @@ class ChatWindowUI { // Show a brief activation message with the skill title const icons = { - 'dsa': '🧠', - 'behavioral': '💼', - 'sales': '💰', - 'presentation': '🎤', - 'data-science': '📊', - 'programming': '💻', - 'devops': '🚀', - 'system-design': '🏗️', - 'negotiation': '🤝' + 'general': '🧠', + 'coding': '💻', + 'meeting': '📋' }; const icon = icons[skillName] || '🎯'; diff --git a/src/ui/llm-response-window.js b/src/ui/llm-response-window.js index cad0fd82..f0ac46dc 100644 --- a/src/ui/llm-response-window.js +++ b/src/ui/llm-response-window.js @@ -4,7 +4,7 @@ class LLMResponseWindowUI { constructor() { this.currentLayout = "split"; this.hasCode = false; - this.currentSkill = "dsa"; + this.currentSkill = "general"; this.isInteractive = false; this.scrollableElements = []; @@ -92,10 +92,7 @@ class LLMResponseWindowUI { timestamp: new Date().toISOString(), }); - // Add a small delay to ensure DOM is ready - setTimeout(() => { - this.handleDisplayResponse(data); - }, 50); + this.handleDisplayResponse(data); }); // Interaction state handlers @@ -247,8 +244,6 @@ class LLMResponseWindowUI { result, }); - // Use a more reliable delay mechanism - await new Promise((resolve) => setTimeout(resolve, 200)); this.displayResponseContent(data); } catch (error) { logger.error("Failed to expand window", { @@ -345,29 +340,27 @@ class LLMResponseWindowUI { this.requestWindowResize(contentMetrics); // Final verification - setTimeout(() => { - const isLoadingHidden = this.elements.loading - ? this.elements.loading.classList.contains("hidden") - : true; - const isContentVisible = this.elements.responseContent - ? !this.elements.responseContent.classList.contains("hidden") - : false; - - logger.info("Display state verification", { + const isLoadingHidden = this.elements.loading + ? this.elements.loading.classList.contains("hidden") + : true; + const isContentVisible = this.elements.responseContent + ? !this.elements.responseContent.classList.contains("hidden") + : false; + + logger.info("Display state verification", { + component: "LLMResponseWindowUI", + loadingHidden: isLoadingHidden, + contentVisible: isContentVisible, + windowVisible: !document.hidden, + }); + + if (!isLoadingHidden || !isContentVisible) { + logger.warn("Display state inconsistent - forcing correction", { component: "LLMResponseWindowUI", - loadingHidden: isLoadingHidden, - contentVisible: isContentVisible, - windowVisible: !document.hidden, }); - - if (!isLoadingHidden || !isContentVisible) { - logger.warn("Display state inconsistent - forcing correction", { - component: "LLMResponseWindowUI", - }); - this.hideLoadingState(); - this.showResponseContent(); - } - }, 100); + this.hideLoadingState(); + this.showResponseContent(); + } logger.debug("displayResponseContent completed - END"); } catch (error) { @@ -400,23 +393,21 @@ class LLMResponseWindowUI { async requestWindowResize(contentMetrics) { try { - setTimeout(async () => { - logger.debug("Requesting window resize based on content metrics", { - component: "LLMResponseWindowUI", - metrics: contentMetrics, - }); + logger.debug("Requesting window resize based on content metrics", { + component: "LLMResponseWindowUI", + metrics: contentMetrics, + }); - const { ipcRenderer } = require("electron"); - const result = await ipcRenderer.invoke( - "resize-llm-window-for-content", - contentMetrics - ); + const { ipcRenderer } = require("electron"); + const result = await ipcRenderer.invoke( + "resize-llm-window-for-content", + contentMetrics + ); - logger.debug("Window resize result", { - component: "LLMResponseWindowUI", - result, - }); - }, 200); + logger.debug("Window resize result", { + component: "LLMResponseWindowUI", + result, + }); } catch (error) { logger.error("Failed to resize window", { component: "LLMResponseWindowUI", diff --git a/src/ui/main-window.js b/src/ui/main-window.js index e6ab7cc4..9bf63fc9 100644 --- a/src/ui/main-window.js +++ b/src/ui/main-window.js @@ -10,7 +10,7 @@ class MainWindowUI { constructor() { this.isInteractive = false; this.isHidden = false; - this.currentSkill = 'dsa'; // Default, will be updated from settings + this.currentSkill = 'general'; this.statusDot = null; this.skillIndicator = null; this.micButton = null; @@ -18,9 +18,10 @@ class MainWindowUI { this.speechAvailable = false; // track availability this._popoverHideTimeout = null; - // Define available skills for navigation this.availableSkills = [ - 'dsa' + 'general', + 'coding', + 'meeting' ]; this.init(); @@ -283,17 +284,10 @@ class MainWindowUI { } }); - // Skill indicator click handler toggles DSA skill + // Skill indicator click handler cycles to next skill this.skillIndicator.addEventListener('click', () => { if (!this.isInteractive) return; - const newSkill = 'dsa'; - if (window.electronAPI && window.electronAPI.updateActiveSkill) { - window.electronAPI.updateActiveSkill(newSkill).then(() => { - this.handleSkillActivated(newSkill); - }); - } else { - this.handleSkillActivated(newSkill); - } + this.navigateSkill(1); }); // Check for required elements (settingsIndicator is optional) @@ -505,15 +499,9 @@ class MainWindowUI { handleLLMResponse(data) { const skill = data.skill || data.metadata?.skill || 'General'; const skillNames = { - 'dsa': 'DSA', - 'behavioral': 'Behavioral', - 'sales': 'Sales', - 'presentation': 'Presentation', - 'data-science': 'Data Science', - 'programming': 'Programming', - 'devops': 'DevOps', - 'system-design': 'System Design', - 'negotiation': 'Negotiation' + 'general': 'General', + 'coding': 'Coding', + 'meeting': 'Meeting' }; const displaySkill = skillNames[skill] || skill.toUpperCase(); @@ -645,15 +633,9 @@ class MainWindowUI { updateSkillIndicator() { const skillNames = { - 'dsa': 'DSA', - 'behavioral': 'Behavioral', - 'sales': 'Sales', - 'presentation': 'Presentation', - 'data-science': 'Data Science', - 'programming': 'Programming', - 'devops': 'DevOps', - 'system-design': 'System Design', - 'negotiation': 'Negotiation' + 'general': 'General', + 'coding': 'Coding', + 'meeting': 'Meeting' }; logger.info('Updating skill indicator', { @@ -669,6 +651,7 @@ class MainWindowUI { const skillName = skillNames[this.currentSkill] || this.currentSkill.toUpperCase(); const skillSpan = this.skillIndicator.querySelector('span'); + const skillIcon = this.skillIndicator.querySelector('i'); logger.info('Looking for skill span element', { component: 'MainWindowUI', @@ -676,6 +659,27 @@ class MainWindowUI { skillName: skillName }); + // Update the icon based on skill + const skillIcons = { + 'general': 'fa-brain', + 'coding': 'fa-code', + 'meeting': 'fa-users' + }; + if (skillIcon) { + // Remove all existing fa-* classes except fa-solid + skillIcon.className = skillIcon.className + .split(' ') + .filter(c => c === 'fas' || c === 'fa-solid') + .join(' '); + skillIcon.classList.add(skillIcons[this.currentSkill] || 'fa-brain'); + } + + // Show/hide language selector based on skill + const languageSelector = document.getElementById('languageSelector'); + if (languageSelector) { + languageSelector.style.display = this.currentSkill === 'coding' ? '' : 'none'; + } + if (skillSpan) { const oldText = skillSpan.textContent; skillSpan.textContent = skillName; @@ -758,15 +762,9 @@ class MainWindowUI { showSkillChangeNotification(skill, direction) { const skillNames = { - 'dsa': 'DSA', - 'behavioral': 'Behavioral', - 'sales': 'Sales', - 'presentation': 'Presentation', - 'data-science': 'Data Science', - 'programming': 'Programming', - 'devops': 'DevOps', - 'system-design': 'System Design', - 'negotiation': 'Negotiation' + 'general': 'General', + 'coding': 'Coding', + 'meeting': 'Meeting' }; const displayName = skillNames[skill] || skill.toUpperCase(); diff --git a/src/ui/settings-window.js b/src/ui/settings-window.js index e0270258..94373538 100644 --- a/src/ui/settings-window.js +++ b/src/ui/settings-window.js @@ -13,12 +13,23 @@ document.addEventListener('DOMContentLoaded', () => { const whisperModelInput = document.getElementById('whisperModel'); const whisperLanguageInput = document.getElementById('whisperLanguage'); const whisperSegmentMsInput = document.getElementById('whisperSegmentMs'); + const groqSttModelInput = document.getElementById('groqSttModel'); + const ttsEnabledInput = document.getElementById('ttsEnabled'); + const ttsVoiceInput = document.getElementById('ttsVoice'); + const ttsSpeedInput = document.getElementById('ttsSpeed'); const geminiKeyInput = document.getElementById('geminiKey'); const windowGapInput = document.getElementById('windowGap'); const codingLanguageSelect = document.getElementById('codingLanguage'); const activeSkillSelect = document.getElementById('activeSkill'); const iconGrid = document.getElementById('iconGrid'); + // LLM provider fields + const llmProviderSelect = document.getElementById('llmProvider'); + const openrouterKeyInput = document.getElementById('openrouterKey'); + const openrouterModelInput = document.getElementById('openrouterModel'); + const groqKeyInput = document.getElementById('groqKey'); + const groqModelInput = document.getElementById('groqModel'); + // Check if window.api exists if (!window.api) { console.error('window.api not available'); @@ -81,8 +92,18 @@ document.addEventListener('DOMContentLoaded', () => { if (whisperModelInput) whisperModelInput.value = settings.whisperModel || ''; if (whisperLanguageInput) whisperLanguageInput.value = settings.whisperLanguage || ''; if (whisperSegmentMsInput) whisperSegmentMsInput.value = settings.whisperSegmentMs || ''; + if (groqSttModelInput) groqSttModelInput.value = settings.groqSttModel || 'whisper-large-v3-turbo'; + if (ttsEnabledInput) ttsEnabledInput.checked = settings.ttsEnabled !== false; + if (ttsVoiceInput) ttsVoiceInput.value = settings.ttsVoice || 'tara'; + if (ttsSpeedInput) ttsSpeedInput.value = settings.ttsSpeed || '1.0'; if (geminiKeyInput) geminiKeyInput.value = settings.geminiKey || ''; if (windowGapInput) windowGapInput.value = settings.windowGap || ''; + if (llmProviderSelect) llmProviderSelect.value = settings.llmProvider || 'gemini'; + if (openrouterKeyInput) openrouterKeyInput.value = settings.openrouterKey || ''; + if (openrouterModelInput) openrouterModelInput.value = settings.openrouterModel || 'openrouter/free'; + if (groqKeyInput) groqKeyInput.value = settings.groqKey || ''; + if (groqModelInput) groqModelInput.value = settings.groqModel || 'llama-3.3-70b-versatile'; + updateLLMFieldStates(); // Set C++ as default if no coding language is specified if (codingLanguageSelect) { @@ -127,6 +148,25 @@ document.addEventListener('DOMContentLoaded', () => { }); } + const updateLLMFieldStates = () => { + const provider = llmProviderSelect ? llmProviderSelect.value : 'gemini'; + const geminiGroup = document.getElementById('geminiLlmFields'); + const openrouterGroup = document.getElementById('openrouterLlmFields'); + const openrouterNote = document.getElementById('openrouterNote'); + const groqGroup = document.getElementById('groqLlmFields'); + + if (geminiGroup) geminiGroup.style.display = provider === 'gemini' ? '' : 'none'; + if (openrouterGroup) openrouterGroup.style.display = provider === 'openrouter' ? '' : 'none'; + if (openrouterNote) openrouterNote.style.display = provider === 'openrouter' ? '' : 'none'; + if (groqGroup) groqGroup.style.display = provider === 'groq' ? '' : 'none'; + + if (geminiKeyInput) geminiKeyInput.disabled = provider !== 'gemini'; + if (openrouterKeyInput) openrouterKeyInput.disabled = provider !== 'openrouter'; + if (openrouterModelInput) openrouterModelInput.disabled = provider !== 'openrouter'; + if (groqKeyInput) groqKeyInput.disabled = provider !== 'groq'; + if (groqModelInput) groqModelInput.disabled = provider !== 'groq'; + }; + // Save settings helper function const saveSettings = () => { const settings = {}; @@ -137,10 +177,19 @@ document.addEventListener('DOMContentLoaded', () => { if (whisperModelInput) settings.whisperModel = whisperModelInput.value; if (whisperLanguageInput) settings.whisperLanguage = whisperLanguageInput.value; if (whisperSegmentMsInput) settings.whisperSegmentMs = whisperSegmentMsInput.value; + if (groqSttModelInput) settings.groqSttModel = groqSttModelInput.value; + if (ttsEnabledInput) settings.ttsEnabled = ttsEnabledInput.checked; + if (ttsVoiceInput) settings.ttsVoice = ttsVoiceInput.value; + if (ttsSpeedInput) settings.ttsSpeed = ttsSpeedInput.value; if (geminiKeyInput) settings.geminiKey = geminiKeyInput.value; if (windowGapInput) settings.windowGap = windowGapInput.value; if (codingLanguageSelect) settings.codingLanguage = codingLanguageSelect.value; if (activeSkillSelect) settings.activeSkill = activeSkillSelect.value; + if (llmProviderSelect) settings.llmProvider = llmProviderSelect.value; + if (openrouterKeyInput) settings.openrouterKey = openrouterKeyInput.value; + if (openrouterModelInput) settings.openrouterModel = openrouterModelInput.value; + if (groqKeyInput) settings.groqKey = groqKeyInput.value; + if (groqModelInput) settings.groqModel = groqModelInput.value; window.api.send('save-settings', settings); }; @@ -154,6 +203,7 @@ document.addEventListener('DOMContentLoaded', () => { const azureGroup = document.getElementById('azureFields'); const whisperGroup = document.getElementById('whisperFields'); const azureNote = document.getElementById('azureFieldsNote'); + const groqSttGroup = document.getElementById('groqSttFields'); if (azureGroup) { azureGroup.style.display = provider === 'azure' ? '' : 'none'; @@ -164,6 +214,9 @@ document.addEventListener('DOMContentLoaded', () => { if (azureNote) { azureNote.style.display = provider === 'azure' ? '' : 'none'; } + if (groqSttGroup) { + groqSttGroup.style.display = provider === 'groq' ? '' : 'none'; + } // Also toggle disabled attribute for any leftover direct field refs [azureKeyInput, azureRegionInput].forEach(input => { @@ -172,6 +225,9 @@ document.addEventListener('DOMContentLoaded', () => { [whisperCommandInput, whisperModelInput, whisperLanguageInput, whisperSegmentMsInput].forEach(input => { if (input) input.disabled = provider !== 'whisper'; }); + [groqSttModelInput].forEach(input => { + if (input) input.disabled = provider !== 'groq'; + }); }; // Add event listeners for all inputs @@ -183,8 +239,17 @@ document.addEventListener('DOMContentLoaded', () => { whisperModelInput, whisperLanguageInput, whisperSegmentMsInput, + groqSttModelInput, + ttsEnabledInput, + ttsVoiceInput, + ttsSpeedInput, geminiKeyInput, - windowGapInput + windowGapInput, + llmProviderSelect, + openrouterKeyInput, + openrouterModelInput, + groqKeyInput, + groqModelInput ]; inputs.forEach(input => { @@ -201,6 +266,13 @@ document.addEventListener('DOMContentLoaded', () => { }); } + if (llmProviderSelect) { + llmProviderSelect.addEventListener('change', () => { + updateLLMFieldStates(); + saveSettings(); + }); + } + // Language selection handler if (codingLanguageSelect) { codingLanguageSelect.addEventListener('change', (e) => {