Rag readiness

.github/workflows/cd.yml

@@ -42,7 +42,6 @@ jobs:
           | File | Type | Notes |
           |------|------|-------|
           | `Notely Setup *.exe` | **Installer** (recommended) | Installs to your system, fast launch every time |
-          | `Notely *.exe` | **Portable** | Single file, no install — extracts on every launch |
           | `Notely-*-win.zip` | **Portable ZIP** | Extract once, run directly — portable + fast |
         files: |
           release/*.exe

ai/core/AIConfig.js

@@ -7,8 +7,16 @@ const fs = require('fs');
 const { app, safeStorage } = require('electron');
 
 class AIConfig {
-  constructor() {
-    this.appDataDir = app.getPath('appData');
+  constructor(customAppDataDir = null) {
+    if (customAppDataDir) {
+      this.appDataDir = customAppDataDir;
+    } else {
+      try {
+        this.appDataDir = app ? app.getPath('appData') : path.join(process.env.APPDATA || process.env.HOME || '', 'Notely');
+      } catch {
+        this.appDataDir = path.join(process.env.APPDATA || process.env.HOME || '', 'Notely');
+      }
+    }
     this.configDir = path.join(this.appDataDir, 'notely');
     this.configPath = path.join(this.configDir, 'ai-config.json');
     this.ensureConfigDir();

ai/core/Agent.js

@@ -185,11 +185,11 @@ class Agent {
   /**
    * Build relationship graph
    */
-  async buildRelationshipGraph() {
+  async buildRelationshipGraph(onProgress = null) {
     if (!this.graphBuilder) {
       return { success: false, error: 'Graph builder not initialized' };
     }
-    return this.graphBuilder.rebuild();
+    return this.graphBuilder.rebuild(onProgress);
   }
 
   /**

ai/core/QueryExecutor.js

@@ -73,18 +73,7 @@ class QueryExecutor {
       ...contextEngineTools
     };
 
-    // Prune tools for conversational follow-up queries to prevent redundant tool execution
-    const cleanQuery = query.toLowerCase().trim();
-    const followUpKeywords = [
-      'suggest', 'pick', 'choose', 'first', 'second', 'third', 'next', 'which',
-      'ok', 'great', 'fine', 'yes', 'no', 'sure', 'why', 'how about', 'what do you think'
-    ];
-    const isFollowUp = cleanQuery.length < 50 && followUpKeywords.some(kw => cleanQuery.includes(kw));
-
     let toolChoice = 'auto';
-    if (isFollowUp && ceMessages.length > 0) {
-      toolChoice = 'none';
-    }
 
     // Build messages array
     let messages = [];

ai/core/system_prompt.md

@@ -30,15 +30,19 @@ You have access to the user's local workspace context:
 - Do not repeat lists of items or recapping the same information multiple times unless explicitly requested.
 
 ### C. Specific Tools
-- `get_tasks`: Retrieve open tasks from the workspace notes. Use this when the user specifically asks to find, list, or summarize their tasks.
-- `read_note`: Retrieve the content of a specific note file. Supports `start_line` and `end_line` parameters for paginating large documents.
-- `search_notes`: Perform a unified RRF (Reciprocal Rank Fusion) hybrid search combining semantic vector search and knowledge graph traversal.
-- `exploreGraph` or `explore_graph`: Query the knowledge graph database for note relations, wikilinks, and tags.
-- `git_diff`: Retrieve the active git workspace diff showing local unstaged/modified changes to notes.
-- `git_commit`: Stage and commit modified files with a user-supplied message.
-- `read_pdf`: Read and extract plain text from local PDF note attachments.
-- `resolve_folder_link`: Resolve folder contents and list markdown notes inside subdirectories.
-- `get_current_date`: Retrieve the current date and time. **Required**: Execute this tool first before answering any questions about relative dates (e.g. "today", "yesterday", "this week").
+- `read_note`: Retrieve the contents of a specific note file in the workspace. Use `startLine` and `maxLines` to paginate/limit output.
+- `create_note`: Create a new note with a title, initial content, and target folder in the workspace.
+- `move_note`: Move or rename a note within the workspace.
+- `get_tasks`: Extract checklist tasks across notes in the workspace. Supports filtering by status (open, completed, all) and note path.
+- `search_notes`: Search note files matching a query string in the workspace.
+- `semantic_search`: Find semantically similar notes using vector embeddings.
+- `hybrid_search`: Perform a hybrid search combining full-text keyword search and semantic vector similarity.
+- `get_graph`: Traverse knowledge graph relationships for a given note.
+- `find_clusters`: Get semantic topic clusters across the workspace.
+- `knowledge_status`: Retrieve the indexing and health status of the knowledge engines.
+- `reindex_knowledge`: Trigger background reindexing of the knowledge graph and embeddings.
+- `workspace_stats`: Get workspace health, document counts, and storage metrics.
+- `recent_activity`: Get a list of recently modified notes in the workspace.
 
 ---
 
@@ -62,6 +66,7 @@ You have access to the user's local workspace context:
 2. **NEVER** speculate about what tasks the user "might" have or invent task checklist items to make lists look complete.
 3. **NEVER** invent links between notes (wikilinks) unless the graph retriever explicitly confirms the relationship exists.
 4. **NEVER** use pre-training knowledge to describe workspace content. All workspace information must come strictly from the live tool outputs.
+5. **NEVER** attempt to access, refer to, or edit any file paths located outside the active workspace root. All operations are strictly sandboxed within the workspace boundaries.
 
 ### Strict Verification Loop (Mental Checklist)
 1. Is every note path cited as a `file:///` link present in the raw tool outputs? If not, delete it.

ai/embeddings/ModelDownloader.js

@@ -20,24 +20,24 @@
     this.vocabUrl = `https://huggingface.co/Xenova/bge-small-en-v1.5/${vocabUrlPart}`;
     this.progressCallback = null;
 
-    this.qwenONNXDir = path.join(this.modelDir, 'qwen-onnx');
-    this.qwenONNXFiles = [
-      { name: 'config.json', url: 'https://huggingface.co/onnx-community/Qwen2.5-0.5B-Instruct/resolve/main/config.json' },
-      { name: 'generation_config.json', url: 'https://huggingface.co/onnx-community/Qwen2.5-0.5B-Instruct/resolve/main/generation_config.json' },
-      { name: 'special_tokens_map.json', url: 'https://huggingface.co/onnx-community/Qwen2.5-0.5B-Instruct/resolve/main/special_tokens_map.json' },
-      { name: 'tokenizer.json', url: 'https://huggingface.co/onnx-community/Qwen2.5-0.5B-Instruct/resolve/main/tokenizer.json' },
-      { name: 'tokenizer_config.json', url: 'https://huggingface.co/onnx-community/Qwen2.5-0.5B-Instruct/resolve/main/tokenizer_config.json' },
-      { name: 'onnx/model_quantized.onnx', url: 'https://huggingface.co/onnx-community/Qwen2.5-0.5B-Instruct/resolve/main/onnx/model_quantized.onnx' }
+    this.smolLM2ONNXDir = path.join(this.modelDir, 'smollm2-135m-onnx');
+    this.smolLM2ONNXFiles = [
+      { name: 'config.json', url: 'https://huggingface.co/onnx-community/SmolLM2-135M-Instruct-ONNX/resolve/main/config.json' },
+      { name: 'generation_config.json', url: 'https://huggingface.co/onnx-community/SmolLM2-135M-Instruct-ONNX/resolve/main/generation_config.json' },
+      { name: 'special_tokens_map.json', url: 'https://huggingface.co/onnx-community/SmolLM2-135M-Instruct-ONNX/resolve/main/special_tokens_map.json' },
+      { name: 'tokenizer.json', url: 'https://huggingface.co/onnx-community/SmolLM2-135M-Instruct-ONNX/resolve/main/tokenizer.json' },
+      { name: 'tokenizer_config.json', url: 'https://huggingface.co/onnx-community/SmolLM2-135M-Instruct-ONNX/resolve/main/tokenizer_config.json' },
+      { name: 'onnx/model_quantized.onnx', url: 'https://huggingface.co/onnx-community/SmolLM2-135M-Instruct-ONNX/resolve/main/onnx/model_quantized.onnx' }
     ];
   }
 
   isGraphModelDownloaded() {
-    return this.qwenONNXFiles.every(file => fs.existsSync(path.join(this.qwenONNXDir, file.name)));
+    return this.smolLM2ONNXFiles.every(file => fs.existsSync(path.join(this.smolLM2ONNXDir, file.name)));
   }
 
   async downloadGraphModel(onProgress = null) {
     if (this.isGraphModelDownloaded()) {
-      log.info('Graph Qwen ONNX model already downloaded');
+      log.info('Graph SmolLM2 ONNX model already downloaded');
       return true;
     }
     if (isDownloadingGraph) {
@@ -50,28 +50,28 @@
     this.progressCallback = onProgress;
 
     try {
-      if (!fs.existsSync(this.qwenONNXDir)) {
-        fs.mkdirSync(this.qwenONNXDir, { recursive: true });
+      if (!fs.existsSync(this.smolLM2ONNXDir)) {
+        fs.mkdirSync(this.smolLM2ONNXDir, { recursive: true });
       }
 
-      log.info('Starting Qwen 2.5 ONNX model download from HuggingFace...');
+      log.info('Starting SmolLM2 ONNX model download from HuggingFace...');
 
       let completedCount = 0;
-      for (const file of this.qwenONNXFiles) {
-        const destPath = path.join(this.qwenONNXDir, file.name);
+      for (const file of this.smolLM2ONNXFiles) {
+        const destPath = path.join(this.smolLM2ONNXDir, file.name);
         const destDir = path.dirname(destPath);
         if (!fs.existsSync(destDir)) {
           fs.mkdirSync(destDir, { recursive: true });
         }
 
-        log.info(`Downloading Qwen ONNX asset: ${file.name}...`);
+        log.info(`Downloading SmolLM2 ONNX asset: ${file.name}...`);
 
         const isModelFile = file.name.endsWith('.onnx');
 
         await this.downloadFile(file.url, destPath, (bytesRead, totalBytes) => {
-          if (isModelFile && totalBytes > 0) {
-            const baseProgress = Math.round((completedCount / this.qwenONNXFiles.length) * 100);
-            const currentFileProgress = Math.round((bytesRead / totalBytes) * (100 / this.qwenONNXFiles.length));
+          if (totalBytes > 0) {
+            const baseProgress = Math.round((completedCount / this.smolLM2ONNXFiles.length) * 100);
+            const currentFileProgress = Math.round((bytesRead / totalBytes) * (100 / this.smolLM2ONNXFiles.length));
             graphProgress = Math.min(99, baseProgress + currentFileProgress);
             if (this.progressCallback) {
               this.progressCallback(graphProgress);
@@ -80,19 +80,19 @@
         });
 
         completedCount++;
-        graphProgress = Math.round((completedCount / this.qwenONNXFiles.length) * 100);
+        graphProgress = Math.round((completedCount / this.smolLM2ONNXFiles.length) * 100);
         if (this.progressCallback) {
           this.progressCallback(graphProgress);
         }
       }
 
-      log.info('Qwen 2.5 ONNX model downloaded successfully');
+      log.info('SmolLM2 ONNX model downloaded successfully');
       isDownloadingGraph = false;
       graphProgress = 100;
       return true;
     } catch (err) {
       isDownloadingGraph = false;
-      log.error('Failed to download Qwen ONNX model', err);
+      log.error('Failed to download SmolLM2 ONNX model', err);
       throw err;
     }
   }
@@ -117,6 +117,33 @@
     };
   }
 
+  deleteModel() {
+    try {
+      const modelPath = path.join(this.modelDir, 'model.onnx');
+      const vocabPath = path.join(this.modelDir, 'vocab.txt');
+      if (fs.existsSync(modelPath)) fs.unlinkSync(modelPath);
+      if (fs.existsSync(vocabPath)) fs.unlinkSync(vocabPath);
+      log.info('Deleted local embedding model files.');
+      return true;
+    } catch (err) {
+      log.error('Failed to delete embedding model files', err);
+      throw err;
+    }
+  }
+
+  deleteGraphModel() {
+    try {
+      if (fs.existsSync(this.smolLM2ONNXDir)) {
+        fs.rmSync(this.smolLM2ONNXDir, { recursive: true, force: true });
+      }
+      log.info('Deleted local graph ONNX model files.');
+      return true;
+    } catch (err) {
+      log.error('Failed to delete graph model files', err);
+      throw err;
+    }
+  }
+
   async download(onProgress = null) {
     if (this.isModelDownloaded()) {
       log.info('Model already downloaded');
@@ -171,7 +198,12 @@
       const file = fs.createWriteStream(dest);
 
       const request = (targetUrl) => {
-        https.get(targetUrl, (response) => {
+        const options = {
+          headers: {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) NotelyApp/0.1.27 Chrome/120.0.0.0 Electron/28.0.0 Safari/537.36'
+          }
+        };
+        https.get(targetUrl, options, (response) => {
           if (response.statusCode === 301 || response.statusCode === 302 || response.statusCode === 303 || response.statusCode === 307 || response.statusCode === 308) {
             // Handle redirects (including relative paths)
             let redirectUrl = response.headers.location;
@@ -201,6 +233,10 @@
 
           response.on('end', () => {
             file.end();
+          });
+
+          file.on('finish', () => {
+            file.close();
             resolve();
           });
         }).on('error', (err) => {

ai/embeddings/ONNXEmbedder.js

@@ -18,12 +18,15 @@ class ONNXEmbedder {
     if (this.isLoaded) return;
     try {
       log.info('Loading local ONNX embedding session...');
-      try {
-        this.ort = require('onnxruntime-node');
-      } catch (err) {
-        log.warn('Failed to load native onnxruntime-node. Trying onnxruntime-web (WASM) fallback:', err.message);
-        this.ort = require('onnxruntime-web');
+      this.ort = require('onnxruntime-web');
+      this.ort.env.wasm.numThreads = 1;
+
+      const { pathToFileURL } = require('url');
+      let wasmDir = path.dirname(require.resolve('onnxruntime-web')) + path.sep;
+      if (wasmDir.includes('app.asar')) {
+        wasmDir = wasmDir.replace('app.asar', 'app.asar.unpacked');
       }
+      this.ort.env.wasm.wasmPaths = pathToFileURL(wasmDir).href;
 
       const modelPath = path.join(this.modelDir, 'model.onnx');
       const vocabPath = path.join(this.modelDir, 'vocab.txt');

ai/graph/GraphBuilder.js

@@ -19,7 +19,7 @@ class GraphBuilder {
   /**
    * Scan notes and rebuild the Knowledge Graph
    */
-  async rebuild() {
+  async rebuild(onProgress = null) {
     if (this.isRebuilding) {
       log.warn('Rebuild already in progress');
       return { success: false, error: 'Rebuild already in progress' };
@@ -43,13 +43,21 @@ class GraphBuilder {
 
       // Find all markdown files in the workspace
       const workspaceFiles = this._getWorkspaceMarkdownFiles();
-      log.info(`Found ${workspaceFiles.length} markdown notes to index for graph`);
-      logDb.addLog('graph', `Found ${workspaceFiles.length} markdown notes to index for graph`, 'info');
+      const total = workspaceFiles.length;
+      log.info(`Found ${total} markdown notes to index for graph`);
+      logDb.addLog('graph', `Found ${total} markdown notes to index for graph`, 'info');
 
       let processedCount = 0;
       let failedCount = 0;
 
-      for (const filePath of workspaceFiles) {
+      for (let i = 0; i < total; i++) {
+        // Yield event loop between heavy CPU/LLM processing steps so main thread stays 100% responsive
+        await new Promise(resolve => setTimeout(resolve, 50));
+
+        const filePath = workspaceFiles[i];
+        if (typeof onProgress === 'function') {
+          onProgress({ current: i + 1, total, noteName: path.basename(filePath) });
+        }
         try {
           if (!fs.existsSync(filePath)) {
             failedCount++;

ai/graph/GraphDB.js

@@ -186,6 +186,24 @@ class GraphDB {
     return { nodeCount, edgeCount, sizeBytes };
   }
 
+  getNodeCount() {
+    if (!this.db) return 0;
+    try {
+      return this.db.prepare('SELECT COUNT(*) as count FROM entities').get()?.count || 0;
+    } catch {
+      return 0;
+    }
+  }
+
+  getEdgeCount() {
+    if (!this.db) return 0;
+    try {
+      return this.db.prepare('SELECT COUNT(*) as count FROM relationships').get()?.count || 0;
+    } catch {
+      return 0;
+    }
+  }
+
   /**
    * Purge all entities and relationships
    */