From 3df9c6f6ff2fa346767fc7de22aac249be2327c3 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Fri, 31 Jul 2026 09:46:47 -0400 Subject: [PATCH 1/2] Add multi-select metadata extraction Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- application/single_app/config.py | 2 +- .../single_app/route_backend_documents.py | 63 ++++ .../route_backend_group_documents.py | 80 +++++ .../route_backend_public_documents.py | 75 +++++ .../static/js/public/public_workspace.js | 62 ++++ .../js/workspace/workspace-documents.js | 74 ++++- .../templates/group_workspaces.html | 74 ++++- .../templates/public_workspaces.html | 3 + .../single_app/templates/workspace.html | 3 + .../test_multiselect_metadata_extraction.py | 277 ++++++++++++++++++ 10 files changed, 709 insertions(+), 4 deletions(-) create mode 100644 functional_tests/test_multiselect_metadata_extraction.py diff --git a/application/single_app/config.py b/application/single_app/config.py index 9f8af2cc3..e7b780d6e 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -95,7 +95,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.250.105" +VERSION = "0.250.106" IS_DEVELOPMENT = is_development_env_enabled() SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') diff --git a/application/single_app/route_backend_documents.py b/application/single_app/route_backend_documents.py index 18ac783d2..97b23f509 100644 --- a/application/single_app/route_backend_documents.py +++ b/application/single_app/route_backend_documents.py @@ -1279,6 +1279,69 @@ def api_extract_user_metadata(document_id): 'document_id': document_id }), 200 + @bp.route('/api/documents/extract_metadata', methods=['POST']) + @swagger_route(security=get_auth_security()) + @login_required + @user_required + @enabled_required("enable_user_workspace") + def api_extract_user_metadata_batch(): + """ + POST /api/documents/extract_metadata + Queues background metadata extraction jobs for selected user documents. + """ + user_id = get_current_user_id() + if not user_id: + return jsonify({'error': 'User not authenticated'}), 401 + + settings = get_settings() + if not settings.get('enable_extract_meta_data'): + return jsonify({'error': 'Metadata extraction not enabled'}), 403 + + payload = request.get_json(silent=True) or {} + document_ids = payload.get('document_ids') + if not isinstance(document_ids, list): + document_id = payload.get('document_id') + document_ids = [document_id] if document_id else [] + document_ids = list(dict.fromkeys( + str(document_id).strip() + for document_id in document_ids + if str(document_id or '').strip() + )) + if not document_ids: + return jsonify({'error': 'At least one document ID is required.'}), 400 + + queued = [] + errors = [] + for document_id in document_ids: + try: + document_item = get_document_metadata(document_id=document_id, user_id=user_id) + if not document_item: + errors.append({'document_id': document_id, 'error': 'Document not found.'}) + continue + if document_item.get('user_id') != user_id: + errors.append({'document_id': document_id, 'error': 'Only the document owner can extract metadata for this document.'}) + continue + + current_app.extensions['executor'].submit_stored( + f"{document_id}_metadata", + process_metadata_extraction_background, + document_id=document_id, + user_id=user_id + ) + queued.append({'document_id': document_id}) + except Exception as e: + errors.append({'document_id': document_id, 'error': str(e)}) + + if queued: + invalidate_personal_search_cache(user_id) + + status_code = 202 if queued and not errors else (207 if queued else 400) + return jsonify({ + 'message': f'Queued {len(queued)} document(s) for metadata extraction.', + 'queued': queued, + 'errors': errors, + }), status_code + @bp.route('/api/documents/reprocess_extraction', methods=['POST']) @swagger_route(security=get_auth_security()) @login_required diff --git a/application/single_app/route_backend_group_documents.py b/application/single_app/route_backend_group_documents.py index 386a976a7..ff4af9938 100644 --- a/application/single_app/route_backend_group_documents.py +++ b/application/single_app/route_backend_group_documents.py @@ -1185,6 +1185,86 @@ def api_extract_group_metadata(document_id): 'document_id': document_id }), 200 + @bp.route('/api/group_documents/extract_metadata', methods=['POST']) + @swagger_route(security=get_auth_security()) + @login_required + @user_required + @enabled_required("enable_group_workspaces") + def api_extract_group_metadata_batch(): + """ + POST /api/group_documents/extract_metadata + Queues background metadata extraction jobs for selected group documents. + """ + user_id = get_current_user_id() + if not user_id: + return jsonify({'error': 'User not authenticated'}), 401 + + settings = get_settings() + if not settings.get('enable_extract_meta_data'): + return jsonify({'error': 'Metadata extraction not enabled'}), 403 + + active_group_id, group_doc, _, error_response = _require_active_group_document_context( + user_id, + allowed_roles=("Owner", "Admin", "DocumentManager"), + permission_message='You do not have permission to extract metadata for group documents', + ) + if error_response: + return error_response + + allowed, reason = check_group_status_allows_operation(group_doc, 'upload') + if not allowed: + return jsonify({'error': reason}), 403 + + payload = request.get_json(silent=True) or {} + document_ids = payload.get('document_ids') + if not isinstance(document_ids, list): + document_id = payload.get('document_id') + document_ids = [document_id] if document_id else [] + document_ids = list(dict.fromkeys( + str(document_id).strip() + for document_id in document_ids + if str(document_id or '').strip() + )) + if not document_ids: + return jsonify({'error': 'At least one document ID is required.'}), 400 + + queued = [] + errors = [] + for document_id in document_ids: + try: + document_item = get_document_metadata( + document_id=document_id, + user_id=user_id, + group_id=active_group_id, + ) + if not document_item: + errors.append({'document_id': document_id, 'error': 'Document not found.'}) + continue + if document_item.get('group_id') != active_group_id: + errors.append({'document_id': document_id, 'error': 'Only documents in the active group can have metadata extracted.'}) + continue + + current_app.extensions['executor'].submit_stored( + f"{document_id}_group_metadata", + process_metadata_extraction_background, + document_id=document_id, + user_id=user_id, + group_id=active_group_id + ) + queued.append({'document_id': document_id}) + except Exception as e: + errors.append({'document_id': document_id, 'error': str(e)}) + + if queued: + invalidate_group_search_cache(active_group_id) + + status_code = 202 if queued and not errors else (207 if queued else 400) + return jsonify({ + 'message': f'Queued {len(queued)} document(s) for metadata extraction.', + 'queued': queued, + 'errors': errors, + }), status_code + @bp.route('/api/group_documents/reprocess_extraction', methods=['POST']) @swagger_route(security=get_auth_security()) @login_required diff --git a/application/single_app/route_backend_public_documents.py b/application/single_app/route_backend_public_documents.py index f9da07f43..f2db5771c 100644 --- a/application/single_app/route_backend_public_documents.py +++ b/application/single_app/route_backend_public_documents.py @@ -1019,6 +1019,81 @@ def api_extract_metadata_public_document(doc_id): executor.submit(process_metadata_extraction_background, document_id=doc_id, user_id=user_id, public_workspace_id=active_ws) return jsonify({'message':'Extraction queued'}), 200 + @bp.route('/api/public_documents/extract_metadata', methods=['POST']) + @swagger_route(security=get_auth_security()) + @login_required + @user_required + @enabled_required('enable_public_workspaces') + def api_extract_metadata_public_documents_batch(): + user_id = get_current_user_id() + if not user_id: + return jsonify({'error': 'User not authenticated'}), 401 + + settings = get_settings() + if not settings.get('enable_extract_meta_data'): + return jsonify({'error': 'Metadata extraction not enabled'}), 403 + + active_ws, ws_doc, _role, error_response = _require_active_public_workspace_response( + user_id, + PUBLIC_WORKSPACE_MANAGER_ROLES, + ) + if error_response: + return error_response + + allowed, reason = check_public_workspace_status_allows_operation(ws_doc, 'upload') + if not allowed: + return jsonify({'error': reason}), 403 + + payload = request.get_json(silent=True) or {} + document_ids = payload.get('document_ids') + if not isinstance(document_ids, list): + doc_id = payload.get('document_id') + document_ids = [doc_id] if doc_id else [] + document_ids = list(dict.fromkeys( + str(document_id).strip() + for document_id in document_ids + if str(document_id or '').strip() + )) + if not document_ids: + return jsonify({'error': 'At least one document ID is required.'}), 400 + + queued = [] + errors = [] + for document_id in document_ids: + try: + document_item = get_document_metadata( + document_id=document_id, + user_id=user_id, + public_workspace_id=active_ws, + ) + if not document_item: + errors.append({'document_id': document_id, 'error': 'Document not found.'}) + continue + if document_item.get('public_workspace_id') != active_ws: + errors.append({'document_id': document_id, 'error': 'Only documents in the active public workspace can have metadata extracted.'}) + continue + + current_app.extensions['executor'].submit_stored( + f"{document_id}_public_metadata", + process_metadata_extraction_background, + document_id=document_id, + user_id=user_id, + public_workspace_id=active_ws + ) + queued.append({'document_id': document_id}) + except Exception as e: + errors.append({'document_id': document_id, 'error': str(e)}) + + if queued: + invalidate_public_workspace_search_cache(active_ws) + + status_code = 202 if queued and not errors else (207 if queued else 400) + return jsonify({ + 'message': f'Queued {len(queued)} document(s) for metadata extraction.', + 'queued': queued, + 'errors': errors, + }), status_code + @bp.route('/api/public_documents/reprocess_extraction', methods=['POST']) @swagger_route(security=get_auth_security()) @login_required diff --git a/application/single_app/static/js/public/public_workspace.js b/application/single_app/static/js/public/public_workspace.js index 3a81ca311..947b0640d 100644 --- a/application/single_app/static/js/public/public_workspace.js +++ b/application/single_app/static/js/public/public_workspace.js @@ -81,6 +81,60 @@ function getPublicDeleteModalContent(documentCount) { }; } +async function requestPublicSelectedMetadataExtraction(documentIds) { + const response = await fetch('/api/public_documents/extract_metadata', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ document_ids: documentIds }), + }); + const data = await response.json().catch(() => ({})); + if (!response.ok && !(Array.isArray(data.queued) && data.queued.length > 0)) { + throw new Error(data.error || data.message || 'Unable to queue metadata extraction.'); + } + return data; +} + +function showPublicSelectedMetadataExtractionResult(data) { + const queuedCount = Array.isArray(data.queued) ? data.queued.length : 0; + const errorCount = Array.isArray(data.errors) ? data.errors.length : 0; + const message = errorCount > 0 + ? `Queued metadata extraction for ${queuedCount} document(s); ${errorCount} item(s) were skipped.` + : (data.message || `Queued metadata extraction for ${queuedCount} document(s).`); + showPublicWorkspaceToast(message, errorCount > 0 ? 'warning' : 'success'); +} + +async function extractPublicSelectedMetadata() { + const documentIds = Array.from(publicSelectedDocuments); + if (documentIds.length === 0) { + return; + } + if (!(window.enable_extract_meta_data === true || window.enable_extract_meta_data === 'true')) { + showPublicWorkspaceToast('Metadata extraction is not enabled.', 'info'); + return; + } + + const extractMetadataBtn = document.getElementById('public-extract-selected-metadata-btn'); + if (extractMetadataBtn) { + extractMetadataBtn.disabled = true; + extractMetadataBtn.innerHTML = 'Extracting...'; + } + + try { + const data = await requestPublicSelectedMetadataExtraction(documentIds); + showPublicSelectedMetadataExtractionResult(data); + publicSelectedDocuments.clear(); + syncPublicSelectionModeUI(); + fetchPublicDocs(); + } catch (error) { + showPublicWorkspaceToast(error.message, 'danger'); + } finally { + if (extractMetadataBtn) { + extractMetadataBtn.disabled = false; + extractMetadataBtn.innerHTML = 'Extract Metadata'; + } + } +} + function showPublicDocumentDeleteFeedback(message, variant = 'danger') { if (typeof window.showToast === 'function') { window.showToast(message, variant); @@ -614,11 +668,13 @@ document.addEventListener('DOMContentLoaded', ()=>{ const publicDownloadSelectedBtn = document.getElementById('public-download-selected-btn'); const publicClearSelectionBtn = document.getElementById('public-clear-selection-btn'); const publicChatSelectedBtn = document.getElementById('public-chat-selected-btn'); + const publicExtractSelectedMetadataBtn = document.getElementById('public-extract-selected-metadata-btn'); if (publicDeleteSelectedBtn) publicDeleteSelectedBtn.addEventListener('click', deletePublicSelectedDocuments); if (publicDownloadSelectedBtn) publicDownloadSelectedBtn.addEventListener('click', downloadPublicSelectedDocuments); if (publicClearSelectionBtn) publicClearSelectionBtn.addEventListener('click', clearPublicSelection); if (publicChatSelectedBtn) publicChatSelectedBtn.addEventListener('click', chatWithPublicSelected); + if (publicExtractSelectedMetadataBtn) publicExtractSelectedMetadataBtn.addEventListener('click', extractPublicSelectedMetadata); document.getElementById('public-toggle-selection-btn')?.addEventListener('click', togglePublicSelectionMode); document.addEventListener('click', handlePublicDocumentCardClick); }); @@ -2227,17 +2283,22 @@ function updatePublicBulkActionButtons() { const deleteBtn = document.getElementById('public-delete-selected-btn'); const downloadBtn = document.getElementById('public-download-selected-btn'); const reprocessDropdown = document.getElementById('public-reprocess-selected-dropdown'); + const extractMetadataBtn = document.getElementById('public-extract-selected-metadata-btn'); if (publicSelectedDocuments.size > 0) { if (bulkActionsBar) bulkActionsBar.style.display = 'block'; if (selectedCountSpan) selectedCountSpan.textContent = publicSelectedDocuments.size; const canManage = ['Owner', 'Admin', 'DocumentManager'].includes(userRoleInActivePublic); + const canModify = canManage && (window.currentPublicStatus || 'active') === 'active'; + const metadataEnabled = window.enable_extract_meta_data === true || window.enable_extract_meta_data === 'true'; if (deleteBtn) deleteBtn.style.display = canManage ? 'inline-block' : 'none'; if (downloadBtn) downloadBtn.classList.toggle('d-none', !publicFileDownloadsEnabled); if (reprocessDropdown) reprocessDropdown.classList.toggle('d-none', !canManage); + if (extractMetadataBtn) extractMetadataBtn.classList.toggle('d-none', !(canModify && metadataEnabled)); } else { if (bulkActionsBar) bulkActionsBar.style.display = 'none'; if (downloadBtn) downloadBtn.classList.add('d-none'); + if (extractMetadataBtn) extractMetadataBtn.classList.add('d-none'); } } @@ -2511,6 +2572,7 @@ window.clearPublicSelection = clearPublicSelection; window.chatWithPublicSelected = chatWithPublicSelected; window.reprocessPublicDocumentExtraction = reprocessPublicDocumentExtraction; window.reprocessPublicSelectedDocumentExtraction = reprocessPublicSelectedDocumentExtraction; +window.extractPublicSelectedMetadata = extractPublicSelectedMetadata; // Prompts function canManagePublicPrompts() { diff --git a/application/single_app/static/js/workspace/workspace-documents.js b/application/single_app/static/js/workspace/workspace-documents.js index ddabc09dd..76c53f539 100644 --- a/application/single_app/static/js/workspace/workspace-documents.js +++ b/application/single_app/static/js/workspace/workspace-documents.js @@ -32,6 +32,7 @@ const docsSharedOnlyFilter = document.getElementById("docs-shared-only-filter"); const deleteSelectedBtn = document.getElementById("delete-selected-btn"); const downloadSelectedBtn = document.getElementById("download-selected-btn"); const chatSelectedBtn = document.getElementById("chat-selected-btn"); +const extractSelectedMetadataBtn = document.getElementById("extract-selected-metadata-btn"); const clearSelectionBtn = document.getElementById("clear-selection-btn"); const documentDeleteModalElement = document.getElementById("documentDeleteModal"); const documentDeleteModal = documentDeleteModalElement ? new bootstrap.Modal(documentDeleteModalElement) : null; @@ -55,6 +56,10 @@ function getDocumentConversationUrl(doc) { return ""; } +function isWorkspaceMetadataExtractionEnabled() { + return window.enable_extract_meta_data === true || window.enable_extract_meta_data === "true"; +} + function setDocumentConversationStatusElement(element, doc) { if (!element) { return; @@ -2575,6 +2580,60 @@ window.reprocessSelectedDocumentExtraction = async function(extractionMode) { } }; +async function requestSelectedDocumentMetadataExtraction(documentIds) { + const response = await fetch('/api/documents/extract_metadata', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ document_ids: documentIds }), + }); + const data = await response.json().catch(() => ({})); + if (!response.ok && !(Array.isArray(data.queued) && data.queued.length > 0)) { + throw new Error(data.error || data.message || 'Unable to queue metadata extraction.'); + } + return data; +} + +function showSelectedDocumentMetadataExtractionResult(data) { + const queuedCount = Array.isArray(data.queued) ? data.queued.length : 0; + const errorCount = Array.isArray(data.errors) ? data.errors.length : 0; + const message = errorCount > 0 + ? `Queued metadata extraction for ${queuedCount} document(s); ${errorCount} item(s) were skipped.` + : (data.message || `Queued metadata extraction for ${queuedCount} document(s).`); + + showToast(message, errorCount > 0 ? 'warning' : 'success'); +} + +window.extractSelectedMetadata = async function() { + const documentIds = Array.from(selectedDocuments); + if (documentIds.length === 0) { + return; + } + if (!isWorkspaceMetadataExtractionEnabled()) { + showToast("Metadata extraction is not enabled.", 'info'); + return; + } + + if (extractSelectedMetadataBtn) { + extractSelectedMetadataBtn.disabled = true; + extractSelectedMetadataBtn.innerHTML = 'Extracting...'; + } + + try { + const data = await requestSelectedDocumentMetadataExtraction(documentIds); + showSelectedDocumentMetadataExtractionResult(data); + selectedDocuments.clear(); + syncDocumentSelectionModeUI(); + fetchUserDocuments(); + } catch (error) { + showToast(error.message, 'danger'); + } finally { + if (extractSelectedMetadataBtn) { + extractSelectedMetadataBtn.disabled = false; + extractSelectedMetadataBtn.innerHTML = 'Extract Metadata'; + } + } +}; + // Make fetchUserDocuments globally available for workspace-init.js window.fetchUserDocuments = fetchUserDocuments; @@ -2610,6 +2669,7 @@ function updateBulkActionButtons() { const bulkActionsBar = document.getElementById('bulkActionsBar'); const selectedCountSpan = document.getElementById('selectedCount'); const downloadBtn = document.getElementById('download-selected-btn'); + const extractMetadataBtn = document.getElementById('extract-selected-metadata-btn'); if (selectedDocuments.size > 0) { // Show bulk actions bar with count @@ -2623,7 +2683,10 @@ function updateBulkActionButtons() { if (downloadBtn) { downloadBtn.classList.toggle('d-none', !personalWorkspaceFileDownloadsEnabled); } - + if (extractMetadataBtn) { + extractMetadataBtn.classList.toggle('d-none', !isWorkspaceMetadataExtractionEnabled()); + } + } else { // Hide bulk actions bar if (bulkActionsBar) { @@ -2633,6 +2696,9 @@ function updateBulkActionButtons() { if (downloadBtn) { downloadBtn.classList.add('d-none'); } + if (extractMetadataBtn) { + extractMetadataBtn.classList.add('d-none'); + } } } @@ -2799,7 +2865,11 @@ document.addEventListener('DOMContentLoaded', function() { if (chatSelectedBtn) { chatSelectedBtn.addEventListener('click', window.chatWithSelected); } - + + if (extractSelectedMetadataBtn) { + extractSelectedMetadataBtn.addEventListener('click', window.extractSelectedMetadata); + } + // Clear selection button if (clearSelectionBtn) { clearSelectionBtn.addEventListener('click', window.clearDocumentSelection); diff --git a/application/single_app/templates/group_workspaces.html b/application/single_app/templates/group_workspaces.html index 5cb0d7b6f..d1b91c6f2 100644 --- a/application/single_app/templates/group_workspaces.html +++ b/application/single_app/templates/group_workspaces.html @@ -873,6 +873,9 @@

No active group selected

    • + + + diff --git a/functional_tests/test_multiselect_metadata_extraction.py b/functional_tests/test_multiselect_metadata_extraction.py new file mode 100644 index 000000000..9eada3e6b --- /dev/null +++ b/functional_tests/test_multiselect_metadata_extraction.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +# test_multiselect_metadata_extraction.py +""" +Functional test for multi-select metadata extraction. +Version: 0.250.106 +Implemented in: 0.250.106 + +This test ensures personal, group, and public workspace multi-select actions +can queue metadata extraction and that the shared extraction path updates +document titles. +""" + +import ast +import re +import sys +from pathlib import Path + + +ROOT_DIR = Path(__file__).resolve().parents[1] +SINGLE_APP_DIR = ROOT_DIR / "application" / "single_app" +CONFIG_FILE = SINGLE_APP_DIR / "config.py" +FUNCTIONS_DOCUMENTS_FILE = SINGLE_APP_DIR / "functions_documents.py" +PERSONAL_ROUTE_FILE = SINGLE_APP_DIR / "route_backend_documents.py" +GROUP_ROUTE_FILE = SINGLE_APP_DIR / "route_backend_group_documents.py" +PUBLIC_ROUTE_FILE = SINGLE_APP_DIR / "route_backend_public_documents.py" +WORKSPACE_TEMPLATE_FILE = SINGLE_APP_DIR / "templates" / "workspace.html" +GROUP_TEMPLATE_FILE = SINGLE_APP_DIR / "templates" / "group_workspaces.html" +PUBLIC_TEMPLATE_FILE = SINGLE_APP_DIR / "templates" / "public_workspaces.html" +WORKSPACE_JS_FILE = SINGLE_APP_DIR / "static" / "js" / "workspace" / "workspace-documents.js" +PUBLIC_JS_FILE = SINGLE_APP_DIR / "static" / "js" / "public" / "public_workspace.js" +EXPECTED_VERSION = "0.250.106" + + +ROUTE_CASES = [ + { + "name": "personal", + "file": PERSONAL_ROUTE_FILE, + "function": "api_extract_user_metadata_batch", + "path": "/api/documents/extract_metadata", + "enabled_setting": "enable_user_workspace", + "scope_checks": [ + "get_document_metadata(document_id=document_id, user_id=user_id)", + "document_item.get('user_id') != user_id", + "invalidate_personal_search_cache(user_id)", + ], + }, + { + "name": "group", + "file": GROUP_ROUTE_FILE, + "function": "api_extract_group_metadata_batch", + "path": "/api/group_documents/extract_metadata", + "enabled_setting": "enable_group_workspaces", + "scope_checks": [ + "_require_active_group_document_context", + "check_group_status_allows_operation(group_doc, 'upload')", + "group_id=active_group_id", + "invalidate_group_search_cache(active_group_id)", + ], + }, + { + "name": "public", + "file": PUBLIC_ROUTE_FILE, + "function": "api_extract_metadata_public_documents_batch", + "path": "/api/public_documents/extract_metadata", + "enabled_setting": "enable_public_workspaces", + "scope_checks": [ + "_require_active_public_workspace_response", + "check_public_workspace_status_allows_operation(ws_doc, 'upload')", + "public_workspace_id=active_ws", + "invalidate_public_workspace_search_cache(active_ws)", + ], + }, +] + + +def read_file(path): + """Read a UTF-8 text file from the repository.""" + return path.read_text(encoding="utf-8") + + +def parse_file(path): + """Parse a Python file into an AST and return source text too.""" + source = read_file(path) + return ast.parse(source, filename=str(path)), source + + +def dotted_name(node): + """Return a dotted name for AST name, call, and attribute nodes.""" + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + parent = dotted_name(node.value) + return f"{parent}.{node.attr}" if parent else node.attr + if isinstance(node, ast.Call): + return dotted_name(node.func) + return "" + + +def route_path(route_decorator): + """Return the literal route path from a Flask route decorator.""" + if route_decorator.args and isinstance(route_decorator.args[0], ast.Constant): + return str(route_decorator.args[0].value) + return "" + + +def get_function(module_ast, function_name): + """Return a named function from a parsed module.""" + matches = [ + node for node in ast.walk(module_ast) + if isinstance(node, ast.FunctionDef) and node.name == function_name + ] + assert matches, f"Missing function: {function_name}" + return matches[0] + + +def get_route_decorator(function_node): + """Return the Flask route decorator from a route function.""" + route_decorators = [ + decorator for decorator in function_node.decorator_list + if isinstance(decorator, ast.Call) and dotted_name(decorator.func).endswith(".route") + ] + assert route_decorators, f"Missing route decorator on {function_node.name}" + return route_decorators[0] + + +def decorator_names(function_node): + """Return decorator names from a function.""" + return tuple( + dotted_name(decorator.func if isinstance(decorator, ast.Call) else decorator) + for decorator in function_node.decorator_list + ) + + +def test_batch_metadata_routes_are_registered_and_secured(): + """Verify all workspace batch routes are present and decorated.""" + print("Testing batch metadata route registration...") + + for route_case in ROUTE_CASES: + module_ast, _source = parse_file(route_case["file"]) + function_node = get_function(module_ast, route_case["function"]) + route_decorator = get_route_decorator(function_node) + names = decorator_names(function_node) + + assert route_path(route_decorator) == route_case["path"], ( + f"{route_case['name']} route path mismatch" + ) + assert "swagger_route" in names, f"{route_case['name']} route missing swagger_route" + assert "login_required" in names, f"{route_case['name']} route missing login_required" + assert "user_required" in names, f"{route_case['name']} route missing user_required" + assert "enabled_required" in names, f"{route_case['name']} route missing enabled_required" + assert route_case["enabled_setting"] in ast.unparse(function_node), ( + f"{route_case['name']} route missing expected feature flag" + ) + + print("Batch metadata route registration passed") + return True + + +def test_batch_metadata_routes_queue_authorized_background_jobs(): + """Verify routes parse selected IDs, validate scope, and queue extraction.""" + print("Testing batch metadata route queueing...") + + for route_case in ROUTE_CASES: + module_ast, _source = parse_file(route_case["file"]) + function_source = ast.unparse(get_function(module_ast, route_case["function"])) + + assert "document_ids" in function_source, f"{route_case['name']} route must accept document_ids" + assert "queued = []" in function_source, f"{route_case['name']} route must report queued docs" + assert "errors = []" in function_source, f"{route_case['name']} route must report skipped docs" + assert "process_metadata_extraction_background" in function_source, ( + f"{route_case['name']} route must use shared metadata extraction background job" + ) + assert "submit_stored" in function_source, ( + f"{route_case['name']} route must track queued background jobs" + ) + for expected_snippet in route_case["scope_checks"]: + assert expected_snippet in function_source, ( + f"{route_case['name']} route missing scope check: {expected_snippet}" + ) + + print("Batch metadata route queueing passed") + return True + + +def test_bulk_metadata_ui_wiring_exists_for_all_workspaces(): + """Verify multi-select bars expose metadata extraction and call batch routes.""" + print("Testing bulk metadata UI wiring...") + + workspace_template = read_file(WORKSPACE_TEMPLATE_FILE) + group_template = read_file(GROUP_TEMPLATE_FILE) + public_template = read_file(PUBLIC_TEMPLATE_FILE) + workspace_js = read_file(WORKSPACE_JS_FILE) + public_js = read_file(PUBLIC_JS_FILE) + + assert 'id="extract-selected-metadata-btn"' in workspace_template + assert "window.extractSelectedMetadata" in workspace_js + assert "/api/documents/extract_metadata" in workspace_js + + assert 'id="group-extract-selected-metadata-btn"' in group_template + assert "extractGroupSelectedMetadata" in group_template + assert "/api/group_documents/extract_metadata" in group_template + + assert 'id="public-extract-selected-metadata-btn"' in public_template + assert "extractPublicSelectedMetadata" in public_js + assert "/api/public_documents/extract_metadata" in public_js + + print("Bulk metadata UI wiring passed") + return True + + +def test_metadata_extraction_updates_title(): + """Verify metadata extraction persists title updates to documents and chunks.""" + print("Testing metadata title update path...") + + module_ast, source = parse_file(FUNCTIONS_DOCUMENTS_FILE) + background_function = ast.unparse(get_function(module_ast, "process_metadata_extraction_background")) + final_metadata_function = ast.unparse(get_function(module_ast, "_run_final_metadata_extraction")) + update_document_function = ast.unparse(get_function(module_ast, "update_document")) + + assert '"title": metadata.get(\'title\')' in source, ( + "Manual metadata extraction must pass title into update_document" + ) + assert "document_metadata.items()" in final_metadata_function, ( + "Final metadata extraction should not exclude title from update fields" + ) + assert "update_callback(**update_fields)" in final_metadata_function, ( + "Final metadata extraction must persist extracted fields" + ) + assert "'title', 'authors', 'file_name', 'document_classification', 'tags'" in source, ( + "Document updates must mark title changes for chunk sync" + ) + assert "chunk_updates['title'] = existing_document.get('title')" in update_document_function, ( + "Title updates must propagate to search chunks" + ) + + print("Metadata title update path passed") + return True + + +def test_config_version_bumped_for_multiselect_metadata_extraction(): + """Verify config.py version was bumped for this change.""" + print("Testing config version bump...") + + config_source = read_file(CONFIG_FILE) + version_match = re.search(r'VERSION = "([0-9.]+)"', config_source) + assert version_match, "Could not find VERSION in config.py" + assert version_match.group(1) == EXPECTED_VERSION, ( + f"Expected config.py version {EXPECTED_VERSION}" + ) + + print("Config version bump passed") + return True + + +if __name__ == "__main__": + tests = [ + test_batch_metadata_routes_are_registered_and_secured, + test_batch_metadata_routes_queue_authorized_background_jobs, + test_bulk_metadata_ui_wiring_exists_for_all_workspaces, + test_metadata_extraction_updates_title, + test_config_version_bumped_for_multiselect_metadata_extraction, + ] + + results = [] + for test in tests: + print(f"Running {test.__name__}...") + try: + results.append(test()) + except Exception as test_error: + print(f"Test failed: {test_error}") + import traceback + traceback.print_exc() + results.append(False) + + success = all(results) + print(f"Results: {sum(results)}/{len(results)} tests passed") + sys.exit(0 if success else 1) From 4316f9caf47a52f75ab64e7bcbf91503fd4f5996 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Fri, 31 Jul 2026 09:48:53 -0400 Subject: [PATCH 2/2] Update release notes for multiselect metadata extraction Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/explanation/release_notes.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index b902b880e..141b4f6ed 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,15 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](/explanation/features/) and [Fixes by Version](/explanation/fixes/). +### **(v0.250.106)** + +#### New Features + +* **Multi-Select Metadata Extraction** + * Personal, group, and public workspace document multi-select bars now include an **Extract Metadata** action when metadata extraction is enabled. + * Selected documents are queued through the shared metadata extraction background workflow, preserving generated titles along with authors, abstracts, keywords, publication dates, and organization metadata. + * (Ref: Closes #1134, `route_backend_documents.py`, `route_backend_group_documents.py`, `route_backend_public_documents.py`, workspace document multi-select actions) + ### **(v0.250.105)** #### User Interface Enhancements