diff --git a/application/single_app/config.py b/application/single_app/config.py index f1905806..e5e7ad5e 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -95,7 +95,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.250.107" +VERSION = "0.250.108" IS_DEVELOPMENT = is_development_env_enabled() SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') diff --git a/application/single_app/functions_data_management.py b/application/single_app/functions_data_management.py index afa328ce..c6aba31b 100644 --- a/application/single_app/functions_data_management.py +++ b/application/single_app/functions_data_management.py @@ -259,7 +259,7 @@ DATA_MANAGEMENT_MIGRATION_MODE_DELTA_UPSERT, DATA_MANAGEMENT_MIGRATION_MODE_MIRROR, } -DATA_MANAGEMENT_MIRROR_CONFIRMATION = "MIRROR WITH DELETIONS" +DATA_MANAGEMENT_MIRROR_CONFIRMATION = "MAKE DESTINATION MATCH SOURCE" DATA_MANAGEMENT_RESTORE_POLICY_CREATE_ONLY = "create_only" DATA_MANAGEMENT_RESTORE_POLICY_OVERWRITE = "overwrite_existing" DATA_MANAGEMENT_RESTORE_POLICIES = { @@ -1107,29 +1107,62 @@ def test_target_cosmos_connection(settings=None, migration_plan=None): normalized_settings, normalized_plan, ) - capacity = None - if normalized_settings.get("migration_temporary_destination_ru_enabled"): - inspected_capacity = _inspect_target_cosmos_migration_capacity( - normalized_settings, - normalized_plan, - ) - capacity = { - "target_ru": inspected_capacity.get("target_ru"), - "database_mode": inspected_capacity.get("database_mode"), - "database_current_ru": inspected_capacity.get("database_current_ru"), - "targets": inspected_capacity.get("targets"), - } result = { "success": True, "target": "cosmos", "database_name": properties.get("id") or DATA_MANAGEMENT_TARGET_COSMOS_DATABASE_NAME, "authentication_type": normalized_settings.get("target_cosmos_authentication_type"), "migration_access": migration_access, - "capacity": capacity, } return result +def test_target_cosmos_capacity_management(settings=None, migration_plan=None): + """Validate destination Cosmos ARM throughput permissions for RU Boost.""" + normalized_settings = _normalize_data_management_settings_from_payload(settings) + normalized_plan = normalize_data_management_migration_plan({ + "migration_plan": migration_plan if isinstance(migration_plan, dict) else {}, + }) + inspected_capacity = _inspect_target_cosmos_migration_capacity( + normalized_settings, + normalized_plan, + ) + write_results = [] + for target in inspected_capacity.get("targets") or []: + current_ru = _safe_int(target.get("current_ru"), default=0, minimum=0) + if not current_ru: + raise DataManagementSettingsValidationError( + "Destination Cosmos RU Boost test could not determine the current RU/s value." + ) + scale_result = set_database_throughput( + inspected_capacity["management_settings"], + current_ru, + initiated_by="data_management_ru_boost_test", + reason="validate_data_management_ru_boost_permissions", + decision={ + "scope": target.get("scope"), + "container_name": target.get("container_name") or "", + "target_mode": target.get("mode"), + }, + ) + write_results.append({ + "scope": target.get("scope"), + "container_name": target.get("container_name") or "", + "mode": target.get("mode"), + "current_ru": current_ru, + "write_verified": True, + "verified_ru": scale_result.get("to_ru", current_ru), + }) + return { + "success": True, + "target": "cosmos_ru_boost", + "target_ru": inspected_capacity.get("target_ru"), + "database_mode": inspected_capacity.get("database_mode"), + "database_current_ru": inspected_capacity.get("database_current_ru"), + "targets": write_results, + } + + def test_target_search_connection(settings=None): normalized_settings = _normalize_data_management_settings_from_payload(settings) endpoint = _safe_text(normalized_settings.get("target_ai_search_endpoint")) @@ -1786,7 +1819,7 @@ def _validate_incremental_baseline_job(candidate_job, current_job, current_confi source_cutoff_at = _safe_text(candidate_state.get("source_cutoff_at")) if _parse_iso_datetime(source_cutoff_at) is None: raise DataManagementSettingsValidationError( - "Incremental migration baseline does not contain a valid source watermark." + "Previous migration does not contain a valid source checkpoint." ) return source_cutoff_at diff --git a/application/single_app/route_backend_data_management.py b/application/single_app/route_backend_data_management.py index 9959e047..d5d97dfb 100644 --- a/application/single_app/route_backend_data_management.py +++ b/application/single_app/route_backend_data_management.py @@ -54,6 +54,7 @@ summarize_data_management_migration_plan, submit_data_management_job, test_backup_storage_connection, + test_target_cosmos_capacity_management, test_target_cosmos_connection, test_target_enhanced_citation_storage_connection, test_target_search_connection, @@ -260,6 +261,38 @@ def test_admin_data_management_target_cosmos(): return jsonify({"success": False, "error": "Target Cosmos connection test failed."}), 400 return jsonify(result), 200 + @bp.route("/api/admin/data-management/target/cosmos/ru-boost/test", methods=["POST"]) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def test_admin_data_management_target_cosmos_ru_boost(): + payload = request.get_json(silent=True) or {} + settings_payload = payload.get("settings") if isinstance(payload.get("settings"), dict) else None + migration_plan = payload.get("migration_plan") if isinstance(payload.get("migration_plan"), dict) else None + try: + result = test_target_cosmos_capacity_management( + settings=settings_payload, + migration_plan=migration_plan, + ) + except DataManagementSettingsValidationError as exc: + log_event( + "[DataManagement] Target Cosmos RU Boost permission test validation failed.", + {"error": str(exc)}, + level=logging.WARNING, + ) + return jsonify({ + "success": False, + "error": "Target Cosmos RU Boost permission test request is invalid.", + }), 400 + except Exception as exc: + log_event( + "[DataManagement] Target Cosmos RU Boost permission test failed.", + {"error": str(exc)}, + level=logging.WARNING, + ) + return jsonify({"success": False, "error": "Target Cosmos RU Boost permission test failed."}), 400 + return jsonify(result), 200 + @bp.route("/api/admin/data-management/target/search/test", methods=["POST"]) @swagger_route(security=get_auth_security()) @login_required @@ -736,6 +769,25 @@ def get_admin_data_management_migration_catalog(target_type): return jsonify({"success": False, "error": str(exc)}), 400 return jsonify({"success": True, **catalog}), 200 + @bp.route("/api/admin/data-management/restore/review", methods=["POST"]) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def review_admin_data_management_restore(): + payload = request.get_json(silent=True) or {} + restore_plan = payload.get("restore_plan") if isinstance(payload.get("restore_plan"), dict) else {} + try: + review = review_data_management_restore(restore_plan) + except Exception as exc: + log_event( + "[DataManagement] Restore review failed.", + {"error": str(exc)}, + level=logging.ERROR, + exceptionTraceback=True, + ) + return jsonify({"success": False, "error": "Restore review could not be completed."}), 400 + return jsonify({"success": True, "review": review}), 200 + @bp.route("/api/admin/data-management/migration/summary", methods=["POST"]) @swagger_route(security=get_auth_security()) @login_required diff --git a/application/single_app/static/css/styles.css b/application/single_app/static/css/styles.css index 65230c0f..5e35e2b0 100644 --- a/application/single_app/static/css/styles.css +++ b/application/single_app/static/css/styles.css @@ -1732,6 +1732,17 @@ main { border-top: 1px solid var(--bs-border-color); } +.restore-stepper { + grid-template-columns: repeat(5, minmax(8rem, 1fr)); + margin: 0; + border: 1px solid var(--bs-border-color); + border-radius: var(--bs-border-radius-lg); +} + +.restore-stage { + min-height: 24rem; +} + @media (max-width: 991.98px) { .migration-stepper { grid-auto-flow: column; diff --git a/application/single_app/static/js/admin/admin_data_management.js b/application/single_app/static/js/admin/admin_data_management.js index 61f3a975..3e0bd9c8 100644 --- a/application/single_app/static/js/admin/admin_data_management.js +++ b/application/single_app/static/js/admin/admin_data_management.js @@ -7,7 +7,7 @@ const backupStorageAuthManagedIdentity = "managed_identity"; const backupStorageAuthConnectionString = "connection_string"; const targetCosmosDatabaseName = "SimpleChat"; const cosmosEditorConfirmationPhrase = "I understand this can damage system data"; -const migrationMirrorConfirmationPhrase = "MIRROR WITH DELETIONS"; +const migrationMirrorConfirmationPhrase = "MAKE DESTINATION MATCH SOURCE"; const restoreOverwriteConfirmationPhrase = "RESTORE WITH OVERWRITE"; const elements = {}; let dataManagementModified = false; @@ -67,7 +67,6 @@ const migrationWorkflowState = { submissionAccepted: false, currentJob: null, }; - document.addEventListener("DOMContentLoaded", () => { bindElements(); if (!elements.tabPane) { @@ -152,6 +151,7 @@ function bindElements() { "data_management_target_cosmos_subscription_id", "data_management_target_cosmos_resource_group", "data-management-test-target-cosmos-btn", + "data-management-test-target-cosmos-ru-boost-btn", "data-management-target-ai-search-section", "data_management_target_ai_search_auth", "data_management_target_ai_search_endpoint", @@ -330,6 +330,7 @@ function bindEvents() { elements.dataManagementGenerateKeyBtn?.addEventListener("click", generateEncryptionKey); elements.dataManagementTestStorageBtn?.addEventListener("click", testBackupStorage); elements.dataManagementTestTargetCosmosBtn?.addEventListener("click", testTargetCosmos); + elements.dataManagementTestTargetCosmosRuBoostBtn?.addEventListener("click", testTargetCosmosRuBoost); elements.dataManagementTestMigrationAccessBtn?.addEventListener("click", testMigrationAccess); elements.dataManagementTestTargetSearchBtn?.addEventListener("click", testTargetSearch); elements.dataManagementTestTargetEcStorageBtn?.addEventListener("click", testTargetEnhancedCitationStorage); @@ -844,9 +845,16 @@ function setMigrationTargetVisibility() { function updateMigrationCapacityVisibility() { const enabled = Boolean(elements.datamanagementmigrationtemporarydestinationruenabled?.checked); setElementVisible(elements.dataManagementMigrationTemporaryRuField, enabled); - if (elements.datamanagementmigrationtemporarydestinationru) { - elements.datamanagementmigrationtemporarydestinationru.disabled = !enabled; - } + [ + elements.datamanagementmigrationtemporarydestinationru, + elements.datamanagementtargetcosmossubscriptionid, + elements.datamanagementtargetcosmosresourcegroup, + elements.dataManagementTestTargetCosmosRuBoostBtn, + ].forEach((element) => { + if (element) { + element.disabled = !enabled; + } + }); } function updateBackupCapacityVisibility() { @@ -1544,8 +1552,7 @@ async function testTargetCosmos() { }); const verifiedCount = Number(data.migration_access?.container_count || 0); const accessText = verifiedCount ? ` Verified ${formatNumber(verifiedCount)} planned container(s).` : ""; - const capacityText = data.capacity?.target_ru ? ` Temporary capacity can reach ${formatNumber(data.capacity.target_ru)} RU/s.` : ""; - setStatus(`Target Cosmos connection succeeded. Database: ${data.database_name || targetCosmosDatabaseName}.${accessText}${capacityText}`, "success"); + setStatus(`Target Cosmos data access succeeded. Database: ${data.database_name || targetCosmosDatabaseName}.${accessText}`, "success"); showToast("Target Cosmos connection succeeded.", "success"); } catch (error) { setStatus(error.message || "Target Cosmos connection test failed.", "danger"); @@ -1555,6 +1562,24 @@ async function testTargetCosmos() { } } +async function testTargetCosmosRuBoost() { + setBusy(elements.dataManagementTestTargetCosmosRuBoostBtn, true, "Testing..."); + try { + const data = await requestJson("/api/admin/data-management/target/cosmos/ru-boost/test", { + method: "POST", + body: JSON.stringify({ settings: collectSettings(), migration_plan: buildMigrationPlan() }), + }); + const targetCount = Array.isArray(data.targets) ? data.targets.length : 0; + setStatus(`RU Boost permission test succeeded. Verified ${formatNumber(targetCount)} destination capacity target(s) up to ${formatNumber(data.target_ru || 0)} RU/s.`, "success"); + showToast("RU Boost permission test succeeded.", "success"); + } catch (error) { + setStatus(error.message || "RU Boost permission test failed.", "danger"); + showToast(error.message || "RU Boost permission test failed.", "danger"); + } finally { + setBusy(elements.dataManagementTestTargetCosmosRuBoostBtn, false); + } +} + async function testMigrationAccess() { setBusy(elements.dataManagementTestMigrationAccessBtn, true, "Validating..."); try { @@ -1563,8 +1588,7 @@ async function testMigrationAccess() { body: JSON.stringify({ settings: collectSettings(), migration_plan: buildMigrationPlan() }), }); const verifiedCount = Number(data.migration_access?.container_count || 0); - const capacityText = data.capacity?.target_ru ? ` Destination capacity management is ready up to ${formatNumber(data.capacity.target_ru)} RU/s.` : ""; - setStatus(`Cosmos migration access validation succeeded. ${formatNumber(verifiedCount)} planned Cosmos container(s) can be read and written.${capacityText}`, "success"); + setStatus(`Cosmos data-copy access validation succeeded. ${formatNumber(verifiedCount)} planned Cosmos container(s) can be read and written. Use Test RU Boost for destination capacity permissions.`, "success"); showToast("Cosmos migration access validation succeeded.", "success"); } catch (error) { setStatus(error.message || "Cosmos migration access validation failed.", "danger"); @@ -1699,7 +1723,7 @@ async function queueMigration(dryRun) { migrationPlan.migration_mode === "mirror_with_deletions" && migrationPlan.mirror_confirmation !== migrationMirrorConfirmationPhrase ) { - const message = `Type ${migrationMirrorConfirmationPhrase} before running a mirror migration.`; + const message = `Type ${migrationMirrorConfirmationPhrase} before running this destination cleanup.`; elements.datamanagementmigrationmirrorconfirmationphrase?.classList.add("is-invalid"); elements.datamanagementmigrationmirrorconfirmationphrase?.focus(); setStatus(message, "danger"); @@ -2241,8 +2265,8 @@ function updateMigrationModeVisibility() { elements.dataManagementMigrationMirrorConfirmation?.classList.toggle("d-none", !isMirror); const descriptions = { new_only: "Copies source items that are absent from the destination. Existing destination data is never updated or deleted.", - delta_upsert: "Copies new items and updates changed migration-owned items since the prior successful watermark. Destination-only data is retained.", - mirror_with_deletions: "Runs delta/upsert, then deletes destination-only items that carry successful SimpleChat migration ownership. Unowned data is retained.", + delta_upsert: "Copies new items and updates changed migration-owned items from the previous completed migration. Destination-only data is retained.", + mirror_with_deletions: "Catches up changed items, then removes destination-only items that were created by SimpleChat migration. Unowned data is retained.", }; if (elements.dataManagementMigrationModeDescription) { elements.dataManagementMigrationModeDescription.textContent = descriptions[migrationMode] || descriptions.new_only; @@ -2398,6 +2422,13 @@ function createMigrationSummaryCard(targetType, targetSummary) { function renderMigrationReviewChecks(checks) { const container = elements.dataManagementMigrationReviewChecks; + if (!container) { + return; + } + renderDataManagementReviewChecks(container, checks); +} + +function renderDataManagementReviewChecks(container, checks) { if (!container) { return; } @@ -3086,7 +3117,7 @@ function createBackupActionCell(backup) { if (canRestore) { const restoreButton = document.createElement("button"); restoreButton.type = "button"; - restoreButton.className = "btn btn-outline-danger btn-sm ms-1"; + restoreButton.className = "btn btn-outline-warning btn-sm ms-1"; restoreButton.append(createIcon("bi bi-arrow-counterclockwise me-1"), document.createTextNode("Restore")); restoreButton.addEventListener("click", () => openRestoreModal(backup)); cell.appendChild(restoreButton); @@ -4469,6 +4500,14 @@ function formatStatusLabel(value) { } function formatActivityLabel(value) { + const friendlyLabels = { + new_only: "Copy Missing Items Only", + delta_upsert: "Catch Up Changed Items", + mirror_with_deletions: "Make Destination Match Source", + }; + if (friendlyLabels[value]) { + return friendlyLabels[value]; + } return String(value || "") .replace(/[_-]/g, " ") .replace(/\b\w/g, (letter) => letter.toUpperCase()); diff --git a/application/single_app/static/js/admin/admin_sidebar_nav.js b/application/single_app/static/js/admin/admin_sidebar_nav.js index 0b9c9d60..fbdb4f35 100644 --- a/application/single_app/static/js/admin/admin_sidebar_nav.js +++ b/application/single_app/static/js/admin/admin_sidebar_nav.js @@ -226,6 +226,7 @@ function scrollToSection(sectionId) { // Security tab sections 'keyvault-section': 'keyvault-section', // Data Management tab sections + 'data-management-readiness-section': 'data-management-readiness-section', 'data-management-backup-section': 'data-management-backup-section', 'data-management-schedule-section': 'data-management-schedule-section', 'data-management-storage-section': 'data-management-storage-section', diff --git a/application/single_app/templates/_sidebar_nav.html b/application/single_app/templates/_sidebar_nav.html index 17fa9151..d093fcd9 100644 --- a/application/single_app/templates/_sidebar_nav.html +++ b/application/single_app/templates/_sidebar_nav.html @@ -669,9 +669,14 @@
  • - Data Management + Backup, Migrate & Restore