diff --git a/app/application.ml b/app/application.ml index 2618ba2..f4cc7b0 100644 --- a/app/application.ml +++ b/app/application.ml @@ -1132,6 +1132,81 @@ let presentation_for_day _state day = Journal_calendar.present_journal_day day |> Result.to_option ;; +let report_app_feed ?request_generation ?feed ?complete ?row_revision ~stage ~reason before after = + try + let uuid value = match Logseq_db_types.Graph_types.Uuid.of_string value with + | Ok u -> `String (Logseq_db_types.Graph_types.Uuid.to_string u) + | Error _ -> `String "invalid" in + let roots timeline = Journal_timeline_state.fold_slots + (fun (count, sample) -> function + | Journal_timeline_state.Top_level e -> + count + 1, (if count < 16 then uuid (Journal_model.id e.block) :: sample else sample) + | _ -> count, sample) (0, []) timeline in + let count, sample = roots after.timeline in + let incoming = Option.fold ~none:[] ~some:(fun (f : Journal_graph_projection.feed) -> + List.concat_map (fun (d : Journal_graph_projection.day_feed) -> + List.map (fun (e : Journal_graph_projection.timeline_entry) -> e.block) d.entries) f.days) feed in + let pending = match Journal_timeline_state.pending_request before.timeline with + | None -> `Null + | Some (g, _) -> `String (Int64.to_string g) in + let fields = + [ "stage", `String stage; "reason", `String reason + ; "graph", Option.fold ~none:`Null ~some:(fun u -> `String (Logseq_db_types.Graph_types.Uuid.to_string u)) after.graph_state.graph_id + ; "selectedGraph", Option.fold ~none:`Null ~some:(fun u -> `String (Logseq_db_types.Graph_types.Uuid.to_string u)) (current_graph_generation after) + ; "graphGeneration", Option.fold ~none:`Null ~some:(fun (m : Graph_service.snapshot) -> `Int m.startup.graph_generation) after.manager + ; "accountGeneration", Option.fold ~none:`Null ~some:(fun (m : Graph_service.snapshot) -> `Int m.startup.account_generation) after.manager + ; "workerGraphGeneration", `Int after.graph_state.generation + ; "requestGeneration", Option.fold ~none:`Null ~some:(fun g -> `String (Int64.to_string g)) request_generation + ; "pendingGeneration", pending + ; "refreshGeneration", Option.fold ~none:`Null ~some:(fun (r : feed_refresh) -> `String (Int64.to_string r.generation)) before.feed_refresh + ; "refreshGraph", Option.fold ~none:`Null + ~some:(fun u -> `String (Logseq_db_types.Graph_types.Uuid.to_string u)) + (Option.bind before.feed_refresh (fun (r : feed_refresh) -> r.graph_generation)) + ; "refreshGraphMatches", Option.fold ~none:`Null + ~some:(fun (r : feed_refresh) -> `Bool (r.graph_generation = current_graph_generation before)) before.feed_refresh + ; "refreshContextMatches", (match before.feed_refresh, before.calendar with + | Some r, Some c -> `Bool (equal_feed_projection_context r.context (feed_projection_context c)) + | _ -> `Null) + ; "graphReady", `Bool before.graph_ready + ; "calendarAvailable", `Bool (Option.is_some before.calendar) + ; "day", Option.fold ~none:`Null ~some:(fun c -> `Int (Journal_calendar.local_day c)) after.calendar + ; "slotCount", `Int (Journal_timeline_state.retained_slot_count after.timeline) + ; "rootCount", `Int count; "roots", `List (List.rev sample) + ; "incomingRootCount", `Int (List.length incoming) + ; "incomingRoots", `List (List.take 16 incoming |> List.map (fun b -> uuid (Journal_model.id b))) + ; "timelineChanged", `Bool (before.timeline != after.timeline) + ; "truncated", `Bool (count > 16 || List.length incoming > 16) + ] @ Option.fold ~none:[] ~some:(fun v -> ["complete", `Bool v]) complete + @ Option.fold ~none:[] ~some:(fun v -> ["rowRevision", `Int v]) row_revision in + let text = Yojson.Safe.to_string (`Assoc fields) in + if String.length text <= 8192 then (output_string stderr ("journal-timeline: " ^ text ^ "\n"); flush stderr) + with _ -> () +;; + +let report_app_response before after (response : Journal_graph_runtime.response) = + try + match response.payload with + | Feed_refresh_started {request_generation} -> + let reason = match before.calendar, before.feed_refresh with + | _, Some r when r.generation > request_generation -> "discard-older-refresh" + | Some _, _ when before.graph_ready -> "accept-refresh" + | _ -> "discard-not-ready" in + report_app_feed ~request_generation ~stage:"app-feed-start" ~reason before after + | Feed_loaded {request_generation; feed; complete} -> + let reason = + if before.timeline != after.timeline then "accept-feed" + else match before.feed_refresh with + | Some r when r.generation = request_generation -> "discard-refresh-context" + | _ -> (match Journal_timeline_state.pending_request before.timeline with + | Some (g, Feed _) when g <> request_generation -> "discard-request-generation" + | Some (_, Day _) -> "discard-pending-day" + | None -> "discard-no-pending" + | Some (_, Feed _) -> "no-timeline-change") in + report_app_feed ~request_generation ~feed ~complete ~stage:"app-feed-result" ~reason before after + | _ -> () + with _ -> () +;; + let back_state state = let detail_root = Option.map Journal_detail.root (Journal_routes.detail state.routes) in let routes = Journal_routes.back state.routes in @@ -4345,7 +4420,10 @@ let start in if state.timeline_structure_revision = timeline_structure_revision then state - else { state with timeline_structure_revision } + else ( + report_app_feed ~row_revision:timeline_structure_revision ~stage:"app-row-store" + ~reason:"publish-row-store" state state; + { state with timeline_structure_revision }) in ignore (Journal_timeline.Store.synchronize timeline_store initial_state.timeline : int); let media_context = ref None in @@ -4612,7 +4690,10 @@ let start Some (request, Admission_refresh.Inspection_unavailable) | _ -> None in match completion with - | None -> Root_navigation.step state (Completed response), effects + | None -> + let next = Root_navigation.step state (Completed response) in + report_app_response state next response; + next, effects | Some (request, result) -> let admission_refresh, directive = Admission_refresh.complete state.admission_refresh ~request ~result diff --git a/app/journal_graph_runtime.ml b/app/journal_graph_runtime.ml index 725dfaa..8aef4d7 100644 --- a/app/journal_graph_runtime.ml +++ b/app/journal_graph_runtime.ml @@ -3,6 +3,57 @@ module Protocol = Logseq_db_worker.Protocol module Error = Logseq_db_worker.Error module Projection = Journal_graph_projection +let timeline_uuid value = + match Graph.Uuid.of_string value with + | Ok uuid -> `String (Graph.Uuid.to_string uuid) + | Error _ -> `String "invalid" +;; + +let timeline_version value = + let digits s = s <> "" && String.length s <= 19 + && String.for_all (fun c -> c >= '0' && c <= '9') s + && Option.is_some (Int64.of_string_opt s) in + let valid = String.length value <= 96 && match String.split_on_char ':' value with + | ["generation"; "v1"; epoch; sequence] -> digits epoch && digits sequence + | [("projection" | "change-window"); "v1"; number] -> digits number + | _ -> false in + `String (if valid then value else "invalid") +;; + +let report_timeline stage fields = + try + let text = Yojson.Safe.to_string (`Assoc (("stage", `String stage) :: fields)) in + if String.length text <= 8192 then ( + output_string stderr ("journal-timeline: " ^ text ^ "\n"); flush stderr) + with _ -> () +;; + +let report_capture_evidence ~stage ?result ?page_mutation ~request ~target + (command : Projection.capture) = + (* Capture IDs originate at the UI boundary. Never echo malformed input. *) + try + let capture_mutation = + match Graph.Uuid.of_string command.mutation_id with + | Ok uuid -> Graph.Uuid.to_string uuid + | Error _ -> "invalid" + in + let fields = + [ "stage", `String stage + ; "request", `String (Graph.Uuid.to_string request) + ; "target", `String (Graph.Uuid.to_string target) + ; "captureMutation", `String capture_mutation + ; "day", `Int (Journal_time.local_day command.creation_time) + ] + @ Option.fold ~none:[] ~some:(fun value -> [ "result", `String value ]) result + @ Option.fold ~none:[] + ~some:(fun uuid -> [ "pageMutation", `String (Graph.Uuid.to_string uuid) ]) + page_mutation + in + output_string stderr ("journal-capture: " ^ Yojson.Safe.to_string (`Assoc fields) ^ "\n"); + flush stderr + with _ -> () +;; + type worker_failure = { operation : string ; request_id : Graph.Uuid.t @@ -1155,16 +1206,33 @@ let read_root_batch t owner (page : Projection.page) cursor limit = ; root_todo = [] } in - read - t - (Root_tree batch) - (Protocol.V2_get_page_tree - { page = Graph.Uuid.of_string page.id |> Result.get_ok - ; maximum_depth = 0 - ; limit - ; cursor - ; revision = None - }) + let request = + read + t + (Root_tree batch) + (Protocol.V2_get_page_tree + { page = Graph.Uuid.of_string page.id |> Result.get_ok + ; maximum_depth = 0 + ; limit + ; cursor + ; revision = None + }) + in + (match owner with + | Feed_roots scan -> + report_timeline + "runtime-feed-tree-request" + [ "request", `String (Graph.Uuid.to_string request.request_id) + ; "requestGeneration", `String (Int64.to_string scan.root_generation) + ; "target", timeline_uuid page.id + ; "day", `Int page.day + ; "limit", `Int limit + ; "maximumDepth", `Int 0 + ; "hasContinuation", `Bool (Option.is_some cursor) + ; "kind", `String "roots" + ] + | Changed_roots _ -> ()); + request ;; let read_changed_roots t page limit = @@ -1207,17 +1275,30 @@ let read_changed_roots t page limit = ;; let read_root_dates t scan = - read - t - (Root_dates scan) - (Protocol.V2_list_journals - { from_day = 0 - ; through_day = - Option.fold ~none:99_999_999 ~some:previous_journal_day scan.root_before_day - ; limit = scan.root_list_size - ; cursor = scan.root_journal_cursor - ; revision = None - }) + let request = + read + t + (Root_dates scan) + (Protocol.V2_list_journals + { from_day = 0 + ; through_day = + Option.fold ~none:99_999_999 ~some:previous_journal_day scan.root_before_day + ; limit = scan.root_list_size + ; cursor = scan.root_journal_cursor + ; revision = None + }) + in + report_timeline + "runtime-feed-request" + [ "request", `String (Graph.Uuid.to_string request.request_id) + ; "requestGeneration", `String (Int64.to_string scan.root_generation) + ; "beforeDay", Option.fold ~none:`Null ~some:(fun d -> `Int d) scan.root_before_day + ; "dayLimit", `Int scan.root_list_size + ; "rootRemaining", `Int scan.root_remaining + ; "hasContinuation", `Bool (Option.is_some scan.root_journal_cursor) + ; "kind", `String "date-catalog" + ]; + request ;; let root_date_continuation scan = @@ -1232,16 +1313,29 @@ let root_date_continuation scan = when every discovered date is future and the catalog has more packets. *) let read_priority_today t scan = let today = Option.get scan.root_skipped_day in - read - t - (Root_today scan) - (Protocol.V2_list_journals - { from_day = today - ; through_day = today - ; limit = 1 - ; cursor = None - ; revision = None - }) + let request = + read + t + (Root_today scan) + (Protocol.V2_list_journals + { from_day = today + ; through_day = today + ; limit = 1 + ; cursor = None + ; revision = None + }) + in + report_timeline + "runtime-feed-request" + [ "request", `String (Graph.Uuid.to_string request.request_id) + ; "requestGeneration", `String (Int64.to_string scan.root_generation) + ; "day", `Int today + ; "dayLimit", `Int 1 + ; "rootRemaining", `Int scan.root_remaining + ; "hasContinuation", `Bool false + ; "kind", `String "priority-today" + ]; + request ;; let root_feed_response scan complete = let days = scan.root_days in @@ -1252,12 +1346,41 @@ let root_feed_response scan complete = (fun n (d : Projection.day_feed) -> n + 1 + List.length d.entries) 0 days - (* Keep the date entry point even while a day has its own cursor: a later + (* Keep the date entry point even while a day has its own cursor: a later authoritative page reconciliation can legitimately remove that cursor. *) ; continuation = root_date_continuation scan ; prioritized_day = scan.root_skipped_day } in + report_timeline + "runtime-feed-result" + [ "requestGeneration", `String (Int64.to_string scan.root_generation) + ; "complete", `Bool complete + ; "dayCount", `Int (List.length days) + ; "slotCount", `Int feed.slot_count + ; "rootCount", `Int (feed.slot_count - List.length days) + ; "truncated", `Bool (List.length days > 16 || feed.slot_count - List.length days > 16) + ; ( "pages" + , `List + (List.take 16 days + |> List.map (fun (d : Projection.day_feed) -> + `Assoc + [ "page", timeline_uuid d.page.id + ; "day", `Int d.page.day + ; "rootCount", `Int (List.length d.entries) + ; "hasContinuation", `Bool d.has_more_entries + ])) ) + ; ( "roots" + , `List + (List.concat_map + (fun (d : Projection.day_feed) -> + List.map + (fun (e : Projection.timeline_entry) -> Journal_model.id e.block) + d.entries) + days + |> List.take 16 + |> List.map timeline_uuid) ) + ]; response (Feed_loaded { request_generation = scan.root_generation; feed; complete }) ;; let next_feed_batch t scan = @@ -1453,15 +1576,31 @@ let complete_root_batch t batch = ;; let read_root_preview t preview = - read - t - (Root_preview preview) - (Protocol.V2_get_children - { parent = preview.preview_root.block.uuid - ; limit = preview.preview_limit - ; cursor = preview.preview_cursor - ; revision = None - }) + let request = + read + t + (Root_preview preview) + (Protocol.V2_get_children + { parent = preview.preview_root.block.uuid + ; limit = preview.preview_limit + ; cursor = preview.preview_cursor + ; revision = None + }) + in + (match preview.preview_batch.root_owner with + | Feed_roots scan -> + report_timeline + "runtime-feed-tree-request" + [ "request", `String (Graph.Uuid.to_string request.request_id) + ; "requestGeneration", `String (Int64.to_string scan.root_generation) + ; "target", `String (Graph.Uuid.to_string preview.preview_root.block.uuid) + ; "day", `Int preview.preview_batch.root_page.day + ; "limit", `Int preview.preview_limit + ; "hasContinuation", `Bool (Option.is_some preview.preview_cursor) + ; "kind", `String "child-preview" + ] + | Changed_roots _ -> ()); + request ;; let next_root_preview t batch = @@ -2749,6 +2888,22 @@ let hydration_for_changes t ~request_generation windows = structures || List.exists (fun page -> Option.is_some (registered_page t page)) pages in + report_timeline "runtime-interest" + [ "requestGeneration", `String (Int64.to_string request_generation) + ; "journalChanged", `Bool journal_changed + ; "hasInitialFeed", `Bool (Option.is_some t.initial_feed_spec) + ; "registeredPageHit", `Bool (List.exists (fun page -> Option.is_some (registered_page t page)) pages) + ; "pageCount", `Int (List.length pages) + ; "pages", `List (List.take 16 pages |> List.map (fun u -> `String (Graph.Uuid.to_string u))) + ; "blockCount", `Int (List.length blocks) + ; "blocks", `List (List.take 16 blocks |> List.map (fun u -> `String (Graph.Uuid.to_string u))) + ; "interestCount", `Int (List.length structures) + ; "interests", `List (List.take 16 structures |> List.map (function + | Protocol.V2_journal_index_interest -> `Assoc ["kind", `String "journal-index"] + | V2_children_interest u -> `Assoc ["kind", `String "children"; "target", `String (Graph.Uuid.to_string u)] + | V2_page_tree_interest u -> `Assoc ["kind", `String "page-tree"; "target", `String (Graph.Uuid.to_string u)])) + ; "truncated", `Bool (List.length pages > 16 || List.length blocks > 16 || List.length structures > 16) + ]; let journal = match journal_changed, t.initial_feed_spec with | true, Some (root_limit, day_limit) -> @@ -3498,18 +3653,21 @@ let receive_response t (protocol_response : Protocol.response) = { requests = catalog; responses = [ response Graph_ready ] } | Capture_format { command; page_uuid; page_revision } -> let journal_day = Journal_time.local_day command.creation_time in - requests - [ mutate + let page_mutation = request_uuid t in + let request = mutate t (Capture_create_page { command; page_uuid }) (Protocol.V2_create_journal_page - { mutation_id = request_uuid t + { mutation_id = page_mutation ; page = page_uuid ; journal_day ; title = journal_title ~format:journal_title_format journal_day ; preconditions = preconditions ~pages:[ page_uuid, page_revision ] () }) - ] + in + report_capture_evidence ~stage:"create-page" ~page_mutation + ~request:request.request_id ~target:page_uuid command; + requests [ request ] | _ -> failure_output t operation request_id "Unexpected graph-info response.") | V2_task_statuses_outcome { property_uuid; default_status; items; next_cursor } -> (match operation with @@ -3775,6 +3933,8 @@ let receive_response t (protocol_response : Protocol.response) = Hashtbl.replace t.graph_pages (Graph.Uuid.to_string page.uuid) page; detail_children t ~generation ~page ~root ~limit ~after) | V2_present_page { page; revision }, Capture_page command -> + report_capture_evidence ~stage:"page-lookup" ~result:"present" + ~request:request_id ~target:page.uuid command; remember_page_revision t page.uuid revision; (match page_of_graph_page page with | None -> reject "The target journal page is unavailable." @@ -3793,6 +3953,8 @@ let receive_response t (protocol_response : Protocol.response) = }) ] | V2_missing_page { uuid; revision }, Capture_page command -> + report_capture_evidence ~stage:"page-lookup" ~result:"missing" + ~request:request_id ~target:uuid command; remember_page_revision t uuid revision; requests [ read @@ -4367,6 +4529,17 @@ let receive_response t (protocol_response : Protocol.response) = | V2_changes { generation; through; windows; _ } -> (match operation with | Pull_changes { request_generation; _ } -> + report_timeline "runtime-change-window" + [ "phase", `String "received"; "request", `String (Graph.Uuid.to_string request_id) + ; "requestGeneration", `String (Int64.to_string request_generation) + ; "generation", timeline_version generation; "through", timeline_version through + ; "windowCount", `Int (List.length windows) + ; "windows", `List (List.take 16 windows |> List.map (fun (w : Protocol.v2_change_window) -> + `Assoc ["id", timeline_version w.id; "predecessor", timeline_version w.predecessor; + "successor", timeline_version w.successor; "blockCount", `Int (List.length w.block_uuids); + "pageCount", `Int (List.length w.page_uuids); "interestCount", `Int (List.length w.structure_interests)])) + ; "truncated", `Bool (List.length windows > 16) + ]; t.change_generation <- Some generation; t.change_cursor <- Some through; let acknowledgement = @@ -4420,7 +4593,10 @@ let receive t (protocol_response : Protocol.response) = let owned = match pending with | Some { delivery = Dispatched; _ } -> true | None | Some _ -> false in - if not owned then empty else ( + if not owned then ( + report_timeline "runtime-response" + ["request", `String (Graph.Uuid.to_string request_id); "reason", `String "discard-not-owned"]; + empty) else ( let background = Hashtbl.mem t.hydration_active key in Hashtbl.remove t.hydration_active key; let root_current = match pending with @@ -4649,6 +4825,18 @@ let abandon t (request : Protocol.request) = ;; let reconcile_push t ~request_generation push = + (match push with + | Protocol.V2_changes_available { generation; through; _ } -> + report_timeline "runtime-change-window" + [ "phase", `String "push"; "generation", timeline_version generation + ; "through", timeline_version through + ; "requestGeneration", `String (Int64.to_string request_generation) + ] + | V2_resync_required_push { generation; _ } -> + report_timeline "runtime-change-window" + [ "phase", `String "resync"; "generation", timeline_version generation + ; "requestGeneration", `String (Int64.to_string request_generation) + ]); let output = match push with | Protocol.V2_changes_available { generation; _ } -> diff --git a/docs/agent-guide/implemented/bugfix/2026-10-25-2026-10-25-integrity-preimage-diagnostics.md b/docs/agent-guide/implemented/bugfix/2026-10-25-2026-10-25-integrity-preimage-diagnostics.md new file mode 100644 index 0000000..a760682 --- /dev/null +++ b/docs/agent-guide/implemented/bugfix/2026-10-25-2026-10-25-integrity-preimage-diagnostics.md @@ -0,0 +1,119 @@ +# 2026 10 25 Integrity Preimage Diagnostics + +## Problem + +The phone reports accepted create-journal-page payload-mismatch (expected five +datoms, actual seven) after Web updates, reported sync catchup, an empty Timeline +and Capture. Counts do not identify the missing page fields or redundant +protected-value operations; later logical reads can be masked by the outbox. + +## Decision + +On published main bfd603e, add bounded structured console diagnostics at the +integrity failure's authoritative root_before, plus Capture page lookup/create +correlation and Worker Missing_page snapshot versions. Allow only UUIDs, dates, +scope/version tokens, finite attribute/type labels, counts and boolean flags. +Never print values, plaintext/ciphertext, exception strings, tokens or keys. +Keep existing errors and their 512-byte carriers intact. Do not modify sync, +retry, commit or validation decisions. No protected spec interface change is +needed. The initial investigation stayed local; later publication authorization is recorded below. + +Implement the three private observation points without changing public interfaces +or synchronization decisions. The overlay database owns the immutable replay +preimage; pure reducers cannot inspect it. Temporary behavioral/privacy checks +therefore exercise the public Database boundary, with separate checks of the +actual Worker snapshot and Capture runtime correlation. No permanent CI test or +Dune change is needed for this local investigation. + +During integration with the unified30-root Timeline, observe its current date catalog, exact-Today probe, depth-zero root queries and bounded child-preview queries. Preserve request UUID allocation and query arguments. Report the actual Feed completion flag and root counts from the new scanner; removed tree-slot budget fields and the deleted legacy Feed owner are not restored. The initial-interest flag reads the new initial_feed_spec owner. No additional graph read is issued for logging. + +## Alternatives considered + +### Ordinary post-failure get_pages + +The optimistic Capture outbox may already synthesize a complete page, concealing +the authoritative preimage. Inspect the validator's existing preimage instead. + +### Broad raw transaction or exception logging + +Rejected because it can disclose graph content and credentials. A compact +allowlisted technical shape is sufficient to distinguish the hypotheses. + +## Consequences + +Five temporary behavioral/privacy checks, the existing Overlay synchronization +suite, Runtime locality suite and Worker integration check passed. The exact +source candidate built and ran in the authorized Simulator/Web test graph after +the user approved its data-preserving experimental installation. Both an existing +date-page update and a previously absent future date-page/reference control +displayed automatically before Capture; both subsequent Captures synchronized +successfully. The Captures targeted the existing current-day page and did not +exercise accepted create-journal-page failure. This does not establish the phone +root cause or full cross-ABI validation. Preserve that limitation and keep the +change local until publication is separately requested. + +## Timeline correlation follow-up + +The user subsequently prioritized the earlier equal-TX/empty-Timeline condition. +Three real Web/A135 controls on 6708 passed: foreground empty Today to a nonempty +remote root, cold catchup, and background/resume followed by ordinary refresh. +They do not reproduce the phone failure. The user authorized additional safe +diagnostics, not a speculative synchronization fix or another data-App install. + +Add private bounded observation points for authoritative commit/checkpoint and +logical interests; Worker query/window versions; Runtime change-window, Feed +request/response ownership; Application Feed acceptance/discard reasons and row +store publication. Restrict samples to sixteen UUIDs, encode int64 generations as +strings, reject unexpected external version labels, and contain output failures. +Never log content, property values, exception strings, URL/query strings or crypto +material. Keep the existing state transitions, validation and public interfaces. + +The App's existing current_graph_generation helper actually returns the selected +graph UUID. Record that UUID separately from account/graph/Worker counters rather +than mislabelling it as an incarnation counter. This naming observation does not +prove a stale-response bug: graph retirement also cancels request leases and +clears Runtime ownership. No business-rule correction is included here. + +Temporary checks exercise public Overlay, Worker, Runtime and Application +boundaries. Deliberately withholding notifications or retiring a completed feed +is a mechanism/control experiment, not a reproduction of production loss. +Baseline behavior and failed-output behavior must remain equivalent. No permanent +CI tests, protected spec files or Dune files are changed. Build a separately +identified experimental candidate; report its exact inputs and compatibility +limits and obtain specific approval before any data-preserving A135 install. + +## Acceptance criteria + +- Write the privacy and evidence tests first and observe missing-log failures. +- Synthetic payload mismatch emits exact field presence/types, add/retract counts + and protected same-value pair flags without body or private attribute leakage. +- Existing error, checkpoint and outbox preservation tests still pass. +- Final candidate builds with pinned dependency provenance; no sync algorithm fix. +- Actual Simulator/Web tests use only ocaml-sync-test and preserve other graphs. + Report lock/auth/ABI/install blockers and actual coverage without claiming an + unreproduced phone root cause or verified cross ABI. + +## Risks + +- Technical UUID/version correlation is intentionally logged, but no graph values. +- Logs may not recover a prior event; preserve outbox and wait for a safe retry. +- Simulator host restamping is experimental; install over an existing data App + requires the separately scoped risk confirmation specified by the parent. + +## Questions + +- Diagnostic scope and actual test-graph writes were approved in the user's + 2026-10-25 instruction relayed by the parent; no implementation question remains. +- Phone build remains unknown; the selected baseline is verified main bfd603e. + +## PR preparation + +The user authorized an independent diagnostic PR on 2026-10-25. This change adds +observability and does not fix the reported Capture/integrity or stale Timeline +problem. Final checks pass: six temporary diagnostic/privacy/failed-output tests, +98 Overlay sync tests, 152 Runtime locality tests and 15 Worker integration tests. +The 6709 exact-source candidate completed real test-graph foreground and cold-start +Web/Simulator correlations with 972/972 and zero outbox; both ordinary paths +worked, so neither is evidence the phone defect is fixed. ABI remains unvalidated. +The subsequent conditional Feed/empty-state findings are a separate investigation; +no corresponding business repair is included in this diagnostic PR. diff --git a/logseq_db_worker/lib/effect_runner/effect_runner.ml b/logseq_db_worker/lib/effect_runner/effect_runner.ml index 8be5702..4b256eb 100644 --- a/logseq_db_worker/lib/effect_runner/effect_runner.ml +++ b/logseq_db_worker/lib/effect_runner/effect_runner.ml @@ -932,6 +932,17 @@ let publish_projection_change t session = function ; structure_interests = List.map structure_interest_to_protocol structure_interests }; + (try + report_warning ("journal-timeline: " ^ Yojson.Safe.to_string (`Assoc + [ "stage", `String "worker-change-window"; "generation", `String generation + ; "window", `String id + ; "beforeProjection", `String (Overlay.Projection_revision.to_string before_revision) + ; "projection", `String (Overlay.Projection_revision.to_string after_revision) + ; "blockCount", `Int (List.length block_uuids) + ; "pageCount", `Int (List.length page_uuids) + ; "interestCount", `Int (List.length structure_interests) + ]) ^ "\n") + with _ -> ()); t.post (Core.Projection_push (Protocol.V2_changes_available @@ -1244,6 +1255,39 @@ let read_snapshot ~on_enrichment_error database request command = Fun.protect ~finally:(fun () -> Database.release_snapshot snapshot) (fun () -> + let query_evidence query ?target items has_continuation = + try + let version = Database.snapshot_version snapshot in + let fields = + [ "stage", `String "worker-query"; "query", `String query + ; "request", `String (Graph.Uuid.to_string request.request_id) + ; "generation", `String (Overlay.Generation.to_string version.generation) + ; "projection", `String (Overlay.Projection_revision.to_string version.projection_revision) + ; "itemCount", `Int (List.length items) + ; "items", `List (List.take 16 items |> List.map (fun u -> `String (Graph.Uuid.to_string u))) + ; "hasContinuation", `Bool has_continuation + ; "truncated", `Bool (List.length items > 16) + ] @ Option.fold ~none:[] ~some:(fun u -> ["target", `String (Graph.Uuid.to_string u)]) target in + report_warning ("journal-timeline: " ^ Yojson.Safe.to_string (`Assoc fields) ^ "\n") + with _ -> () + in + let page_read_evidence result uuid = + try + let version = Database.snapshot_version snapshot in + report_warning + ("journal-page-read: " + ^ Yojson.Safe.to_string + (`Assoc + [ "result", `String result + ; "request", `String (Graph.Uuid.to_string request.request_id) + ; "target", `String (Graph.Uuid.to_string uuid) + ; "generation", `String (Overlay.Generation.to_string version.generation) + ; "projection", `String + (Overlay.Projection_revision.to_string version.projection_revision) + ]) + ^ "\n") + with _ -> () + in let titles = Hashtbl.create 16 in let block_record (value : Overlay.block_record) = let missing = @@ -1400,6 +1444,9 @@ let read_snapshot ~on_enrichment_error database request command = with | Error error -> read_failure request error | Ok result -> + query_evidence "journals" + (List.map (fun (item : Overlay.journal_item) -> item.page.page.uuid) result.items) + (Option.is_some result.next_cursor); let items = result.items |> List.map (fun (item : Overlay.journal_item) -> @@ -1415,6 +1462,7 @@ let read_snapshot ~on_enrichment_error database request command = | V2_get_page { page; _ } -> (match Database.get_pages snapshot [ page ] with | Ok [ Overlay.Present_page { value; revision } ] -> + page_read_evidence "present" value.page.uuid; response request (Protocol.V2_page_outcome @@ -1423,6 +1471,7 @@ let read_snapshot ~on_enrichment_error database request command = ; revision = Overlay.Page_state_revision.to_string revision })) | Ok [ Missing_page { uuid; revision } ] -> + page_read_evidence "missing" uuid; response request (Protocol.V2_page_outcome @@ -1542,6 +1591,9 @@ let read_snapshot ~on_enrichment_error database request command = | Ok (Overlay.Children_result { parent; revision_scope; scope_revision; items; next_cursor }) -> + query_evidence "children" ~target:parent + (List.map (fun (item : Overlay.child_member) -> item.block.block.uuid) items) + (Option.is_some next_cursor); response request (Protocol.V2_children_outcome @@ -1569,6 +1621,9 @@ let read_snapshot ~on_enrichment_error database request command = (Overlay.Page_tree { page; maximum_depth; limit; cursor }) with | Ok (Overlay.Page_tree_result { page; maximum_depth; items; next_cursor }) -> + query_evidence "page-tree" ~target:page + (List.map (fun (item : Overlay.tree_member) -> item.block.block.uuid) items) + (Option.is_some next_cursor); response request (Protocol.V2_page_tree_outcome diff --git a/logseq_overlay_db/lib/database.ml b/logseq_overlay_db/lib/database.ml index 1f28ab1..8f2459b 100644 --- a/logseq_overlay_db/lib/database.ml +++ b/logseq_overlay_db/lib/database.ml @@ -7252,7 +7252,134 @@ let validate_active_transaction_origin owner cursor root_before transaction_data ()) ;; -let validate_active_transaction_origins owner batch transaction_data roots_after = +(* Only schema identifiers in this list may leave the database in evidence. + Values, arbitrary attribute names and wire payloads must never be emitted. *) +let integrity_evidence_attributes = + [ "block/uuid"; "block/title"; "block/name"; "block/journal-day" + ; "block/created-at"; "block/updated-at"; "block/tags"; "block/parent" + ; "block/page"; "block/order"; "block/refs"; "block/tx-id" + ] +;; + +let integrity_evidence_value_type = function + | Datascript.Nil -> "nil" + | Int64 _ -> "integer" + | Float _ -> "float" + | String _ -> "string" + | Symbol _ -> "symbol" + | Bool _ -> "boolean" + | Keyword _ -> "keyword" + | Uuid _ -> "uuid" + | Instant _ -> "instant" + | Regex _ -> "regex" + | Ref _ | Ref_to _ -> "reference" + | List _ -> "list" + | Vector _ -> "vector" + | Map _ -> "map" + | Set _ -> "set" + | Tuple _ -> "tuple" + | TxRef -> "transaction-reference" +;; + +let report_integrity_evidence owner batch record cursor root_before expected actual operations = + (* This best-effort observation must not affect validation or publication. *) + try + let target, day = + match record.mutation with + | Types.Create_journal_page { page; journal_day; _ } -> page, Some journal_day + | Insert_blocks { tree; _ } -> tree.uuid, None + | Delete_blocks { root; _ } -> root, None + | Save_block { block; _ } | Set_task_status { block; _ } + | Clear_task_status { block; _ } | Publish_asset { block; _ } + | Set_asset_reference { block; _ } -> block, None + in + let entity = entity_of_uuid root_before target in + let before_fields = + List.map + (fun attribute -> + let sample = + match entity with + | None -> [] + | Some entity -> + Datascript.datoms root_before Datascript.Eavt ~e:entity ~a:attribute () + |> Seq.take 4 |> List.of_seq + in + let values = List.map (fun (d : Datascript.datom) -> d.v) sample in + let types = List.map integrity_evidence_value_type values |> List.sort_uniq compare in + `Assoc + [ "attribute", `String attribute + ; "datoms", `Int (min 3 (List.length sample)) + ; "distinct", `Int (min 3 (List.length (List.sort_uniq compare values))) + ; "types", `List (List.map (fun s -> `String s) types) + ; "truncated", `Bool (List.length sample > 3) + ]) + integrity_evidence_attributes + in + let delta = Hashtbl.create 16 in + List.iter + (fun (datom : Datascript.datom) -> + let attribute = + if List.mem datom.a integrity_evidence_attributes then datom.a else "other" + in + let added, retracted = Option.value ~default:(0, 0) (Hashtbl.find_opt delta attribute) in + Hashtbl.replace delta attribute + (added + (if datom.added then 1 else 0), retracted + (if datom.added then 0 else 1))) + actual; + let pairs = Hashtbl.create 4 in + List.iter + (fun operation -> + let item = + match operation with + | Datascript.Add (entity, attribute, value) -> Some (entity, attribute, value, true) + | Retract (entity, attribute, Some value) -> Some (entity, attribute, value, false) + | _ -> None + in + match item with + | Some (entity, ("block/title" | "block/name" as attribute), value, added) -> + let key = entity, attribute, value in + let a, r = Option.value ~default:(false, false) (Hashtbl.find_opt pairs key) in + Hashtbl.replace pairs key (a || added, r || not added) + | _ -> ()) + operations; + let pair_count attribute = + Hashtbl.fold + (fun (_, a, _) (added, retracted) count -> + if String.equal a attribute && added && retracted then count + 1 else count) + pairs 0 + in + let str s = `String s in + let evidence = + `Assoc + ([ "reason", str "payload-mismatch" + ; "mutationId", str (Graph.Uuid.to_string record.mutation_id) + ; "target", str (Graph.Uuid.to_string target) + ; "graph", str (Graph.Uuid.to_string owner.graph_uuid) + ; "generation", str (Types.Generation.to_string owner.generation) + ; "projection", str (Types.Projection_revision.to_string (projection_revision owner.projection)) + ; "checkpoint", str (Int64.to_string (Types.Server_cursor.to_int64 owner.checkpoint)) + ; "through", str (Int64.to_string (Types.Server_cursor.to_int64 (Types.authoritative_batch_through batch))) + ; "expectedCursor", str (Int64.to_string (Types.Server_cursor.to_int64 cursor)) + ; "expectedDatoms", `Int (List.length expected) + ; "actualDatoms", `Int (List.length actual) + ; "beforeFields", `List before_fields + ; "actualDelta", `List + (Hashtbl.to_seq delta |> List.of_seq |> List.sort compare + |> List.map (fun (attribute, (added, retracted)) -> + `Assoc [ "attribute", str attribute; "added", `Int added; "retracted", `Int retracted ])) + ; "protectedPairs", `Assoc + [ "block/title", `Int (pair_count "block/title") + ; "block/name", `Int (pair_count "block/name") ] + ] + @ List.map (fun (k, v) -> k, str v) (integrity_record_context record) + @ Option.fold ~none:[] ~some:(fun day -> [ "day", `Int day ]) day) + in + let encoded = Yojson.Safe.to_string evidence in + if String.length encoded <= 8192 + then report_warning ("journal-integrity: " ^ encoded ^ "\n") + with _ -> report_warning "journal-integrity: {\"reason\":\"evidence-unavailable\"}\n" +;; + +let validate_active_transaction_origins owner batch transaction_data roots_after decoded_transactions = let cursors = Types.authoritative_batch_transactions batch |> List.map Types.authoritative_transaction_cursor @@ -7353,9 +7480,20 @@ let validate_active_transaction_origins owner batch transaction_data roots_after | Error _ -> ( "expected-simulation-failed" , [ "actualDatoms", string_of_int (List.length actual) ] ) - | Ok expected -> + | Ok expected_data -> + let rec operations_at_cursor cursors transactions = + match cursors, transactions with + | cursor :: rest, operations :: more -> + if Types.Server_cursor.equal cursor expected + then operations + else operations_at_cursor rest more + | _ -> [] + in + report_integrity_evidence owner batch record expected + root_before expected_data actual + (operations_at_cursor cursors decoded_transactions); ( "payload-mismatch" - , [ "expectedDatoms", string_of_int (List.length expected) + , [ "expectedDatoms", string_of_int (List.length expected_data) ; "actualDatoms", string_of_int (List.length actual) ] )) in @@ -7883,6 +8021,7 @@ let prepare_authoritative_candidate preparation ~decrypted = preparation.authoritative_batch expected_tx_data roots_after + transactions with | Error message -> Error (Types.Authoritative_integrity_failure (integrity_batch_source preparation.authoritative_owner preparation.authoritative_batch message)) @@ -9263,8 +9402,43 @@ let replay_authoritative database prepared = | exn -> Error (Printexc.to_string exn)) ;; +let report_timeline_commit graph_uuid checkpoint_before (commit : Types.authoritative_commit) = + try + let kind, blocks, pages, interests = match commit.logical_change_summary with + | Types.No_logical_change -> "none", [], [], [] + | Logical_resync_required _ -> "resync", [], [], [] + | Exact_logical_change value -> + "exact", value.block_uuids, value.page_uuids, value.structure_interests + in + let uuids values = `List (List.take 16 values |> List.map (fun u -> `String (Graph.Uuid.to_string u))) in + let interest = function + | Types.Children_interest u -> `Assoc ["kind", `String "children"; "target", `String (Graph.Uuid.to_string u)] + | Page_tree_interest u -> `Assoc ["kind", `String "page-tree"; "target", `String (Graph.Uuid.to_string u)] + | Journal_index_interest -> `Assoc ["kind", `String "journal-index"] + in + let fields = + [ "stage", `String "overlay-commit" + ; "graph", `String (Graph.Uuid.to_string graph_uuid) + ; "generation", `String (Types.Generation.to_string commit.generation) + ; "checkpointBefore", `String (Int64.to_string (Types.Server_cursor.to_int64 checkpoint_before)) + ; "checkpoint", `String (Int64.to_string (Types.Server_cursor.to_int64 commit.checkpoint)) + ; "beforeProjection", `String (Types.Projection_revision.to_string commit.before_projection_revision) + ; "projection", `String (Types.Projection_revision.to_string commit.after_projection_revision) + ; "changeKind", `String kind + ; "blockCount", `Int (List.length blocks); "blocks", uuids blocks + ; "pageCount", `Int (List.length pages); "pages", uuids pages + ; "interestCount", `Int (List.length interests) + ; "interests", `List (List.take 16 interests |> List.map interest) + ; "truncated", `Bool (List.length blocks > 16 || List.length pages > 16 || List.length interests > 16) + ] in + let text = Yojson.Safe.to_string (`Assoc fields) in + if String.length text <= 8192 then report_warning ("journal-timeline: " ^ text ^ "\n") + with _ -> () +;; + let commit_authoritative_candidate database prepared = - serialize_commit database (fun () -> + let checkpoint_before = ref None in + let outcome = serialize_commit database (fun () -> let change, outcome = Eio.Mutex.use_rw ~protect:true database.lock (fun () -> let preparation = prepared.authoritative_preparation in @@ -9726,6 +9900,7 @@ let commit_authoritative_candidate database prepared = database.durable_outbox <- durable_outbox; database.durable_sync_revision <- database.sync_revision + 1; preparation.authoritative_state <- Authoritative_consumed; + checkpoint_before := Some database.checkpoint; let before_projection = database.projection in let before_snapshot = snapshot_of_database database in (match replay_authoritative database prepared with @@ -9818,7 +9993,11 @@ let commit_authoritative_candidate database prepared = ; logical_change_summary } ))))) in - change, outcome) + change, outcome) in + (match outcome, !checkpoint_before with + | Ok commit, Some before -> report_timeline_commit database.graph_uuid before commit + | _ -> ()); + outcome ;; let apply_authoritative database preparation ~decrypted =