From f31a23d2992ab37ceb288dbc9a2d97effa887a09 Mon Sep 17 00:00:00 2001 From: abhay-codes07 Date: Wed, 26 Aug 2026 17:05:01 +0530 Subject: [PATCH 1/2] fix(build): route carried hyperedges through the dedup survivor remap (#3102) build_merge() carries hyperedges from unchanged files across an incremental rebuild (#1574). They were attached to G AFTER build() and entity dedup had finished, so while every edge endpoint was rewired onto the dedup survivor (#2805), a carried hyperedge kept naming the merged-away node: a dangling member in graph.json with no backing node, and a member list that disagreed with the edges about which node the concept is. The carried set now rides into build() on the base chunk, so dedup remaps its members exactly as it remaps the new chunks' hyperedges, and build_from_json validates them against the built node set like any other. The prune-set helpers move ahead of the build call (they never depended on it); a hyperedge the new chunks re-emit under the same id is left to them, preserving the id-dedup the old attach_hyperedges step provided. --- graphify/build.py | 41 ++++++---- tests/test_carried_hyperedge_remap.py | 103 ++++++++++++++++++++++++++ 2 files changed, 131 insertions(+), 13 deletions(-) create mode 100644 tests/test_carried_hyperedge_remap.py diff --git a/graphify/build.py b/graphify/build.py index d9b5e768f..10cdec28f 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -1726,11 +1726,6 @@ def _kept(item: dict) -> bool: file=sys.stderr, ) - base = [{"nodes": existing_nodes, "edges": existing_edges}] if had_graph else [] - - all_chunks = base + list(new_chunks) - G = build(all_chunks, directed=directed, dedup=dedup, dedup_llm_backend=dedup_llm_backend, root=root) - # Prune set for deleted source files — both the raw form (matches nodes that # kept absolute source_file) and the normalised relative form (matches nodes # relativised by _norm_source_file at build time). .resolve() (via _eff_root) @@ -1797,12 +1792,25 @@ def _prune_match(sf: "str | None") -> bool: # deleted (#1574). build() only sees the new chunks' hyperedges, so without # this every --update collapses the graph's hyperedge set down to just the # changed files'. Re-extracted files' prior hyperedges are dropped (their new - # version is already in G — replace-per-source, like nodes/edges); deleted - # files' are dropped via prune_set. id-dedup (attach_hyperedges) so a carried - # hyperedge never duplicates one the new chunks re-emitted. Mirrors watch.py, - # which already preserves existing hyperedges across a rebuild. + # version is already in the new chunks — replace-per-source, like + # nodes/edges); deleted files' are dropped via prune_set; id-dedup so a + # carried hyperedge never duplicates one the new chunks re-emitted. Mirrors + # watch.py, which already preserves existing hyperedges across a rebuild. + # + # The carried set rides INTO build() on the base chunk rather than being + # attached to G afterwards (#3102): entity dedup rewires every edge endpoint + # and every hyperedge member it sees onto the survivor (#2805), but a + # hyperedge attached after the fact kept naming the merged-away node — a + # dangling member with no backing node in the written graph. + carried_hyperedges: list[dict] = [] if existing_hyperedges: - carried = [] + carried = carried_hyperedges + _new_hyperedge_ids = { + he.get("id") + for chunk in new_chunks + for he in (chunk.get("hyperedges") or []) + if isinstance(he, dict) and he.get("id") + } for he in existing_hyperedges: if not isinstance(he, dict): continue @@ -1815,10 +1823,17 @@ def _prune_match(sf: "str | None") -> bool: continue # semantically re-extracted — replaced by the new chunk's version if _prune_match(sf): continue # deleted — pruned + if he.get("id") and he.get("id") in _new_hyperedge_ids: + continue # the new chunks re-emitted it — theirs wins carried.append(he) - if carried: - from graphify.export import attach_hyperedges - attach_hyperedges(G, carried) + + base = ( + [{"nodes": existing_nodes, "edges": existing_edges, "hyperedges": carried_hyperedges}] + if had_graph else [] + ) + + all_chunks = base + list(new_chunks) + G = build(all_chunks, directed=directed, dedup=dedup, dedup_llm_backend=dedup_llm_backend, root=root) # Prune nodes and edges from deleted source files if prune_sources: diff --git a/tests/test_carried_hyperedge_remap.py b/tests/test_carried_hyperedge_remap.py new file mode 100644 index 000000000..d5b4d1047 --- /dev/null +++ b/tests/test_carried_hyperedge_remap.py @@ -0,0 +1,103 @@ +"""Carried-forward hyperedges must follow the dedup survivor remap (#3102). + +build_merge() carries hyperedges from unchanged files across an incremental +rebuild (#1574). They used to be attached to G AFTER build() and entity +dedup had finished, so while every edge endpoint was rewired onto the dedup +survivor (#2805), a carried hyperedge kept naming the merged-away node — a +dangling member in graph.json with no backing node. +""" +from __future__ import annotations + +import json +from pathlib import Path + +from graphify.build import build_from_json, build_merge +from graphify.export import to_json + +# `alpha_a` and `alpha_concept_long_variant_id` label-dedup into one node. +NODES = [ + {"id": "alpha_a", "label": "Alpha Concept", "file_type": "concept", "source_file": "notes/a.md"}, + {"id": "alpha_concept_long_variant_id", "label": "alpha_concept", "file_type": "concept", + "source_file": "notes/b.md"}, + {"id": "beta_node", "label": "Beta", "file_type": "concept", "source_file": "notes/group.md"}, + {"id": "gamma_node", "label": "Gamma", "file_type": "concept", "source_file": "notes/group.md"}, +] +EDGES = [{"source": "alpha_concept_long_variant_id", "target": "beta_node", "relation": "references", + "confidence": "EXTRACTED", "confidence_score": 1.0, "source_file": "notes/b.md"}] +HYPEREDGE = {"id": "the_group", "label": "The Group", + "nodes": ["alpha_concept_long_variant_id", "beta_node", "gamma_node"], + "relation": "participate_in", "confidence": "EXTRACTED", "confidence_score": 1.0, + "source_file": "notes/group.md"} +UNRELATED_CHUNK = { + "nodes": [{"id": "delta_node", "label": "Delta", "file_type": "concept", "source_file": "notes/d.md"}], + "edges": [], "hyperedges": [], +} + + +def _baseline(tmp_path: Path) -> Path: + """A graph written WITHOUT dedup, so the pair is still two nodes on disk + and the hyperedge names the variant — the shape an older build leaves.""" + G = build_from_json({"nodes": NODES, "edges": EDGES, "hyperedges": [HYPEREDGE]}) + p = tmp_path / "graph.json" + to_json(G, {0: list(G.nodes)}, str(p)) + return p + + +def _hyperedges(G): + return {he["id"]: he for he in G.graph.get("hyperedges", [])} + + +def test_a_carried_hyperedge_is_remapped_onto_the_dedup_survivor(tmp_path): + G = build_merge([UNRELATED_CHUNK], _baseline(tmp_path)) + survivors = set(G.nodes) + assert "alpha_concept_long_variant_id" not in survivors # merged away + he = _hyperedges(G)["the_group"] + assert set(he["nodes"]) <= survivors, f"dangling members: {set(he['nodes']) - survivors}" + assert "alpha_a" in he["nodes"] # onto the survivor, not just dropped + assert {"beta_node", "gamma_node"} <= set(he["nodes"]) + + +def test_the_written_graph_has_no_dangling_hyperedge_member(tmp_path): + G = build_merge([UNRELATED_CHUNK], _baseline(tmp_path)) + out = tmp_path / "merged.json" + to_json(G, {0: list(G.nodes)}, str(out), force=True) + data = json.loads(out.read_text(encoding="utf-8")) + ids = {n["id"] for n in data["nodes"]} + for he in data.get("hyperedges", []): + assert set(he["nodes"]) <= ids, f"{he['id']} names a node that is not in the graph" + + +def test_edges_and_hyperedges_agree_on_the_survivor(tmp_path): + """The edge endpoint and the hyperedge member came from the same + merged-away node; both must now name the same survivor.""" + G = build_merge([UNRELATED_CHUNK], _baseline(tmp_path)) + edge_ends = {u for u, v in G.edges} | {v for u, v in G.edges} + assert "alpha_a" in edge_ends + assert "alpha_a" in _hyperedges(G)["the_group"]["nodes"] + + +def test_a_hyperedge_re_emitted_by_the_new_chunk_is_not_duplicated(tmp_path): + fresh = {"nodes": [{"id": "beta_node", "label": "Beta", "file_type": "concept", + "source_file": "notes/group.md"}, + {"id": "gamma_node", "label": "Gamma", "file_type": "concept", + "source_file": "notes/group.md"}], + "edges": [], + "hyperedges": [{**HYPEREDGE, "nodes": ["beta_node", "gamma_node"], "label": "The Group v2"}]} + G = build_merge([fresh], _baseline(tmp_path)) + hes = [he for he in G.graph.get("hyperedges", []) if he["id"] == "the_group"] + assert len(hes) == 1 + assert hes[0]["label"] == "The Group v2" # the re-extracted version wins + + +def test_a_pruned_sources_hyperedge_is_still_dropped(tmp_path): + G = build_merge([UNRELATED_CHUNK], _baseline(tmp_path), prune_sources=["notes/group.md"]) + assert "the_group" not in _hyperedges(G) + + +def test_an_unchanged_hyperedge_with_no_dedup_involved_is_carried_verbatim(tmp_path): + nodes = [n for n in NODES if n["id"] != "alpha_a"] # nothing to dedup now + G0 = build_from_json({"nodes": nodes, "edges": EDGES, "hyperedges": [HYPEREDGE]}) + p = tmp_path / "g.json" + to_json(G0, {0: list(G0.nodes)}, str(p)) + G = build_merge([UNRELATED_CHUNK], p) + assert set(_hyperedges(G)["the_group"]["nodes"]) == set(HYPEREDGE["nodes"]) From a872a3fcfd66c1c5e304b6f73b93fdf9069c09f8 Mon Sep 17 00:00:00 2001 From: abhay-codes07 Date: Wed, 26 Aug 2026 18:31:15 +0530 Subject: [PATCH 2/2] fix(build): materialise new_chunks before build_merge iterates it more than once --- graphify/build.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/graphify/build.py b/graphify/build.py index 10cdec28f..29acd2526 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -1649,6 +1649,9 @@ def build_merge( graph to inherit from. An explicit True/False always overrides the on-disk flag. """ + # Iterated more than once below (source sets, the hyperedge carry, the + # build itself), so a one-shot iterator must be materialised first. + new_chunks = list(new_chunks) graph_path = Path(graph_path if graph_path is not None else _default_graph_json()) _loaded = _load_existing_graph(graph_path) if _loaded is not None: