From 6c2bfb84323c1fa7aa17dafc867a029194ffac95 Mon Sep 17 00:00:00 2001 From: Nimra Khalid Date: Sat, 22 Aug 2026 16:02:01 +0500 Subject: [PATCH 1/3] fix: EmbeddingBasedDocumentSplitter can't split exactly-two-group docs _find_split_points compares each consecutive-pair distance against a percentile threshold computed over all distances. With exactly two sentence groups there is only one distance, and np.percentile of a single-element array always returns that same element - so `distance > threshold` reduced to `distance > distance`, always False, regardless of percentile setting (including the most aggressive, percentile=0.0). Any short document with a clear topic break that happens to tokenize into exactly two sentence groups silently failed to split. Special-case the single-distance scenario: treat the lone gap as a split point unless the two groups are identical. Verified this doesn't change behavior for documents with >=3 groups. Added tests proving fail-then-pass, plus a release note per project convention. --- .../embedding_based_document_splitter.py | 9 ++++ ...-splitter-two-groups-48201a43ca06f07b.yaml | 8 +++ .../test_embedding_based_document_splitter.py | 49 +++++++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 releasenotes/notes/fix-embedding-splitter-two-groups-48201a43ca06f07b.yaml diff --git a/haystack/components/preprocessors/embedding_based_document_splitter.py b/haystack/components/preprocessors/embedding_based_document_splitter.py index 17acefd3b7..099b7a42e2 100644 --- a/haystack/components/preprocessors/embedding_based_document_splitter.py +++ b/haystack/components/preprocessors/embedding_based_document_splitter.py @@ -367,6 +367,15 @@ def _find_split_points(self, embeddings: list[list[float]]) -> list[int]: ) distances.append(distance) + # With a single distance (i.e. exactly two sentence groups) there is no distribution to compute a + # percentile against: np.percentile of a one-element list always returns that same element, so + # `distance > threshold` below would compare the value to itself and could never be True. Without this + # special case, a document that happens to tokenize into exactly two sentence groups could never be + # split, regardless of how dissimilar the groups are or how low `percentile` is set. Instead, treat the + # lone gap as a split point unless the two groups are identical. + if len(distances) == 1: + return [1] if distances[0] > 0 else [] + # Calculate threshold based on percentile threshold = np.percentile(distances, self.percentile * 100) diff --git a/releasenotes/notes/fix-embedding-splitter-two-groups-48201a43ca06f07b.yaml b/releasenotes/notes/fix-embedding-splitter-two-groups-48201a43ca06f07b.yaml new file mode 100644 index 0000000000..6220b36ac8 --- /dev/null +++ b/releasenotes/notes/fix-embedding-splitter-two-groups-48201a43ca06f07b.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + `EmbeddingBasedDocumentSplitter` could never split a document that tokenized into exactly two sentence + groups, no matter how dissimilar the groups were or how low `percentile` was set. With a single distance + to compare, `numpy.percentile` always returns that same value, so the `distance > threshold` check was + comparing the value to itself and could never be `True`. The splitter now treats the lone gap as a split + point unless the two groups are identical. diff --git a/test/components/preprocessors/test_embedding_based_document_splitter.py b/test/components/preprocessors/test_embedding_based_document_splitter.py index 50db2a8aa8..fa70a938ef 100644 --- a/test/components/preprocessors/test_embedding_based_document_splitter.py +++ b/test/components/preprocessors/test_embedding_based_document_splitter.py @@ -170,6 +170,55 @@ def test_find_split_points(self): # Should find a split point after the second embedding (index 2) assert 2 in split_points + def test_find_split_points_two_groups_can_split(self): + """ + With exactly two sentence groups there is only one distance to compare, so np.percentile of that + single-element list always returns the value itself. `distance > threshold` was therefore comparing + the value to itself and could never be True, meaning a document that tokenizes into exactly two + sentence groups could never be split, no matter how dissimilar the groups were or how low + `percentile` was set. Maximally dissimilar (orthogonal) embeddings, even at the most aggressive + percentile=0.0 setting, must produce a split point. + """ + mock_embedder = Mock() + splitter = EmbeddingBasedDocumentSplitter(document_embedder=mock_embedder, percentile=0.0) + + embeddings = [[1.0, 0.0], [0.0, 1.0]] # orthogonal -> maximum possible cosine distance + assert splitter._find_split_points(embeddings) == [1] + + def test_find_split_points_two_identical_groups_do_not_split(self): + """Two identical groups (distance == 0) must still not produce a split point.""" + mock_embedder = Mock() + splitter = EmbeddingBasedDocumentSplitter(document_embedder=mock_embedder, percentile=0.0) + + embeddings = [[1.0, 0.0], [1.0, 0.0]] + assert splitter._find_split_points(embeddings) == [] + + def test_run_splits_document_with_exactly_two_sentence_groups(self): + """ + End-to-end regression test: a document that tokenizes into exactly two sentence groups with + maximally dissimilar content must still be split by run(), even at the most aggressive + percentile=0.0 setting. + """ + text = ( + "Astronomy studies the stars and galaxies of outer space. " + "Sourdough baking relies on wild yeast fermentation." + ) + + def mock_run(documents): + embeddings = [[1.0, 0.0] if i == 0 else [0.0, 1.0] for i in range(len(documents))] + return {"documents": [replace(doc, embedding=emb) for doc, emb in zip(documents, embeddings, strict=True)]} + + mock_embedder = Mock() + mock_embedder.run = Mock(side_effect=mock_run) + + splitter = EmbeddingBasedDocumentSplitter( + document_embedder=mock_embedder, sentences_per_group=1, percentile=0.0, min_length=0, max_length=10000 + ) + splitter.warm_up() + + result = splitter.run(documents=[Document(content=text)]) + assert len(result["documents"]) == 2 + def test_create_splits_from_points(self): mock_embedder = Mock() splitter = EmbeddingBasedDocumentSplitter(document_embedder=mock_embedder) From 2451545cb8498ae476ab9a46359aff221221e554 Mon Sep 17 00:00:00 2001 From: Nimra Khalid Date: Sat, 22 Aug 2026 17:35:24 +0500 Subject: [PATCH 2/3] fix: replace always-split hack with absolute distance floor for two-group docs The previous fix for the two-group case (distances[0] > 0) split on any nonzero distance, which meant percentile was silently ignored whenever a document tokenized into exactly two sentence groups: even at the most conservative percentile=1.0, two near-duplicate groups would still split, while the same near-duplicate pair as part of 3+ groups correctly would not. percentile is mathematically undefined for a single-element distribution, so this now falls back to a fixed absolute cosine-distance floor instead of comparing the single distance to itself or to zero. --- .../embedding_based_document_splitter.py | 21 +++++++++++++------ ...-splitter-two-groups-48201a43ca06f07b.yaml | 6 ++++-- .../test_embedding_based_document_splitter.py | 16 ++++++++++++++ 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/haystack/components/preprocessors/embedding_based_document_splitter.py b/haystack/components/preprocessors/embedding_based_document_splitter.py index 099b7a42e2..a9573d224d 100644 --- a/haystack/components/preprocessors/embedding_based_document_splitter.py +++ b/haystack/components/preprocessors/embedding_based_document_splitter.py @@ -19,6 +19,13 @@ logger = logging.getLogger(__name__) +# When a document tokenizes into exactly two sentence groups, there is only one cosine distance to work with. +# `percentile` has no meaning for a single-element distribution (np.percentile of one value always returns that +# same value, whatever the requested percentile), so it cannot be used to decide whether that lone gap is a split +# point. Instead we compare it against this fixed cosine-distance floor. Two consecutive groups closer than this +# are treated as a paraphrase/near-duplicate rather than a topic change; `percentile` is ignored in this one case. +_MIN_SPLIT_DISTANCE_FOR_SINGLE_GAP = 0.01 + @component class EmbeddingBasedDocumentSplitter: @@ -86,7 +93,9 @@ def __init__( :param document_embedder: The DocumentEmbedder to use for calculating embeddings. :param sentences_per_group: Number of sentences to group together before embedding. :param percentile: Percentile threshold for cosine distance. Distances above this percentile - are treated as break points. + are treated as break points. Does not apply to documents that tokenize into exactly two sentence + groups: with only one distance to evaluate there is no distribution to compute a percentile against, + so a fixed absolute distance floor is used instead (see `_MIN_SPLIT_DISTANCE_FOR_SINGLE_GAP`). :param min_length: Minimum length of splits in characters. Splits below this length will be merged. :param max_length: Maximum length of splits in characters. Splits above this length will be recursively split. :param language: Language for sentence tokenization. @@ -369,12 +378,12 @@ def _find_split_points(self, embeddings: list[list[float]]) -> list[int]: # With a single distance (i.e. exactly two sentence groups) there is no distribution to compute a # percentile against: np.percentile of a one-element list always returns that same element, so - # `distance > threshold` below would compare the value to itself and could never be True. Without this - # special case, a document that happens to tokenize into exactly two sentence groups could never be - # split, regardless of how dissimilar the groups are or how low `percentile` is set. Instead, treat the - # lone gap as a split point unless the two groups are identical. + # `distance > threshold` below would always compare the value to itself and could never be True. Without + # this special case, a document that happens to tokenize into exactly two sentence groups could never be + # split, regardless of how dissimilar the groups are or how low `percentile` is set. `percentile` cannot + # apply here, so we fall back to a fixed absolute distance floor instead. if len(distances) == 1: - return [1] if distances[0] > 0 else [] + return [1] if distances[0] > _MIN_SPLIT_DISTANCE_FOR_SINGLE_GAP else [] # Calculate threshold based on percentile threshold = np.percentile(distances, self.percentile * 100) diff --git a/releasenotes/notes/fix-embedding-splitter-two-groups-48201a43ca06f07b.yaml b/releasenotes/notes/fix-embedding-splitter-two-groups-48201a43ca06f07b.yaml index 6220b36ac8..807090cb8a 100644 --- a/releasenotes/notes/fix-embedding-splitter-two-groups-48201a43ca06f07b.yaml +++ b/releasenotes/notes/fix-embedding-splitter-two-groups-48201a43ca06f07b.yaml @@ -4,5 +4,7 @@ fixes: `EmbeddingBasedDocumentSplitter` could never split a document that tokenized into exactly two sentence groups, no matter how dissimilar the groups were or how low `percentile` was set. With a single distance to compare, `numpy.percentile` always returns that same value, so the `distance > threshold` check was - comparing the value to itself and could never be `True`. The splitter now treats the lone gap as a split - point unless the two groups are identical. + comparing the value to itself and could never be `True`. Since `percentile` requires a distribution of + distances to be meaningful and a single gap doesn't provide one, the splitter now falls back to a fixed + absolute cosine-distance floor for this case: the lone gap becomes a split point once the two groups are + no longer near-duplicates, regardless of `percentile`. diff --git a/test/components/preprocessors/test_embedding_based_document_splitter.py b/test/components/preprocessors/test_embedding_based_document_splitter.py index fa70a938ef..cc48b763fc 100644 --- a/test/components/preprocessors/test_embedding_based_document_splitter.py +++ b/test/components/preprocessors/test_embedding_based_document_splitter.py @@ -193,6 +193,22 @@ def test_find_split_points_two_identical_groups_do_not_split(self): embeddings = [[1.0, 0.0], [1.0, 0.0]] assert splitter._find_split_points(embeddings) == [] + def test_find_split_points_two_near_duplicate_groups_do_not_split_regardless_of_percentile(self): + """ + Regression test: comparing the lone distance to itself (the original bug) or splitting on any + nonzero distance (an earlier, overly aggressive fix) both fail on near-duplicate groups. Two + groups that are almost, but not exactly, identical (a tiny cosine distance well under the + `_MIN_SPLIT_DISTANCE_FOR_SINGLE_GAP` floor) should not split, and since `percentile` does not + apply to the two-group case, this must hold at every percentile setting, including the most + aggressive (0.0). + """ + mock_embedder = Mock() + embeddings = [[1.0, 0.0], [1.0, 0.001]] # cosine distance ~5e-7: near-duplicate, not identical + + for percentile in (0.0, 0.5, 0.95, 1.0): + splitter = EmbeddingBasedDocumentSplitter(document_embedder=mock_embedder, percentile=percentile) + assert splitter._find_split_points(embeddings) == [], f"unexpected split at percentile={percentile}" + def test_run_splits_document_with_exactly_two_sentence_groups(self): """ End-to-end regression test: a document that tokenizes into exactly two sentence groups with From 075cb531b736b18adfc01a255ebc940e62858c57 Mon Sep 17 00:00:00 2001 From: Nimra Khalid Date: Sun, 23 Aug 2026 14:44:19 +0500 Subject: [PATCH 3/3] fix: make two-group split threshold an explicit, opt-in parameter Empirically verified against a real local embedding model (MiniLM) that a fixed absolute cosine-distance floor doesn't generalize: same-topic paraphrase pairs land around 0.09-0.27 distance while genuine topic changes land around 0.98-1.08 for that model, so the previous floor of 0.01 barely filtered anything beyond near-bit-identical text. Different embedding models compress or spread cosine distances very differently, so there is no single constant that works across embedders. Replaced it with `min_distance_for_two_groups`, defaulting to None (never split a two-group document, matching prior behavior) and documented as something users calibrate against their own embedder's distance distribution rather than a value this component can pick for them. --- .../embedding_based_document_splitter.py | 38 +++++++---- ...-splitter-two-groups-48201a43ca06f07b.yaml | 16 +++-- .../test_embedding_based_document_splitter.py | 67 ++++++++++++++----- 3 files changed, 86 insertions(+), 35 deletions(-) diff --git a/haystack/components/preprocessors/embedding_based_document_splitter.py b/haystack/components/preprocessors/embedding_based_document_splitter.py index a9573d224d..ba0df72c68 100644 --- a/haystack/components/preprocessors/embedding_based_document_splitter.py +++ b/haystack/components/preprocessors/embedding_based_document_splitter.py @@ -19,13 +19,6 @@ logger = logging.getLogger(__name__) -# When a document tokenizes into exactly two sentence groups, there is only one cosine distance to work with. -# `percentile` has no meaning for a single-element distribution (np.percentile of one value always returns that -# same value, whatever the requested percentile), so it cannot be used to decide whether that lone gap is a split -# point. Instead we compare it against this fixed cosine-distance floor. Two consecutive groups closer than this -# are treated as a paraphrase/near-duplicate rather than a topic change; `percentile` is ignored in this one case. -_MIN_SPLIT_DISTANCE_FOR_SINGLE_GAP = 0.01 - @component class EmbeddingBasedDocumentSplitter: @@ -86,6 +79,7 @@ def __init__( language: Language = "en", use_split_rules: bool = True, extend_abbreviations: bool = True, + min_distance_for_two_groups: float | None = None, ) -> None: """ Initialize EmbeddingBasedDocumentSplitter. @@ -95,7 +89,7 @@ def __init__( :param percentile: Percentile threshold for cosine distance. Distances above this percentile are treated as break points. Does not apply to documents that tokenize into exactly two sentence groups: with only one distance to evaluate there is no distribution to compute a percentile against, - so a fixed absolute distance floor is used instead (see `_MIN_SPLIT_DISTANCE_FOR_SINGLE_GAP`). + so `min_distance_for_two_groups` is used instead for that case. :param min_length: Minimum length of splits in characters. Splits below this length will be merged. :param max_length: Maximum length of splits in characters. Splits above this length will be recursively split. :param language: Language for sentence tokenization. @@ -104,6 +98,17 @@ def __init__( :param extend_abbreviations: If True, the abbreviations used by NLTK's PunktTokenizer are extended by a list of curated abbreviations. Currently supported languages are: en, de. If False, the default abbreviations are used. + :param min_distance_for_two_groups: Cosine distance threshold used only for documents that tokenize into + exactly two sentence groups. With a single distance to evaluate there is no distribution to compute + `percentile` against, so this fixed, absolute threshold is used instead: the lone gap becomes a split + point if its distance exceeds this value. There is no universally correct default: what counts as + "the same topic, reworded" versus "a genuine topic change" is specific to the embedding model in use, + and the two can differ in cosine distance by an order of magnitude or more between models (some + embedders compress all pairwise distances into a narrow band, others spread them out). Defaults to + `None`, meaning documents with exactly two sentence groups are never split, matching the behavior of + documents with only one group. To enable splitting for this case, measure cosine distances from your + own embedder on a few known same-topic and known different-topic sentence pairs, and set this to a + value between the two. """ self.document_embedder = document_embedder @@ -123,6 +128,10 @@ def __init__( raise ValueError("max_length must be greater than min_length.") self.max_length = max_length + if min_distance_for_two_groups is not None and not 0.0 <= min_distance_for_two_groups <= 2.0: + raise ValueError("min_distance_for_two_groups must be between 0.0 and 2.0.") + self.min_distance_for_two_groups = min_distance_for_two_groups + self.language = language self.use_split_rules = use_split_rules self.extend_abbreviations = extend_abbreviations @@ -378,12 +387,14 @@ def _find_split_points(self, embeddings: list[list[float]]) -> list[int]: # With a single distance (i.e. exactly two sentence groups) there is no distribution to compute a # percentile against: np.percentile of a one-element list always returns that same element, so - # `distance > threshold` below would always compare the value to itself and could never be True. Without - # this special case, a document that happens to tokenize into exactly two sentence groups could never be - # split, regardless of how dissimilar the groups are or how low `percentile` is set. `percentile` cannot - # apply here, so we fall back to a fixed absolute distance floor instead. + # `distance > threshold` below would always compare the value to itself and could never be True. + # `percentile` cannot apply here, so we fall back to `min_distance_for_two_groups` instead, which + # defaults to `None` (never split) since there's no universally correct absolute distance threshold + # across embedding models -- see its docstring in `__init__`. if len(distances) == 1: - return [1] if distances[0] > _MIN_SPLIT_DISTANCE_FOR_SINGLE_GAP else [] + if self.min_distance_for_two_groups is None: + return [] + return [1] if distances[0] > self.min_distance_for_two_groups else [] # Calculate threshold based on percentile threshold = np.percentile(distances, self.percentile * 100) @@ -590,6 +601,7 @@ def to_dict(self) -> dict[str, Any]: language=self.language, use_split_rules=self.use_split_rules, extend_abbreviations=self.extend_abbreviations, + min_distance_for_two_groups=self.min_distance_for_two_groups, ) @classmethod diff --git a/releasenotes/notes/fix-embedding-splitter-two-groups-48201a43ca06f07b.yaml b/releasenotes/notes/fix-embedding-splitter-two-groups-48201a43ca06f07b.yaml index 807090cb8a..754faae6f0 100644 --- a/releasenotes/notes/fix-embedding-splitter-two-groups-48201a43ca06f07b.yaml +++ b/releasenotes/notes/fix-embedding-splitter-two-groups-48201a43ca06f07b.yaml @@ -4,7 +4,15 @@ fixes: `EmbeddingBasedDocumentSplitter` could never split a document that tokenized into exactly two sentence groups, no matter how dissimilar the groups were or how low `percentile` was set. With a single distance to compare, `numpy.percentile` always returns that same value, so the `distance > threshold` check was - comparing the value to itself and could never be `True`. Since `percentile` requires a distribution of - distances to be meaningful and a single gap doesn't provide one, the splitter now falls back to a fixed - absolute cosine-distance floor for this case: the lone gap becomes a split point once the two groups are - no longer near-duplicates, regardless of `percentile`. + comparing the value to itself and could never be `True`. `percentile` cannot decide this case, since it + requires a distribution of distances to be meaningful and a single gap doesn't provide one. Splitting a + two-group document is now available via the new `min_distance_for_two_groups` parameter (see below); + without it, two-group documents keep the previous, safe behavior of never being split. +enhancements: + - | + Added `min_distance_for_two_groups` to `EmbeddingBasedDocumentSplitter`, a cosine-distance threshold used + only for documents that tokenize into exactly two sentence groups, where `percentile` cannot apply. + Defaults to `None` (never split this case). There is no universally correct default value: how far apart + "the same topic, reworded" and "a genuine topic change" sit in cosine distance is specific to the + embedding model in use and can vary by an order of magnitude or more between models, so this is left for + users to calibrate against their own embedder. diff --git a/test/components/preprocessors/test_embedding_based_document_splitter.py b/test/components/preprocessors/test_embedding_based_document_splitter.py index cc48b763fc..af80692e16 100644 --- a/test/components/preprocessors/test_embedding_based_document_splitter.py +++ b/test/components/preprocessors/test_embedding_based_document_splitter.py @@ -32,6 +32,7 @@ def test_init(self): assert splitter.percentile == 0.9 assert splitter.min_length == 50 assert splitter.max_length == 1000 + assert splitter.min_distance_for_two_groups is None def test_init_invalid_sentences_per_group(self): mock_embedder = Mock() @@ -43,6 +44,11 @@ def test_init_invalid_percentile(self): with pytest.raises(ValueError, match="percentile must be between 0.0 and 1.0"): EmbeddingBasedDocumentSplitter(document_embedder=mock_embedder, percentile=1.5) + def test_init_invalid_min_distance_for_two_groups(self): + mock_embedder = Mock() + with pytest.raises(ValueError, match="min_distance_for_two_groups must be between 0.0 and 2.0"): + EmbeddingBasedDocumentSplitter(document_embedder=mock_embedder, min_distance_for_two_groups=2.5) + def test_init_invalid_min_length(self): mock_embedder = Mock() with pytest.raises(ValueError, match="min_length must be greater than or equal to 0"): @@ -170,25 +176,43 @@ def test_find_split_points(self): # Should find a split point after the second embedding (index 2) assert 2 in split_points - def test_find_split_points_two_groups_can_split(self): + def test_find_split_points_two_groups_default_never_splits(self): """ With exactly two sentence groups there is only one distance to compare, so np.percentile of that - single-element list always returns the value itself. `distance > threshold` was therefore comparing - the value to itself and could never be True, meaning a document that tokenizes into exactly two - sentence groups could never be split, no matter how dissimilar the groups were or how low - `percentile` was set. Maximally dissimilar (orthogonal) embeddings, even at the most aggressive - percentile=0.0 setting, must produce a split point. + single-element list always returns the value itself: `distance > threshold` is comparing the value + to itself and can never be True. `percentile` therefore cannot decide this case, and there is no + universally correct absolute distance threshold across embedding models to fall back on either (see + `min_distance_for_two_groups`'s docstring). The safe default is to never split here, matching the + pre-existing behavior for documents with a single sentence group -- even for maximally dissimilar + (orthogonal) embeddings, at the most aggressive percentile=0.0 setting. """ mock_embedder = Mock() splitter = EmbeddingBasedDocumentSplitter(document_embedder=mock_embedder, percentile=0.0) + assert splitter.min_distance_for_two_groups is None embeddings = [[1.0, 0.0], [0.0, 1.0]] # orthogonal -> maximum possible cosine distance - assert splitter._find_split_points(embeddings) == [1] + assert splitter._find_split_points(embeddings) == [] + + def test_find_split_points_two_groups_splits_once_min_distance_for_two_groups_is_configured(self): + """ + Splitting a two-group document is opt-in: once `min_distance_for_two_groups` is set, the lone + distance is compared against it directly, independent of `percentile` (which cannot apply here). + """ + mock_embedder = Mock() + embeddings = [[1.0, 0.0], [0.0, 1.0]] # orthogonal -> cosine distance of 1.0 + + for percentile in (0.0, 0.5, 0.95, 1.0): + splitter = EmbeddingBasedDocumentSplitter( + document_embedder=mock_embedder, percentile=percentile, min_distance_for_two_groups=0.5 + ) + assert splitter._find_split_points(embeddings) == [1], f"expected split at percentile={percentile}" def test_find_split_points_two_identical_groups_do_not_split(self): - """Two identical groups (distance == 0) must still not produce a split point.""" + """Two identical groups (distance == 0) must not produce a split point even at the threshold's edge.""" mock_embedder = Mock() - splitter = EmbeddingBasedDocumentSplitter(document_embedder=mock_embedder, percentile=0.0) + splitter = EmbeddingBasedDocumentSplitter( + document_embedder=mock_embedder, percentile=0.0, min_distance_for_two_groups=0.0 + ) embeddings = [[1.0, 0.0], [1.0, 0.0]] assert splitter._find_split_points(embeddings) == [] @@ -196,24 +220,25 @@ def test_find_split_points_two_identical_groups_do_not_split(self): def test_find_split_points_two_near_duplicate_groups_do_not_split_regardless_of_percentile(self): """ Regression test: comparing the lone distance to itself (the original bug) or splitting on any - nonzero distance (an earlier, overly aggressive fix) both fail on near-duplicate groups. Two - groups that are almost, but not exactly, identical (a tiny cosine distance well under the - `_MIN_SPLIT_DISTANCE_FOR_SINGLE_GAP` floor) should not split, and since `percentile` does not - apply to the two-group case, this must hold at every percentile setting, including the most - aggressive (0.0). + nonzero distance (an earlier, overly aggressive fix) both fail on near-duplicate groups. Two groups + that are almost, but not exactly, identical (a tiny cosine distance well under a realistic + `min_distance_for_two_groups`) should not split, and since `percentile` does not apply to the + two-group case, this must hold at every percentile setting, including the most aggressive (0.0). """ mock_embedder = Mock() embeddings = [[1.0, 0.0], [1.0, 0.001]] # cosine distance ~5e-7: near-duplicate, not identical for percentile in (0.0, 0.5, 0.95, 1.0): - splitter = EmbeddingBasedDocumentSplitter(document_embedder=mock_embedder, percentile=percentile) + splitter = EmbeddingBasedDocumentSplitter( + document_embedder=mock_embedder, percentile=percentile, min_distance_for_two_groups=0.3 + ) assert splitter._find_split_points(embeddings) == [], f"unexpected split at percentile={percentile}" def test_run_splits_document_with_exactly_two_sentence_groups(self): """ End-to-end regression test: a document that tokenizes into exactly two sentence groups with - maximally dissimilar content must still be split by run(), even at the most aggressive - percentile=0.0 setting. + maximally dissimilar content is split by run() once `min_distance_for_two_groups` is configured, + even at the most aggressive percentile=0.0 setting. """ text = ( "Astronomy studies the stars and galaxies of outer space. " @@ -228,7 +253,12 @@ def mock_run(documents): mock_embedder.run = Mock(side_effect=mock_run) splitter = EmbeddingBasedDocumentSplitter( - document_embedder=mock_embedder, sentences_per_group=1, percentile=0.0, min_length=0, max_length=10000 + document_embedder=mock_embedder, + sentences_per_group=1, + percentile=0.0, + min_length=0, + max_length=10000, + min_distance_for_two_groups=0.5, ) splitter.warm_up() @@ -444,6 +474,7 @@ def test_to_dict(self): assert result["init_parameters"]["percentile"] == 0.9 assert result["init_parameters"]["min_length"] == 50 assert result["init_parameters"]["max_length"] == 1000 + assert result["init_parameters"]["min_distance_for_two_groups"] is None assert "document_embedder" in result["init_parameters"] @pytest.mark.skipif(os.environ.get("OPENAI_API_KEY", "") == "", reason="OPENAI_API_KEY is not set")