From 2189ffe3f3c8fae4691b561838ba58007f4682d8 Mon Sep 17 00:00:00 2001 From: WilliamK112 <164879897+WilliamK112@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:36:31 +0800 Subject: [PATCH 1/2] feat: add CleanReport JSON export --- CHANGELOG.md | 2 ++ docs/quickstart.md | 2 ++ examples/08_csv_automation.py | 5 +---- src/freshdata/report.py | 14 ++++++++++++++ tests/test_report.py | 22 ++++++++++++++++++++++ 5 files changed, 41 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff77cfae..9ae62218 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] ### Added +- `CleanReport.to_json()` and `CleanReport.write_json()` for first-class audit + report serialization without manual `json.dumps(...)` calls. - Added a runnable PyJanitor interoperability example that demonstrates both tool orderings while keeping PyJanitor optional. - A dependency-optional Great Expectations recipe demonstrating the diff --git a/docs/quickstart.md b/docs/quickstart.md index 7c10b0ae..1e8f2a29 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -48,6 +48,8 @@ The report is also machine-readable: ```python report.to_frame() # one row per decision, as a DataFrame report.to_dict() # JSON-friendly for logging / dashboards +report.to_json(indent=2) # JSON string +report.write_json("report.json", indent=2) # UTF-8 JSON file ``` ## Preview before cleaning diff --git a/examples/08_csv_automation.py b/examples/08_csv_automation.py index 87cd1cae..990b3f5c 100644 --- a/examples/08_csv_automation.py +++ b/examples/08_csv_automation.py @@ -6,7 +6,6 @@ python examples/08_csv_automation.py """ -import json import tempfile from pathlib import Path @@ -43,9 +42,7 @@ def main() -> None: for path in sorted(inbox.glob("*.csv")): cleaned = cleaner.clean(pd.read_csv(path)) cleaned.to_csv(outbox / path.name, index=False) - (logs / f"{path.stem}.json").write_text( - json.dumps(cleaner.report_.to_dict(), indent=2, default=str) - ) + cleaner.report_.write_json(logs / f"{path.stem}.json", indent=2) print(f"{path.name}: {cleaner.report_.summary().splitlines()[0]}") print(f"\nCleaned {len(list(outbox.glob('*.csv')))} files; " diff --git a/src/freshdata/report.py b/src/freshdata/report.py index b60841b0..e8ebecec 100644 --- a/src/freshdata/report.py +++ b/src/freshdata/report.py @@ -10,8 +10,10 @@ from __future__ import annotations import contextlib +import json from collections.abc import Iterator from dataclasses import dataclass, field +from pathlib import Path from typing import Any import pandas as pd @@ -379,6 +381,18 @@ def to_dict(self) -> dict[str, Any]: payload["profile_replay"] = dict(self.profile_replay) return payload + def to_json(self, **kwargs: Any) -> str: + """Serialize the report's stable audit payload as JSON. + + Keyword arguments are forwarded to :func:`json.dumps`, so callers can + choose options such as ``indent=2`` or ``sort_keys=True``. + """ + return json.dumps(self.to_dict(), **kwargs) + + def write_json(self, path: str | Path, **kwargs: Any) -> None: + """Write the report's JSON audit payload to *path* as UTF-8 text.""" + Path(path).write_text(self.to_json(**kwargs) + "\n", encoding="utf-8") + def revert( self, df: pd.DataFrame, action_ids: list[str] | None = None ) -> pd.DataFrame: diff --git a/tests/test_report.py b/tests/test_report.py index 4fc9fc7a..8768ebce 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -1,4 +1,5 @@ import json +from pathlib import Path import pandas as pd @@ -31,6 +32,27 @@ def test_to_dict_is_json_serializable(messy): assert len(payload_dict["actions"]) == len(report) +def test_to_json_serializes_real_report(messy): + _, report = fd.clean(messy, return_report=True) + + payload = json.loads(report.to_json(indent=2, sort_keys=True)) + + assert payload == report.to_dict() + assert len(payload["actions"]) == len(report) + + +def test_write_json_round_trip(messy, tmp_path: Path): + _, report = fd.clean(messy, return_report=True) + path = tmp_path / "audit" / "report.json" + path.parent.mkdir() + + result = report.write_json(path, indent=2) + + assert result is None + assert json.loads(path.read_text(encoding="utf-8")) == report.to_dict() + assert path.read_bytes().endswith(b"\n") + + def test_to_frame(messy): _, report = fd.clean(messy, return_report=True) frame = report.to_frame() From 55814f337b10b9c94ea6eb964a7a62f75744b5f4 Mon Sep 17 00:00:00 2001 From: WilliamK112 <164879897+WilliamK112@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:36:41 +0800 Subject: [PATCH 2/2] fix(report): serialize nested audit metadata --- src/freshdata/report.py | 5 ++++- tests/test_report.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/freshdata/report.py b/src/freshdata/report.py index e8ebecec..381b08ca 100644 --- a/src/freshdata/report.py +++ b/src/freshdata/report.py @@ -385,8 +385,11 @@ def to_json(self, **kwargs: Any) -> str: """Serialize the report's stable audit payload as JSON. Keyword arguments are forwarded to :func:`json.dumps`, so callers can - choose options such as ``indent=2`` or ``sort_keys=True``. + choose options such as ``indent=2`` or ``sort_keys=True``. Nested audit + values outside JSON's native types are stringified by default; pass a + custom ``default=`` callback to override that behavior. """ + kwargs.setdefault("default", str) return json.dumps(self.to_dict(), **kwargs) def write_json(self, path: str | Path, **kwargs: Any) -> None: diff --git a/tests/test_report.py b/tests/test_report.py index 8768ebce..798c9ab9 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -1,4 +1,5 @@ import json +from datetime import datetime, timezone from pathlib import Path import pandas as pd @@ -53,6 +54,23 @@ def test_write_json_round_trip(messy, tmp_path: Path): assert path.read_bytes().endswith(b"\n") +def test_json_exports_serialize_nested_audit_values(messy, tmp_path: Path): + extracted_at = datetime(2026, 8, 20, 9, 30, tzinfo=timezone.utc) + _, report = fd.clean( + messy, + source_provenance={"AGE": {"extracted_at": extracted_at}}, + return_report=True, + ) + path = tmp_path / "report.json" + + payload = json.loads(report.to_json()) + report.write_json(path) + + assert report.to_dict()["source_provenance"]["AGE"]["extracted_at"] is extracted_at + assert payload["source_provenance"]["AGE"]["extracted_at"] == str(extracted_at) + assert json.loads(path.read_text(encoding="utf-8")) == payload + + def test_to_frame(messy): _, report = fd.clean(messy, return_report=True) frame = report.to_frame()