From 41bc51bd5a403c5982f361ea3fad31cc21b5a557 Mon Sep 17 00:00:00 2001 From: Matthew Marine Date: Thu, 23 Jul 2026 08:00:27 -0400 Subject: [PATCH 1/9] Update WriteDREAM3DFilter testing Add V&V documentation --- .../SimplnxCore/test/DREAM3DFileTest.cpp | 342 ++++++++++++++++++ .../SimplnxCore/vv/WriteDREAM3DFilter.md | 121 +++++++ .../vv/deviations/WriteDREAM3DFilter.md | 41 +++ 3 files changed, 504 insertions(+) create mode 100644 src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md create mode 100644 src/Plugins/SimplnxCore/vv/deviations/WriteDREAM3DFilter.md diff --git a/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp b/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp index d5f1b68dff..a764ff1f1a 100644 --- a/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp +++ b/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp @@ -22,6 +22,7 @@ #include "simplnx/DataStructure/IDataArray.hpp" #include "simplnx/DataStructure/IDataStore.hpp" #include "simplnx/DataStructure/IO/HDF5/DataStructureReader.hpp" +#include "simplnx/DataStructure/ListStore.hpp" #include "simplnx/Filter/Arguments.hpp" #include "simplnx/Filter/FilterHandle.hpp" #include "simplnx/Parameters/Dream3dImportParameter.hpp" @@ -59,6 +60,26 @@ const fs::path k_ExportFilename2 = "export2.dream3d"; const fs::path k_MultiExportFilename1 = "multi_export1.dream3d"; const fs::path k_MultiExportFilename2 = "multi_export2.dream3d"; const fs::path k_MultiExportFilename3 = "multi_export3.dream3d"; + +constexpr StringLiteral k_CellData = "Cell Data"; +constexpr StringLiteral k_DataContainer = "Data Container"; +constexpr StringLiteral k_EdgeGeom = "EdgeGeom"; +constexpr StringLiteral k_ImageGeom = "ImageGeom"; +constexpr StringLiteral k_HexGeom = "HexahedralGeom"; +constexpr StringLiteral k_QuadGeom = "QuadGeom"; +constexpr StringLiteral k_TetrahedralGeom = "TetrahedralGeom"; +constexpr StringLiteral k_TriangleGeom = "TriangleGeom"; +constexpr StringLiteral k_VertexGeom = "VertexGeom"; + +constexpr StringLiteral k_DynamicListArray = "DynamicList"; +constexpr StringLiteral k_NeighborList = "NeighborList"; +constexpr StringLiteral k_StringArray = "String Array"; +constexpr StringLiteral k_VertexList = "Vertices"; +constexpr StringLiteral k_Edges = "Edges"; +constexpr StringLiteral k_Faces = "Faces"; +constexpr StringLiteral k_Polyhedra = "Polyhedrals"; +const ShapeType k_TupleShape{3, 2, 1}; +constexpr int16 k_ListCount = 6; } // namespace Constants std::mutex m_DataMutex; @@ -152,6 +173,188 @@ fs::path GetReMultiExportDataPath() return GetDataDir(*app) / Constants::k_MultiExportFilename3; } +/** + * @brief Creates and sets arrays for a 2D geometry of type T. + * @param dataStructure + * @param name + * @param vertexArray + * @param edgeList + * @param faceList + * @return T* + */ +template +T* Create2DGeom(DataStructure& dataStructure, const std::string& name, const IGeometry::SharedVertexList& vertexArray, const IGeometry::SharedEdgeList& edgeList, + const IGeometry::SharedFaceList& faceList) +{ + auto* geom = T::Create(dataStructure, name); + geom->setVertices(vertexArray); + geom->setEdgeList(edgeList); + geom->setFaceList(faceList); + return geom; +} + +/** + * @brief Creates and sets arrays for a 3D geometry of type T. + * @param dataStructure + * @param name + * @param vertexArray + * @param edgeList + * @param faceList + * @param polyArray + * @return T* + */ +template +T* Create3DGeom(DataStructure& dataStructure, const std::string& name, const IGeometry::SharedVertexList& vertexArray, const IGeometry::SharedEdgeList& edgeList, + const IGeometry::SharedFaceList& faceList, const IGeometry::SharedHexList& polyArray) +{ + auto* geom = T::Create(dataStructure, name); + geom->setVertices(vertexArray); + geom->setEdgeList(edgeList); + geom->setFaceList(faceList); + geom->setPolyhedraList(polyArray); + return geom; +} + +/** + * @brief Fill the data store with values, but do not use values greater than the number of tuples. + * @param dataStore + */ +template +void FillDataStore(AbstractDataStore& dataStore) +{ + const auto numTuples = dataStore.getNumberOfTuples(); + const auto numComponents = dataStore.getNumberOfComponents(); + + for(usize i = 0; i < numTuples; i++) + { + const usize offset = i * numComponents; + for(usize j = 0; j < numComponents; j++) + { + usize value = (i + j) % numTuples; + dataStore[offset + j] = static_cast(value); + } + } +} + +template +void CheckDataStore(const AbstractDataStore& dataStore, usize requiredComponents) +{ + const auto numTuples = dataStore.getNumberOfTuples(); + const auto numComponents = dataStore.getNumberOfComponents(); + REQUIRE(numComponents == requiredComponents); + + for(usize i = 0; i < numTuples; i++) + { + const usize offset = i * numComponents; + for(usize j = 0; j < numComponents; j++) + { + usize value = (i + j) % numTuples; + REQUIRE(dataStore[offset + j] == static_cast(value)); + } + } +} + +void CheckGeom0D(const INodeGeometry0D* geom, DataObject::IdType vertexId) +{ + REQUIRE(geom != nullptr); + REQUIRE(geom->getVertexListId() == vertexId); +} + +void CheckGeom1D(const INodeGeometry1D* geom, DataObject::IdType vertexId, DataObject::IdType edgeId) +{ + REQUIRE(geom != nullptr); + REQUIRE(geom->getEdgeListId() == edgeId); + CheckGeom0D(geom, vertexId); +} + +void CheckGeom2D(const INodeGeometry2D* geom, DataObject::IdType vertexId, DataObject::IdType edgeId, DataObject::IdType faceId) +{ + REQUIRE(geom != nullptr); + REQUIRE(geom->getFaceListId() == faceId); + CheckGeom1D(geom, vertexId, edgeId); +} + +void CheckGeom3D(const INodeGeometry3D* geom, DataObject::IdType vertexId, DataObject::IdType edgeId, DataObject::IdType faceId, DataObject::IdType polyhedraId) +{ + REQUIRE(geom != nullptr); + REQUIRE(geom->getPolyhedronListId().has_value()); + REQUIRE(geom->getPolyhedronListId().value() == polyhedraId); + CheckGeom2D(geom, vertexId, edgeId, faceId); +} + +void CheckTestDataStructure(const DataStructure& dataStructure) +{ + DataPath dataGroupPath({Constants::k_DataContainer}); + REQUIRE(dataStructure.getDataAs(dataGroupPath) != nullptr); + + const auto* neighborList = dataStructure.getDataAs(dataGroupPath.createChildPath(Constants::k_NeighborList)); + REQUIRE(neighborList != nullptr); + const auto storePtr = neighborList->getStore(); + REQUIRE(storePtr != nullptr); + REQUIRE(storePtr->getNumberOfTuples() == 6); + + const auto* vertexArray = dataStructure.getDataAs(dataGroupPath.createChildPath(Constants::k_VertexList)); + REQUIRE(vertexArray != nullptr); + const auto& vertices = vertexArray->getDataStoreRef(); + CheckDataStore(vertices, 3); + + const auto* edgeArray = dataStructure.getDataAs(dataGroupPath.createChildPath(Constants::k_Edges)); + REQUIRE(edgeArray != nullptr); + const auto& edges = edgeArray->getDataStoreRef(); + CheckDataStore(edges, 2); + + const auto* faceArray = dataStructure.getDataAs(dataGroupPath.createChildPath(Constants::k_Faces)); + REQUIRE(faceArray != nullptr); + const auto& faces = faceArray->getDataStoreRef(); + CheckDataStore(faces, 3); + + const auto* polyArray = dataStructure.getDataAs(dataGroupPath.createChildPath(Constants::k_Polyhedra)); + REQUIRE(polyArray != nullptr); + const auto& polyhedra = polyArray->getDataStoreRef(); + CheckDataStore(polyhedra, 4); + + const auto* stringArray = dataStructure.getDataAs(dataGroupPath.createChildPath(Constants::k_StringArray)); + REQUIRE(stringArray != nullptr); + auto stringCount = stringArray->getNumberOfTuples(); + REQUIRE(stringCount == 6); + REQUIRE(stringArray->at(0) == "1"); + REQUIRE(stringArray->at(1) == "2"); + REQUIRE(stringArray->at(2) == "3"); + REQUIRE(stringArray->at(3) == "4"); + REQUIRE(stringArray->at(4) == "5"); + REQUIRE(stringArray->at(5) == "6"); + + const auto* vertexGeom = dataStructure.getDataAs(DataPath({Constants::k_VertexGeom})); + CheckGeom0D(vertexGeom, vertexArray->getId()); + + const auto* edgeGeom = dataStructure.getDataAs(DataPath({Constants::k_EdgeGeom})); + CheckGeom1D(edgeGeom, vertexArray->getId(), edgeArray->getId()); + + const auto* quadGeom = dataStructure.getDataAs(DataPath({Constants::k_QuadGeom})); + CheckGeom2D(quadGeom, vertexArray->getId(), edgeArray->getId(), faceArray->getId()); + + const auto* triGeom = dataStructure.getDataAs(DataPath({Constants::k_TriangleGeom})); + CheckGeom2D(triGeom, vertexArray->getId(), edgeArray->getId(), faceArray->getId()); + + const auto* hexGeom = dataStructure.getDataAs(DataPath({Constants::k_HexGeom})); + CheckGeom3D(hexGeom, vertexArray->getId(), edgeArray->getId(), faceArray->getId(), polyArray->getId()); + + const auto* tetraGeom = dataStructure.getDataAs(DataPath({Constants::k_TetrahedralGeom})); + CheckGeom3D(tetraGeom, vertexArray->getId(), edgeArray->getId(), faceArray->getId(), polyArray->getId()); + + DataPath imageGeomPath({Constants::k_ImageGeom}); + const auto* imageGeom = dataStructure.getDataAs(imageGeomPath); + REQUIRE(imageGeom != nullptr); + auto dims = imageGeom->getDimensions(); + REQUIRE(dims[0] == Constants::k_TupleShape[0]); + REQUIRE(dims[1] == Constants::k_TupleShape[1]); + REQUIRE(dims[2] == Constants::k_TupleShape[2]); + const auto* cellData = dataStructure.getDataAs(imageGeomPath.createChildPath(Constants::k_CellData)); + REQUIRE(cellData != nullptr); + REQUIRE(imageGeom->getCellDataId() == cellData->getId()); + REQUIRE(cellData->getShape() == Constants::k_TupleShape); +}; + DataStructure CreateTestDataStructure() { DataStructure dataStructure; @@ -165,6 +368,61 @@ DataStructure CreateTestDataStructure() Result<> arrayCreationResults = ArrayCreationUtilities::CreateArray(dataStructure, tupleShape, std::vector{1}, DataPath({DataNames::k_Group1Name, DataNames::k_AttributeMatrixName, DataNames::k_Array2Name}), IDataAction::Mode::Execute, ArrayCreationUtilities::k_DefaultDataFormat, "1"); + + // Create Arrays and DataGroup + auto* dataGroup = DataGroup::Create(dataStructure, Constants::k_DataContainer); + + auto listStorePtr = std::make_shared>(Constants::k_TupleShape); + listStorePtr->setList(0, std::vector{1,2}); + listStorePtr->setList(1, std::vector{1,2}); + listStorePtr->setList(2, std::vector{1,2}); + listStorePtr->setList(3, std::vector{1,2}); + listStorePtr->setList(4, std::vector{1,2}); + listStorePtr->setList(5, std::vector{1,2}); + auto* neighborList = Int16NeighborList::Create(dataStructure, Constants::k_NeighborList, listStorePtr, dataGroup->getId()); + + auto vertices = std::make_shared(Constants::k_TupleShape, ShapeType{3}, 0.0f); + auto* vertexArray = Float32Array::Create(dataStructure, Constants::k_VertexList, vertices, dataGroup->getId()); + FillDataStore(*vertices.get()); + + auto edges = std::make_shared(Constants::k_TupleShape, ShapeType{2}, 0); + auto* edgesArray = IGeometry::SharedEdgeList::Create(dataStructure, Constants::k_Edges, edges, dataGroup->getId()); + FillDataStore(*edges.get()); + + auto faces = std::make_shared(Constants::k_TupleShape, ShapeType{3}, 0); + auto* facesArray = IGeometry::SharedTriList::Create(dataStructure, Constants::k_Faces, faces, dataGroup->getId()); + FillDataStore(*faces.get()); + + auto polyhedra = std::make_shared(Constants::k_TupleShape, ShapeType{4}, 0); + auto* polyhedraArray = IGeometry::SharedTriList::Create(dataStructure, Constants::k_Polyhedra, polyhedra, dataGroup->getId()); + FillDataStore(*polyhedra.get()); + + StringArray::collection_type strings = {"1", "2", "3", "4", "5", "6"}; + auto* stringArray = StringArray::CreateWithValues(dataStructure, Constants::k_StringArray, Constants::k_TupleShape, strings, dataGroup->getId()); + + // Create Geometries and make sure special arrays are set. + auto* imageGeom = ImageGeom::Create(dataStructure, Constants::k_ImageGeom); + auto* cellMatrix = AttributeMatrix::Create(dataStructure, Constants::k_CellData, Constants::k_TupleShape, imageGeom->getId()); + imageGeom->setCellData(cellMatrix->getId()); + imageGeom->setDimensions(Constants::k_TupleShape); + + // 0D Geometry + auto* vertexGeom = VertexGeom::Create(dataStructure, Constants::k_VertexGeom); + vertexGeom->setVertices(*vertexArray); + + // 1D Geometry + auto* edgeGeom = EdgeGeom::Create(dataStructure, Constants::k_EdgeGeom); + edgeGeom->setVertices(*vertexArray); + edgeGeom->setEdgeList(*edgesArray); + + // 2D Geometries + auto* quadGeom = Create2DGeom(dataStructure, Constants::k_QuadGeom, *vertexArray, *edgesArray, *facesArray); + auto* triangleGeom = Create2DGeom(dataStructure, Constants::k_TriangleGeom, *vertexArray, *edgesArray, *facesArray); + + // 3D Geometries + auto* hexGeom = Create3DGeom(dataStructure, Constants::k_HexGeom, *vertexArray, *edgesArray, *facesArray, *polyhedraArray); + auto* tetrahedralGeom = Create3DGeom(dataStructure, Constants::k_TetrahedralGeom, *vertexArray, *edgesArray, *facesArray, *polyhedraArray); + return dataStructure; } @@ -449,11 +707,93 @@ GeometryTestCase MakeGeometryTestCase(std::string typeName, std::function lock(m_DataMutex); + + DataStructure dataStructure = CreateTestDataStructure(); + Arguments args; + WriteDREAM3DFilter filter; + + args.insertOrAssign(WriteDREAM3DFilter::k_ExportFilePath, std::make_any(GetIODataPath())); + args.insertOrAssign(WriteDREAM3DFilter::k_WriteXdmf, std::make_any(false)); + args.insertOrAssign(WriteDREAM3DFilter::k_UseCompression, std::make_any(false)); + args.insertOrAssign(WriteDREAM3DFilter::k_CompressionLevel, std::make_any(1)); + + SECTION("Empty FilePath") + { + args.insertOrAssign(WriteDREAM3DFilter::k_ExportFilePath, std::make_any(std::filesystem::path())); + + // Preflight the filter and check result + auto preflightResult = filter.preflight(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_INVALID(preflightResult.outputActions); + } + + SECTION("Bad Compression Level") + { + args.insertOrAssign(WriteDREAM3DFilter::k_UseCompression, std::make_any(true)); + args.insertOrAssign(WriteDREAM3DFilter::k_CompressionLevel, std::make_any(0)); + + // Preflight the filter and check result + auto preflightResult = filter.preflight(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_INVALID(preflightResult.outputActions); + } +} + +TEST_CASE("WriteDREAM3DFilter:Valid Parameters") +{ + UnitTest::LoadPlugins(); + std::lock_guard lock(m_DataMutex); + + DataStructure dataStructure = CreateTestDataStructure(); + Arguments args; + WriteDREAM3DFilter filter; + + args.insertOrAssign(WriteDREAM3DFilter::k_ExportFilePath, std::make_any(GetIODataPath())); + args.insertOrAssign(WriteDREAM3DFilter::k_WriteXdmf, std::make_any(false)); + args.insertOrAssign(WriteDREAM3DFilter::k_UseCompression, std::make_any(false)); + args.insertOrAssign(WriteDREAM3DFilter::k_CompressionLevel, std::make_any(1)); + + // Preflight the filter and check result + auto preflightResult = filter.preflight(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); + + auto result = filter.execute(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(result.result); +} + +TEST_CASE("WriteDREAM3D:Pipeline / WriteXdmf combinations") +{ + UnitTest::LoadPlugins(); + std::lock_guard lock(m_DataMutex); + + bool writeXdmf = GENERATE(true, false); + Pipeline exportPipeline = GENERATE(CreateExportPipeline(), Pipeline()); + + auto writeResult = DREAM3D::WriteFile(GetIODataPath(), CreateTestDataStructure(), exportPipeline, writeXdmf); + SIMPLNX_RESULT_REQUIRE_VALID(writeResult); +} + +TEST_CASE("WriteDREAM3D:Invalid File") +{ + UnitTest::LoadPlugins(); + std::lock_guard lock(m_DataMutex); + + bool writeXdmf = GENERATE(true, false); + Pipeline exportPipeline = GENERATE(CreateExportPipeline(), Pipeline()); + + auto writeResult = DREAM3D::WriteFile(fs::path(), CreateTestDataStructure(), exportPipeline, writeXdmf); + SIMPLNX_RESULT_REQUIRE_INVALID(writeResult); +} + TEST_CASE("DREAM3DFileTest:DREAM3D File IO Test", "[WriteDREAM3DFilter]") { UnitTest::LoadPlugins(); std::lock_guard lock(m_DataMutex); + + bool writeXdmf = GENERATE(true, false); // Write .dream3d file { auto fileData = CreateFileData(); @@ -471,6 +811,8 @@ TEST_CASE("DREAM3DFileTest:DREAM3D File IO Test", "[WriteDREAM3DFilter]") auto [pipeline, dataStructure] = fileResult.value(); + CheckTestDataStructure(dataStructure); + // Test reading the DataStructure REQUIRE(dataStructure.getData(DataPath({DataNames::k_Group1Name})) != nullptr); REQUIRE(dataStructure.getData(DataPath({DataNames::k_Group1Name, DataNames::k_Group2Name})) != nullptr); diff --git a/src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md b/src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md new file mode 100644 index 0000000000..c4e2c9ca08 --- /dev/null +++ b/src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md @@ -0,0 +1,121 @@ +# V&V Report: WriteDREAM3DFilter + +| | | +|-----------------------------|-----------------------------------------------------------------------------| +| Plugin | SimplnxCore | +| SIMPLNX UUID | `b3a95784-2ced-41ec-8d3d-0242ac130003` | +| SIMPLNX Human Name | Write DREAM3D-NX File | +| DREAM3D 6.5.171 equivalent | `DataContainerWriter` — SIMPL UUID `3fcd4c43-9d75-5b86-aad4-4441bc914f37` | +| Verified commit | ** | +| Status | **READY FOR REVIEW** (second-engineer review outstanding — see V&V phase) | +| Sign-off | *pending second-engineer review* | + +## At a glance + +| Aspect | Current state | +|------------------------|---------------| +| Algorithm Relationship | **Rewrite** — same UUID/role as legacy `DataContainerWriter`, but the on-disk format is entirely new (v8 `DataStructure` HDF5 layout + `AtomicFile` atomic-write + optional gzip compression), not a translation of the legacy writer's code. | +| Oracle (confirmed) | **Class 1 (Analytical)** — expected content is the hand-built in-memory `DataStructure`/`Pipeline` the test itself constructed; expected HDF5 physical layout (contiguous vs. chunked+deflate) is a closed-form function of array byte-size and the two compression parameters. 17 fixtures across `DREAM3DFileTest.cpp`, all pass. | +| Code paths enumerated | **14 of 19** exercised; 5 gaps are defensive/unreachable-via-public-API guards (see table). | +| Tests today | **17 TEST_CASEs** (some with `GENERATE`/`DYNAMIC_SECTION` multiplying cases) — preflight validation, full round-trip content fidelity across every geometry/DataObject type, SIMPL args backward-compat, and a 6-test compression sub-suite (layout, bypass threshold, level monotonicity). | +| Exemplar archive | **None.** Every test builds its `DataStructure` inline in C++ and round-trips it through `WriteFile`/`ReadFile` in the same run — no cached `.tar.gz` golden file is used or needed for a Class 1 oracle. | +| Legacy comparison | **Not run** — no like-for-like binary comparison is possible. `DataContainerWriter` (6.5.171) emits the legacy `DataContainers` HDF5 layout with no compression option; SIMPLNX always emits the current v8 `DataStructure` layout regardless of source pipeline. The format change is deliberate (Rewrite), so fidelity is verified independently via round-trip Class 1 tests instead of a legacy diff. | +| Bug flags | None. | +| V&V phase | Oracle chosen, code paths enumerated, test inventory reviewed, deviations documented. Outstanding: second-engineer review of the oracle design and of the 5 uncovered defensive paths. | + +## Summary + +`WriteDREAM3DFilter` serializes the current `DataStructure` (and, when run inside a pipeline, the preceding `Pipeline`) to an HDF5 `.dream3d` file, with an optional companion `.xdmf` sidecar and optional gzip compression of array datasets. It replaces legacy SIMPL's `DataContainerWriter` under the same conceptual role but with an intentionally new v8 file format, so verification is independent of 6.5.171: correctness is established by writing hand-built `DataStructure`s covering every geometry and `DataObject` type, then reading them back and asserting exact structural/content equality (Class 1 Analytical), plus closed-form assertions on the resulting HDF5 physical layout under each compression setting. All 17 test cases pass; no bugs were found. `StatsDataArray`/`StructArray` (SIMPL's per-ensemble statistics types) are out of scope for this cycle — those `DataObject` types do not yet exist in this branch of simplnx (see deviation D2). + +## Algorithm Relationship + +*Classification:* **Rewrite** ~~| Port | Minor changes | New filter~~ + +`WriteDREAM3DFilter` keeps the SIMPL UUID mapping (`3fcd4c43-9d75-5b86-aad4-4441bc914f37` → `WriteDREAM3DFilter`, `SimplnxCoreLegacyUUIDMapping.hpp:170`) and the legacy `DataContainerWriter` role, but the algorithm (`Algorithms/WriteDREAM3D.cpp`, 82 lines) was designed from the start for the current v8 `DataStructure` HDF5 layout, `AtomicFile`-based atomic writes, and (as of PR #1606) optional gzip compression — none of which exist in the legacy 6.5.171 writer. This has never been a line-by-line port of the legacy C++; the file format itself is a clean-sheet design (`k_CurrentFileVersion = "8.0"` vs. legacy's `"7.0"`/`DataContainers` group tag, see `Dream3dIO.cpp:31-42`). + +*Evidence:* `parametersVersion()` is at 2 (compression parameters added after the filter's initial release); `git log --follow` on the algorithm/filter files shows the write path has been restructured multiple times since inception (out-of-core support #1253, atomic-file rework #900, algorithm-class extraction #1544) without ever tracking legacy DataContainerWriter's implementation. + +*Port-time deltas (SIMPL → SIMPLNX argument mapping, `WriteDREAM3DFilter::FromSIMPLJson`):* + +1. `OutputFile` → `export_file_path`, `WriteXdmfFile` → `write_xdmf_file`: direct 1:1 mapping, no behavior change. +2. SIMPL's "Write Time Series" parameter has no SIMPLNX equivalent (dropped) — legacy time-series writing is not part of this filter's scope in NX; not a regression since no NX pipeline concept maps to it. +3. `use_compression` is force-overridden to `false` for any pipeline converted from SIMPL JSON (`WriteDREAM3DFilter.cpp:130`), even though SIMPLNX's own default is `true`. This is deliberate: SIMPL v6 pipelines never wrote compressed files, so a converted pipeline preserves the exact on-disk encoding it shipped with rather than silently changing file size/behavior on re-run. + +*Material PRs since baseline:* #1606 (added HDF5 compression parameters/behavior), #1544 (moved `executeImpl` logic into the `WriteDREAM3D` algorithm class, no behavior change), #1253 (out-of-core support). + +## Oracle + +*Class:* **1 (Analytical)**, primary. `Compression_LevelsRoundTrip` also carries a **Class 4 (Invariant)** companion check (file size must be non-increasing as gzip level rises) alongside its per-level content round-trip. + +*Applied:* Every test constructs its expected answer directly, without ever running the filter to "produce" the expected value: + +- **Content fidelity:** each test builds a `DataStructure` in C++ (`CreateTestDataStructure()`, or an inline array/geometry), writes it, reads it back, and asserts the read-back content equals what was built — by construction, not by comparison to a previously-captured file. `CheckTestDataStructure()` walks every `DataObject` kind the filter must support (nested `DataGroup`s, `AttributeMatrix`, `NeighborList`, `StringArray`, and all seven geometry types: Vertex/Edge/Triangle/Quad/Tetrahedral/Hexahedral/Image) and asserts exact values against the hand-known fill pattern from `FillDataStore()`. +- **HDF5 physical layout:** the filter's documented compression policy (`docs/WriteDREAM3DFilter.md`) states arrays under 16 KiB always stay contiguous/uncompressed regardless of settings, and any larger array is chunked+deflated at the requested level when compression is enabled. This is a closed-form predicate on `(array byte size, UseCompression, CompressionLevel)` — tests assert it directly via `UnitTest::ProbeHdf5Dataset` rather than trusting the filter's own claim about what it wrote. +- **Pipeline embedding:** when run inside an actual `Pipeline`, the written file's embedded pipeline JSON must reproduce the exact filter sequence and count that was executed — asserted against the hand-built pipeline (`pipeline.size() == 3`, filter names checked by index). + +**Encoded tests:** `src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp` (17 `TEST_CASE`s touching Write, several parameterized via `GENERATE`/`DYNAMIC_SECTION`) — all pass. See Test inventory below for the full list. + +*Second-engineer review:* Outstanding. Recommended focus: the Class 1 boundary-of-scope claim in deviation D2 (StatsDataArray/StructArray) and the 5 uncovered defensive paths (Code path coverage below). + +## Code path coverage + +**14 of 19** paths exercised. The 5 gaps are all defensive guards that require conditions unreachable through the public filter/pipeline API (invalid destination mid-write after preflight already validated it, or a detached `PipelineFilter`) rather than genuine untested behavior. + +Source: `src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteDREAM3D.cpp` (82 lines). Preflight guards below live in the sibling `Filters/WriteDREAM3DFilter.cpp` (`preflightImpl`), which the policy still treats as in-scope algorithm surface (parameter validation gates that the Algorithm class depends on). + +| # | Phase | Path | Test case | +|----|--------------------|----------------------------------------------------------------------------------------------|-----------| +| 1 | Preflight | `export_file_path` empty → error `-1` | `"WriteDREAM3DFilter:Invalid Parameters"` § `Empty FilePath` | +| 2 | Preflight | `use_compression=true`, `compression_level < 1` → error `-2` | `"WriteDREAM3DFilter:Invalid Parameters"` § `Bad Compression Level`; `"WriteDREAM3DFilter: Compression_Preflight_RejectsOutOfRangeLevel"` (level=0) | +| 3 | Preflight | `use_compression=true`, `compression_level > 9` → error `-2` | `"WriteDREAM3DFilter: Compression_Preflight_RejectsOutOfRangeLevel"` (level=10) | +| 4 | Preflight | `use_compression=false` → `compression_level` ignored even if out of `[1,9]` | `"WriteDREAM3DFilter: Compression_Preflight_RejectsOutOfRangeLevel"` (level=0, compression off) | +| 5 | Preflight | Valid parameters → success | `"WriteDREAM3DFilter:Valid Parameters"`; implicitly, every passing execute-path test below | +| 6 | Execute — setup | `AtomicFile::Create` fails (unwritable/invalid destination directory) | *Not directly tested.* Preflight only rejects an empty path; a directory-permission failure at execute time would require a filesystem fixture (e.g., a read-only directory) not set up by the current suite. | +| 7 | Execute — setup | `AtomicFile::Create` succeeds | Every passing execute-path test | +| 8 | Execute — pipeline | `PipelineNode != nullptr` and `getPrecedingPipeline()` returns `nullptr` → error `-15` | *Not directly tested.* Only reachable if a `PipelineFilter` is detached from its parent `Pipeline`, which normal `filter.execute()`/`pipeline.execute()` usage never produces. | +| 9 | Execute — pipeline | `PipelineNode != nullptr`, preceding pipeline retrieved successfully → embedded in file | `"DREAM3DFileTest:Import/Export DREAM3D Filter Test"` (`exportPipeline.execute()`); `"DREAM3DFileTest:Import/Export Multi-DREAM3D Filter Test"` (`CreateMultiExportFiles()`) | +| 10 | Execute — pipeline | `PipelineNode == nullptr` → empty pipeline written | `"WriteDREAM3DFilter:Valid Parameters"`, `"DREAM3DFileTest::StringArray"`, all `Compression_*` tests (all call `filter.execute(ds, args)` directly) | +| 11 | Execute — options | `use_compression=true` → `writeOptions.compressionLevel = CompressionLevel` | `"...Compression_On_IsChunkedAndDeflated"`, `"...Compression_SmallArray_Bypasses"`, `"...Compression_LevelsRoundTrip"` | +| 12 | Execute — options | `use_compression=false` → `writeOptions.compressionLevel = 0` | `"...Compression_Off_IsContiguous"`; `"WriteDREAM3DFilter:Valid Parameters"` | +| 13 | Execute — write | `DREAM3D::WriteFile(...)` returns invalid → skip commit, return the error | *Not directly tested* through the full filter/`AtomicFile` path. Once `AtomicFile::Create` has validated the destination, the underlying HDF5 write essentially cannot fail independently — same root gap as Path 6. | +| 14 | Execute — write | `DREAM3D::WriteFile(...)` returns valid → proceed to commit | Every passing execute-path test | +| 15 | Execute — commit | `atomicFile.commit()` fails (rename onto final destination fails) | *Not directly tested.* Would require the destination path to become invalid between `AtomicFile::Create` and `commit()` (e.g., concurrent deletion of the parent directory) — a race not exercised by the suite. | +| 16 | Execute — commit | `atomicFile.commit()` succeeds | Every passing execute-path test (the output file is present and re-readable in every round-trip test) | +| 17 | Execute — xdmf | `write_xdmf_file=true` → rename temp `.xdmf` into place, succeeds | `"DREAM3DFileTest:DREAM3D File IO Test"` (writeXdmf=true), `CreateExportPipeline()`/`CreateMultiExportFiles()` (`write_xdmf_file=true`) | +| 18 | Execute — xdmf | `write_xdmf_file=true`, rename fails → `MakeErrorResult` with system error message | *Not directly tested.* Would require the `.xdmf` destination to become unwritable between the HDF5 write succeeding and the rename — not portably reproducible in the current suite. | +| 19 | Execute — xdmf | `write_xdmf_file=false` → skip rename, return `WriteFile`'s result directly | Most `Compression_*` tests, `"DREAM3DFileTest::StringArray"`, `"WriteDREAM3DFilter:Valid Parameters"` | + +## Test inventory + +| Test case | Status | Notes | +|-----------|--------|-------| +| `WriteDREAM3DFilter:Invalid Parameters` (§ Empty FilePath, § Bad Compression Level) | kept | Preflight-only, inline `DataStructure`. Covers Paths 1, 2. | +| `WriteDREAM3DFilter:Valid Parameters` | kept | Preflight + full execute, inline `DataStructure`, compression off. Covers Paths 5, 7, 10, 12, 14, 16, 19. | +| `WriteDREAM3D:Pipeline / WriteXdmf combinations` | kept | Calls `DREAM3D::WriteFile(path, ds, pipeline, writeXdmf)` directly (bypasses `AtomicFile`/the Algorithm class) across `writeXdmf × {real pipeline, empty pipeline}` (4 cases). Exercises the shared write utility the Algorithm depends on, not the Algorithm's own guards. | +| `WriteDREAM3D:Invalid File` | kept | Same free-function call with an empty path (4 cases via `GENERATE`); asserts the underlying `HDF5::FileIO::WriteFile` failure is surfaced, not `AtomicFile`'s. | +| `DREAM3DFileTest:DREAM3D File IO Test` | kept | The primary Class 1 content-fidelity test. Builds every geometry/`DataObject` type (`CreateTestDataStructure`), writes + reads back (`writeXdmf ∈ {true,false}`), and asserts full structural/content equality (`CheckTestDataStructure`) plus pipeline round-trip (`pipeline.size()==3`, filter names by index). | +| `DREAM3DFileTest::StringArray` | kept | Round-trips a `StringArray` through the actual `WriteDREAM3DFilter`/`ReadDREAM3DFilter` classes (not the free function) — covers Path 10 with real filter execution. | +| `DREAM3DFileTest:Import/Export DREAM3D Filter Test` | kept | Executes `WriteDREAM3DFilter` inside a real `Pipeline` (`exportPipeline.execute()`) — the only test exercising Path 9 (non-null `PipelineNode` with a real preceding pipeline). Also checks preflight-imported vs. executed array store types on the read side. | +| `DREAM3DFileTest:Import/Export Multi-DREAM3D Filter Test` | kept | Two independent export pipelines (`CreateMultiExportFiles`) each executing `WriteDREAM3DFilter` in-pipeline, then a single import pipeline consuming both files. Second confirmation of Path 9. | +| `DREAM3DFileTest: Preflight imports geometry connectivity as metadata-only stores` | kept | Read-side only (consumes a pre-existing `geoms.dream3d` asset); does not exercise `WriteDREAM3DFilter`. Listed for completeness since it shares the tag set. | +| `SimplnxCore::WriteDREAM3DFilter: SIMPL Backwards Compatibility` | kept | `DYNAMIC_SECTION` over SIMPL 6.5 (UUID-keyed) and 6.4 (name-keyed) `DataContainerWriter` fixtures. Asserts UUID resolution, empty comments, and `export_file_path`/`write_xdmf_file` value conversion. Not an oracle test — argument-mapping check only. | +| `DREAM3DFileTest: DataArray datasets are chunked+deflated when WriteOptions requests it` | kept | Calls the `WriteOptions`-aware free function directly with a 2 MB array; asserts chunked+deflate layout at level 5 via `ProbeHdf5Dataset`, then round-trips content. Covers the compression-options contract the Algorithm class relies on. | +| `WriteDREAM3DFilter: Compression_Off_IsContiguous` | kept | Full filter execute, `use_compression=false`. Asserts contiguous/no-deflate layout. Covers Path 12. | +| `WriteDREAM3DFilter: Compression_On_IsChunkedAndDeflated` | kept | Full filter execute, `use_compression=true`, level 5, 500K-element array. Asserts chunked+deflate layout and round-trips content. Covers Path 11. | +| `WriteDREAM3DFilter: Compression_SmallArray_Bypasses` | kept | Full filter execute with one <16 KiB and one >16 KiB array in the same `DataStructure`, `use_compression=true`. Asserts the small array stays contiguous/uncompressed while the large one is chunked+deflated — the Class 1 closed-form threshold check. | +| `WriteDREAM3DFilter: Compression_LevelsRoundTrip` | kept | Full filter execute at levels {1,5,9} on the same 1M-element pattern. Round-trips content at each level and asserts non-increasing file size as level rises (Class 4 companion invariant). | +| `WriteDREAM3DFilter: Compression_Preflight_RejectsOutOfRangeLevel` | kept | Three sequential preflight-only checks: level=0 (invalid), level=10 (invalid), level=0 with compression off (valid — ignored). Covers Paths 2, 3, 4. | +| `DREAM3DFileTest: PreflightCache avoids re-reading unchanged files` | kept | Uses `DREAM3D::WriteFile` only to create read-side fixture files; does not exercise `WriteDREAM3DFilter`'s own behavior. Listed for completeness since it shares source-file/tag space. | + +## Exemplar archive + +None. Every test above builds its input `DataStructure` inline in C++ and never loads a downloaded `.tar.gz` exemplar — the Class 1 oracle's "expected output" is the test's own hand-built input, so no cached golden file is required or used. (`Small_IN100_dream3d_v3.tar.gz`, referenced elsewhere in this test file, backs unrelated `ReadDREAM3DFilter`-only test cases and is not consumed by any Write-side test.) + +## Deviations from DREAM3D 6.5.171 + +Comparison run: file-format inspection only (no data-value diff is meaningful — see below). Two deviations documented: + +- `WriteDREAM3DFilter-D1` — on-disk file format is a deliberate, complete rewrite (v8 `DataStructure` layout vs. legacy v7 `DataContainers` layout; optional gzip compression; atomic write) — see `vv/deviations/WriteDREAM3DFilter.md`. +- `WriteDREAM3DFilter-D2` — `StatsDataArray`/`StructArray` (SIMPL ensemble-statistics types) cannot currently be written because those `DataObject` types do not yet exist in this branch of simplnx — see `vv/deviations/WriteDREAM3DFilter.md`. + +Neither is a bug: D1 is the intended outcome of the Rewrite classification (defended above), and D2 is a scope boundary pending a separate in-progress port of those data types. diff --git a/src/Plugins/SimplnxCore/vv/deviations/WriteDREAM3DFilter.md b/src/Plugins/SimplnxCore/vv/deviations/WriteDREAM3DFilter.md new file mode 100644 index 0000000000..6a9b4d14d6 --- /dev/null +++ b/src/Plugins/SimplnxCore/vv/deviations/WriteDREAM3DFilter.md @@ -0,0 +1,41 @@ +# Deviations from DREAM3D 6.5.171: WriteDREAM3DFilter + +This file lists every documented behavioral difference between this SIMPLNX filter and its DREAM3D 6.5.171 equivalent (`DataContainerWriter`). + +Entries are referenced by stable ID (`WriteDREAM3DFilter-D`) from the V&V report and from public migration guidance. The ID is stable across renames; the Filter UUID field is the permanent cross-reference anchor. + +--- + +## WriteDREAM3DFilter-D1 + +| Field | Value | +|---|---| +| **Deviation ID** | `WriteDREAM3DFilter-D1` | +| **Filter UUID** | `b3a95784-2ced-41ec-8d3d-0242ac130003` | +| **Status** | active | + +**Symptom:** A `.dream3d` file written by SIMPLNX is not byte-compatible with, and cannot be opened by, a DREAM3D 6.5.171 install expecting the legacy layout — and the reverse is only possible through SIMPLNX's dedicated legacy-import code path (`ReadDREAM3DFilter`'s legacy `DataContainers` reader), not by treating the file as interchangeable. + +**Root cause:** Algorithmic choice. SIMPLNX writes a clean-sheet v8 HDF5 layout: a top-level `DataStructure` group (vs. legacy's `DataContainers` group), a `FileVersion` attribute of `"8.0"` (vs. legacy `"7.0"`), an embedded JSON pipeline representation (vs. legacy's own pipeline serialization), atomic-rename write semantics (`AtomicFile`, so a crash mid-write cannot leave a corrupt file at the destination path), and an optional gzip/deflate compression scheme for `DataArray`/`NeighborList` datasets that has no legacy equivalent. This has been true since the filter's introduction — it was never a translation of legacy `DataContainerWriter`'s C++ (see Algorithm Relationship in the V&V report). + +**Affected users:** Anyone attempting to open a SIMPLNX-written `.dream3d` file directly in a DREAM3D 6.5.171 install, or vice versa, outside of DREAM3D-NX's own dual-format `ReadDREAM3DFilter`. Users staying entirely within DREAM3D-NX (write with `WriteDREAM3DFilter`, read with `ReadDREAM3DFilter`) never observe this — round-trip fidelity within the new format is verified by the Class 1 tests in the V&V report. + +**Recommendation:** Trust SIMPLNX. The new format is a deliberate design improvement (atomicity, compression, JSON pipeline embedding) required for capabilities legacy DREAM3D never had (out-of-core datasets, versioned pipeline metadata). It is not "wrong" relative to 6.5.171; it is a different, and newer, on-disk contract. DREAM3D-NX remains able to *read* legacy 6.5.171 files via `ReadDREAM3DFilter`'s legacy import path, so migration is one-directional by design (import old, export new) rather than bidirectional. + +--- + +## WriteDREAM3DFilter-D2 + +| Field | Value | +|---|---| +| **Deviation ID** | `WriteDREAM3DFilter-D2` | +| **Filter UUID** | `b3a95784-2ced-41ec-8d3d-0242ac130003` | +| **Status** | active | + +**Symptom:** A pipeline that would have produced `StatsDataArray` or `StructArray` objects in legacy DREAM3D (e.g., ensemble statistics from a "Generate Ensemble Statistics"-style filter) cannot have those objects written to a `.dream3d` file by `WriteDREAM3DFilter` in the current develop branch — there is nothing in the `DataStructure` for the writer to serialize, because the types themselves do not exist yet. + +**Root cause:** Library (incomplete port, out of scope for this V&V cycle). `StatsDataArray` and `StructArray` are DREAM3D 6.5.171 `DataObject` types whose simplnx equivalents are being implemented on a separate, not-yet-merged branch. `WriteDREAM3DFilter`'s own write path (`Algorithms/WriteDREAM3D.cpp`) has no special-case logic to reject or special-case these types — the gap is entirely upstream, in `DataStructure`/`HDF5::DataStructureWriter` not yet having a type to construct. This V&V pass covers everything `WriteDREAM3DFilter` can currently write; it makes no claim about statistics data because that data cannot currently exist in a simplnx `DataStructure`. + +**Affected users:** Anyone migrating a legacy pipeline that computes per-ensemble statistics. Their exported SIMPLNX `.dream3d` file will simply lack the statistics group entirely (there being no such object to write) until the separate `StatsDataArray`/`StructArray` port lands and this filter is re-verified against it. + +**Recommendation:** Trust 6.5.171 for statistics data until the pending port lands. This is a temporary feature gap, not a correctness defect in `WriteDREAM3DFilter` itself — re-run this V&V cycle's oracle tests once `StatsDataArray`/`StructArray` exist in this branch to confirm the writer handles them correctly. From bc48c0d7fb2c0fb4945f7b9ade2532d4ac39b44d Mon Sep 17 00:00:00 2001 From: Matthew Marine Date: Thu, 23 Jul 2026 08:51:20 -0400 Subject: [PATCH 2/9] Clang format --- src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp b/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp index a764ff1f1a..c811d4f206 100644 --- a/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp +++ b/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp @@ -373,12 +373,12 @@ DataStructure CreateTestDataStructure() auto* dataGroup = DataGroup::Create(dataStructure, Constants::k_DataContainer); auto listStorePtr = std::make_shared>(Constants::k_TupleShape); - listStorePtr->setList(0, std::vector{1,2}); - listStorePtr->setList(1, std::vector{1,2}); - listStorePtr->setList(2, std::vector{1,2}); - listStorePtr->setList(3, std::vector{1,2}); - listStorePtr->setList(4, std::vector{1,2}); - listStorePtr->setList(5, std::vector{1,2}); + listStorePtr->setList(0, std::vector{1, 2}); + listStorePtr->setList(1, std::vector{1, 2}); + listStorePtr->setList(2, std::vector{1, 2}); + listStorePtr->setList(3, std::vector{1, 2}); + listStorePtr->setList(4, std::vector{1, 2}); + listStorePtr->setList(5, std::vector{1, 2}); auto* neighborList = Int16NeighborList::Create(dataStructure, Constants::k_NeighborList, listStorePtr, dataGroup->getId()); auto vertices = std::make_shared(Constants::k_TupleShape, ShapeType{3}, 0.0f); From b9eb4d4eca00ef057cc2c7c1b10c48d5d22facad Mon Sep 17 00:00:00 2001 From: Matthew Marine Date: Fri, 24 Jul 2026 09:44:42 -0400 Subject: [PATCH 3/9] PR requested changes * Fixed DREAM3DIO checking the wrong result. * Added ScalarData and RectGridGeom testing to DREAM3DFileTest. * Improved NeighborList testing * Updated geometries to use arrays of the appropriate tuple components. * Check that the xdmf file exists. --- .../SimplnxCore/test/DREAM3DFileTest.cpp | 266 +++++++++++++++--- .../Utilities/Parsing/DREAM3D/Dream3dIO.cpp | 2 +- 2 files changed, 228 insertions(+), 40 deletions(-) diff --git a/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp b/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp index c811d4f206..f641e09af8 100644 --- a/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp +++ b/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp @@ -23,6 +23,7 @@ #include "simplnx/DataStructure/IDataStore.hpp" #include "simplnx/DataStructure/IO/HDF5/DataStructureReader.hpp" #include "simplnx/DataStructure/ListStore.hpp" +#include "simplnx/DataStructure/ScalarData.hpp" #include "simplnx/Filter/Arguments.hpp" #include "simplnx/Filter/FilterHandle.hpp" #include "simplnx/Parameters/Dream3dImportParameter.hpp" @@ -65,11 +66,15 @@ constexpr StringLiteral k_CellData = "Cell Data"; constexpr StringLiteral k_DataContainer = "Data Container"; constexpr StringLiteral k_EdgeGeom = "EdgeGeom"; constexpr StringLiteral k_ImageGeom = "ImageGeom"; +constexpr StringLiteral k_RectGridGeom = "RectGrid"; constexpr StringLiteral k_HexGeom = "HexahedralGeom"; constexpr StringLiteral k_QuadGeom = "QuadGeom"; constexpr StringLiteral k_TetrahedralGeom = "TetrahedralGeom"; constexpr StringLiteral k_TriangleGeom = "TriangleGeom"; constexpr StringLiteral k_VertexGeom = "VertexGeom"; +constexpr StringLiteral k_XBounds = "X Bounds"; +constexpr StringLiteral k_YBounds = "Y Bounds"; +constexpr StringLiteral k_ZBounds = "Z Bounds"; constexpr StringLiteral k_DynamicListArray = "DynamicList"; constexpr StringLiteral k_NeighborList = "NeighborList"; @@ -77,8 +82,23 @@ constexpr StringLiteral k_StringArray = "String Array"; constexpr StringLiteral k_VertexList = "Vertices"; constexpr StringLiteral k_Edges = "Edges"; constexpr StringLiteral k_Faces = "Faces"; -constexpr StringLiteral k_Polyhedra = "Polyhedrals"; +constexpr StringLiteral k_Polyhedra = "Polyhedra"; +constexpr StringLiteral k_HexaArray = "Hexahedra Array"; +constexpr StringLiteral k_TetraArray = "Tetrahedra Array"; +constexpr StringLiteral k_QuadArray = "Quad Array"; +constexpr StringLiteral k_Int8 = "Int 8"; +constexpr StringLiteral k_Int16 = "Int 16"; +constexpr StringLiteral k_Int32 = "Int 32"; +constexpr StringLiteral k_Int64 = "Int 64"; +constexpr StringLiteral k_UInt8 = "UInt 8"; +constexpr StringLiteral k_UInt16 = "UInt 16"; +constexpr StringLiteral k_UInt32 = "UInt 32"; +constexpr StringLiteral k_UInt64 = "UInt 64"; +constexpr StringLiteral k_Float32 = "Float 32"; +constexpr StringLiteral k_Float64 = "Float 64"; +constexpr int64 k_ScalarValue = 3; const ShapeType k_TupleShape{3, 2, 1}; +const SizeVec3 k_ImageShape{1, 2, 3}; constexpr int16 k_ListCount = 6; } // namespace Constants @@ -118,6 +138,13 @@ fs::path GetIODataPath() return GetDataDir(*app) / Constants::k_Dream3dFilename; } +fs::path GetXdmfPath() +{ + fs::path filePath = GetIODataPath(); + filePath.replace_extension(".xdmf"); + return filePath; +} + fs::path GetExportDataPath() { auto app = Application::Instance(); @@ -282,6 +309,26 @@ void CheckGeom3D(const INodeGeometry3D* geom, DataObject::IdType vertexId, DataO CheckGeom2D(geom, vertexId, edgeId, faceId); } +template +void CheckScalarData(const DataStructure& dataStructure, const DataPath& path) +{ + auto* scalarData = dataStructure.getDataAs>(path); + REQUIRE(scalarData != nullptr); + REQUIRE(scalarData->getValue() == Approx(static_cast(Constants::k_ScalarValue))); +} + +void CheckNeighborListStore(const AbstractListStore& store) +{ + REQUIRE(store.getNumberOfTuples() == Constants::k_ListCount); + for(usize i = 0; i < Constants::k_ListCount; i++) + { + auto list = store.getList(i); + REQUIRE(list.size() == 2); + REQUIRE(list[0] == 1); + REQUIRE(list[1] == 2); + } +} + void CheckTestDataStructure(const DataStructure& dataStructure) { DataPath dataGroupPath({Constants::k_DataContainer}); @@ -291,7 +338,7 @@ void CheckTestDataStructure(const DataStructure& dataStructure) REQUIRE(neighborList != nullptr); const auto storePtr = neighborList->getStore(); REQUIRE(storePtr != nullptr); - REQUIRE(storePtr->getNumberOfTuples() == 6); + CheckNeighborListStore(*storePtr); const auto* vertexArray = dataStructure.getDataAs(dataGroupPath.createChildPath(Constants::k_VertexList)); REQUIRE(vertexArray != nullptr); @@ -308,6 +355,21 @@ void CheckTestDataStructure(const DataStructure& dataStructure) const auto& faces = faceArray->getDataStoreRef(); CheckDataStore(faces, 3); + const auto* quadArray = dataStructure.getDataAs(dataGroupPath.createChildPath(Constants::k_QuadArray)); + REQUIRE(quadArray != nullptr); + const auto& quads = quadArray->getDataStoreRef(); + CheckDataStore(quads, 4); + + const auto* hexaArray = dataStructure.getDataAs(dataGroupPath.createChildPath(Constants::k_HexaArray)); + REQUIRE(hexaArray != nullptr); + const auto& hexa = hexaArray->getDataStoreRef(); + CheckDataStore(hexa, 8); + + const auto* tetraArray = dataStructure.getDataAs(dataGroupPath.createChildPath(Constants::k_TetraArray)); + REQUIRE(tetraArray != nullptr); + const auto& tetra = tetraArray->getDataStoreRef(); + CheckDataStore(tetra, 4); + const auto* polyArray = dataStructure.getDataAs(dataGroupPath.createChildPath(Constants::k_Polyhedra)); REQUIRE(polyArray != nullptr); const auto& polyhedra = polyArray->getDataStoreRef(); @@ -324,6 +386,17 @@ void CheckTestDataStructure(const DataStructure& dataStructure) REQUIRE(stringArray->at(4) == "5"); REQUIRE(stringArray->at(5) == "6"); + CheckScalarData(dataStructure, dataGroupPath.createChildPath(Constants::k_Int8)); + CheckScalarData(dataStructure, dataGroupPath.createChildPath(Constants::k_Int16)); + CheckScalarData(dataStructure, dataGroupPath.createChildPath(Constants::k_Int32)); + CheckScalarData(dataStructure, dataGroupPath.createChildPath(Constants::k_Int64)); + CheckScalarData(dataStructure, dataGroupPath.createChildPath(Constants::k_UInt8)); + CheckScalarData(dataStructure, dataGroupPath.createChildPath(Constants::k_UInt16)); + CheckScalarData(dataStructure, dataGroupPath.createChildPath(Constants::k_UInt32)); + CheckScalarData(dataStructure, dataGroupPath.createChildPath(Constants::k_UInt64)); + CheckScalarData(dataStructure, dataGroupPath.createChildPath(Constants::k_Float32)); + CheckScalarData(dataStructure, dataGroupPath.createChildPath(Constants::k_Float64)); + const auto* vertexGeom = dataStructure.getDataAs(DataPath({Constants::k_VertexGeom})); CheckGeom0D(vertexGeom, vertexArray->getId()); @@ -331,29 +404,56 @@ void CheckTestDataStructure(const DataStructure& dataStructure) CheckGeom1D(edgeGeom, vertexArray->getId(), edgeArray->getId()); const auto* quadGeom = dataStructure.getDataAs(DataPath({Constants::k_QuadGeom})); - CheckGeom2D(quadGeom, vertexArray->getId(), edgeArray->getId(), faceArray->getId()); + CheckGeom2D(quadGeom, vertexArray->getId(), edgeArray->getId(), quadArray->getId()); const auto* triGeom = dataStructure.getDataAs(DataPath({Constants::k_TriangleGeom})); CheckGeom2D(triGeom, vertexArray->getId(), edgeArray->getId(), faceArray->getId()); const auto* hexGeom = dataStructure.getDataAs(DataPath({Constants::k_HexGeom})); - CheckGeom3D(hexGeom, vertexArray->getId(), edgeArray->getId(), faceArray->getId(), polyArray->getId()); + CheckGeom3D(hexGeom, vertexArray->getId(), edgeArray->getId(), hexaArray->getId(), polyArray->getId()); const auto* tetraGeom = dataStructure.getDataAs(DataPath({Constants::k_TetrahedralGeom})); - CheckGeom3D(tetraGeom, vertexArray->getId(), edgeArray->getId(), faceArray->getId(), polyArray->getId()); + CheckGeom3D(tetraGeom, vertexArray->getId(), edgeArray->getId(), tetraArray->getId(), polyArray->getId()); + // Image Geom DataPath imageGeomPath({Constants::k_ImageGeom}); const auto* imageGeom = dataStructure.getDataAs(imageGeomPath); REQUIRE(imageGeom != nullptr); - auto dims = imageGeom->getDimensions(); - REQUIRE(dims[0] == Constants::k_TupleShape[0]); - REQUIRE(dims[1] == Constants::k_TupleShape[1]); - REQUIRE(dims[2] == Constants::k_TupleShape[2]); + REQUIRE(imageGeom->getDimensions() == Constants::k_ImageShape); const auto* cellData = dataStructure.getDataAs(imageGeomPath.createChildPath(Constants::k_CellData)); REQUIRE(cellData != nullptr); REQUIRE(imageGeom->getCellDataId() == cellData->getId()); REQUIRE(cellData->getShape() == Constants::k_TupleShape); -}; + + // RectGrid Geom + DataPath rectGridGeomPath({Constants::k_RectGridGeom}); + const auto* rectGrid = dataStructure.getDataAs(rectGridGeomPath); + REQUIRE(rectGrid != nullptr); + REQUIRE(rectGrid->getCellDataId() == cellData->getId()); + auto rectDims = rectGrid->getDimensions(); + REQUIRE(rectDims[0] == Constants::k_TupleShape[0]); + REQUIRE(rectDims[1] == Constants::k_TupleShape[1]); + REQUIRE(rectDims[2] == Constants::k_TupleShape[2]); + DataPath xPath = rectGridGeomPath.createChildPath(Constants::k_XBounds); + const auto* xBoundsArray = dataStructure.getDataAs(xPath); + REQUIRE(xBoundsArray != nullptr); + DataPath yPath = rectGridGeomPath.createChildPath(Constants::k_YBounds); + const auto* yBoundsArray = dataStructure.getDataAs(yPath); + REQUIRE(yBoundsArray != nullptr); + DataPath zPath = rectGridGeomPath.createChildPath(Constants::k_ZBounds); + const auto* zBoundsArray = dataStructure.getDataAs(zPath); + REQUIRE(zBoundsArray != nullptr); + REQUIRE(rectGrid->getXBoundsId() == xBoundsArray->getId()); + REQUIRE(rectGrid->getYBoundsId() == yBoundsArray->getId()); + REQUIRE(rectGrid->getZBoundsId() == zBoundsArray->getId()); +} + +template +void CreateScalarData(DataStructure& dataStructure, const std::string& name, DataObject::IdType parentId) +{ + auto* scalarData = ScalarData::Create(dataStructure, name, static_cast(Constants::k_ScalarValue), parentId); + REQUIRE(scalarData != nullptr); +} DataStructure CreateTestDataStructure() { @@ -380,6 +480,7 @@ DataStructure CreateTestDataStructure() listStorePtr->setList(4, std::vector{1, 2}); listStorePtr->setList(5, std::vector{1, 2}); auto* neighborList = Int16NeighborList::Create(dataStructure, Constants::k_NeighborList, listStorePtr, dataGroup->getId()); + REQUIRE(neighborList != nullptr); auto vertices = std::make_shared(Constants::k_TupleShape, ShapeType{3}, 0.0f); auto* vertexArray = Float32Array::Create(dataStructure, Constants::k_VertexList, vertices, dataGroup->getId()); @@ -389,39 +490,92 @@ DataStructure CreateTestDataStructure() auto* edgesArray = IGeometry::SharedEdgeList::Create(dataStructure, Constants::k_Edges, edges, dataGroup->getId()); FillDataStore(*edges.get()); - auto faces = std::make_shared(Constants::k_TupleShape, ShapeType{3}, 0); - auto* facesArray = IGeometry::SharedTriList::Create(dataStructure, Constants::k_Faces, faces, dataGroup->getId()); - FillDataStore(*faces.get()); + auto triangles = std::make_shared(Constants::k_TupleShape, ShapeType{3}, 0); + auto* trianglesArray = IGeometry::SharedTriList::Create(dataStructure, Constants::k_Faces, triangles, dataGroup->getId()); + FillDataStore(*triangles.get()); auto polyhedra = std::make_shared(Constants::k_TupleShape, ShapeType{4}, 0); - auto* polyhedraArray = IGeometry::SharedTriList::Create(dataStructure, Constants::k_Polyhedra, polyhedra, dataGroup->getId()); + auto* polyhedraArray = IGeometry::SharedFaceList::Create(dataStructure, Constants::k_Polyhedra, polyhedra, dataGroup->getId()); FillDataStore(*polyhedra.get()); + auto quadStore = std::make_shared(Constants::k_TupleShape, ShapeType{4}, 0); + auto* quadArray = IGeometry::SharedFaceList::Create(dataStructure, Constants::k_QuadArray, quadStore, dataGroup->getId()); + FillDataStore(*quadStore.get()); + + auto hexStore = std::make_shared(Constants::k_TupleShape, ShapeType{8}, 0); + auto* hexArray = IGeometry::SharedHexList::Create(dataStructure, Constants::k_HexaArray, hexStore, dataGroup->getId()); + FillDataStore(*hexStore.get()); + + auto tetraStore = std::make_shared(Constants::k_TupleShape, ShapeType{4}, 0); + auto* tetraArray = IGeometry::SharedTetList::Create(dataStructure, Constants::k_TetraArray, tetraStore, dataGroup->getId()); + FillDataStore(*tetraStore.get()); + StringArray::collection_type strings = {"1", "2", "3", "4", "5", "6"}; auto* stringArray = StringArray::CreateWithValues(dataStructure, Constants::k_StringArray, Constants::k_TupleShape, strings, dataGroup->getId()); + REQUIRE(stringArray != nullptr); + + CreateScalarData(dataStructure, Constants::k_Int8, dataGroup->getId()); + CreateScalarData(dataStructure, Constants::k_Int16, dataGroup->getId()); + CreateScalarData(dataStructure, Constants::k_Int32, dataGroup->getId()); + CreateScalarData(dataStructure, Constants::k_Int64, dataGroup->getId()); + CreateScalarData(dataStructure, Constants::k_UInt8, dataGroup->getId()); + CreateScalarData(dataStructure, Constants::k_UInt16, dataGroup->getId()); + CreateScalarData(dataStructure, Constants::k_UInt32, dataGroup->getId()); + CreateScalarData(dataStructure, Constants::k_UInt64, dataGroup->getId()); + CreateScalarData(dataStructure, Constants::k_Float32, dataGroup->getId()); + CreateScalarData(dataStructure, Constants::k_Float64, dataGroup->getId()); // Create Geometries and make sure special arrays are set. auto* imageGeom = ImageGeom::Create(dataStructure, Constants::k_ImageGeom); + REQUIRE(imageGeom != nullptr); auto* cellMatrix = AttributeMatrix::Create(dataStructure, Constants::k_CellData, Constants::k_TupleShape, imageGeom->getId()); + REQUIRE(cellMatrix != nullptr); imageGeom->setCellData(cellMatrix->getId()); - imageGeom->setDimensions(Constants::k_TupleShape); + imageGeom->setDimensions(Constants::k_ImageShape); + + // RectGrid Data + auto* rectGridGeom = RectGridGeom::Create(dataStructure, Constants::k_RectGridGeom); + REQUIRE(rectGridGeom != nullptr); + ShapeType xShape{Constants::k_TupleShape[0]}; + ShapeType componentBounds{1}; + auto xBounds = std::make_shared(xShape, componentBounds, 0.0f); + auto* xBoundsArray = Float32Array::Create(dataStructure, Constants::k_XBounds, xBounds, rectGridGeom->getId()); + FillDataStore(*xBounds.get()); + ShapeType yShape{Constants::k_TupleShape[1]}; + auto yBounds = std::make_shared(yShape, componentBounds, 0.0f); + auto* yBoundsArray = Float32Array::Create(dataStructure, Constants::k_YBounds, yBounds, rectGridGeom->getId()); + FillDataStore(*yBounds.get()); + ShapeType zShape{Constants::k_TupleShape[1]}; + auto zBounds = std::make_shared(zShape, componentBounds, 0.0f); + auto* zBoundsArray = Float32Array::Create(dataStructure, Constants::k_ZBounds, zBounds, rectGridGeom->getId()); + FillDataStore(*zBounds.get()); + + rectGridGeom->setDimensions(Constants::k_TupleShape); + rectGridGeom->setCellData(cellMatrix->getId()); + rectGridGeom->setBounds(xBoundsArray, yBoundsArray, zBoundsArray); // 0D Geometry auto* vertexGeom = VertexGeom::Create(dataStructure, Constants::k_VertexGeom); + REQUIRE(vertexGeom != nullptr); vertexGeom->setVertices(*vertexArray); // 1D Geometry auto* edgeGeom = EdgeGeom::Create(dataStructure, Constants::k_EdgeGeom); + REQUIRE(edgeGeom != nullptr); edgeGeom->setVertices(*vertexArray); edgeGeom->setEdgeList(*edgesArray); // 2D Geometries - auto* quadGeom = Create2DGeom(dataStructure, Constants::k_QuadGeom, *vertexArray, *edgesArray, *facesArray); - auto* triangleGeom = Create2DGeom(dataStructure, Constants::k_TriangleGeom, *vertexArray, *edgesArray, *facesArray); + auto* quadGeom = Create2DGeom(dataStructure, Constants::k_QuadGeom, *vertexArray, *edgesArray, *quadArray); + REQUIRE(quadGeom != nullptr); + auto* triangleGeom = Create2DGeom(dataStructure, Constants::k_TriangleGeom, *vertexArray, *edgesArray, *trianglesArray); + REQUIRE(triangleGeom != nullptr); // 3D Geometries - auto* hexGeom = Create3DGeom(dataStructure, Constants::k_HexGeom, *vertexArray, *edgesArray, *facesArray, *polyhedraArray); - auto* tetrahedralGeom = Create3DGeom(dataStructure, Constants::k_TetrahedralGeom, *vertexArray, *edgesArray, *facesArray, *polyhedraArray); + auto* hexGeom = Create3DGeom(dataStructure, Constants::k_HexGeom, *vertexArray, *edgesArray, *hexArray, *polyhedraArray); + REQUIRE(hexGeom != nullptr); + auto* tetrahedralGeom = Create3DGeom(dataStructure, Constants::k_TetrahedralGeom, *vertexArray, *edgesArray, *tetraArray, *polyhedraArray); + REQUIRE(tetrahedralGeom != nullptr); return dataStructure; } @@ -707,7 +861,7 @@ GeometryTestCase MakeGeometryTestCase(std::string typeName, std::function lock(m_DataMutex); @@ -741,29 +895,51 @@ TEST_CASE("WriteDREAM3DFilter:Invalid Parameters") } } -TEST_CASE("WriteDREAM3DFilter:Valid Parameters") +TEST_CASE("WriteDREAM3DFilter:Valid Parameters", "[ReadDREAM3DFilter][WriteDREAM3DFilter]") { UnitTest::LoadPlugins(); std::lock_guard lock(m_DataMutex); - DataStructure dataStructure = CreateTestDataStructure(); - Arguments args; - WriteDREAM3DFilter filter; + { + DataStructure dataStructure = CreateTestDataStructure(); + Arguments args; + WriteDREAM3DFilter filter; - args.insertOrAssign(WriteDREAM3DFilter::k_ExportFilePath, std::make_any(GetIODataPath())); - args.insertOrAssign(WriteDREAM3DFilter::k_WriteXdmf, std::make_any(false)); - args.insertOrAssign(WriteDREAM3DFilter::k_UseCompression, std::make_any(false)); - args.insertOrAssign(WriteDREAM3DFilter::k_CompressionLevel, std::make_any(1)); + args.insertOrAssign(WriteDREAM3DFilter::k_ExportFilePath, std::make_any(GetIODataPath())); + args.insertOrAssign(WriteDREAM3DFilter::k_WriteXdmf, std::make_any(false)); + args.insertOrAssign(WriteDREAM3DFilter::k_UseCompression, std::make_any(false)); + args.insertOrAssign(WriteDREAM3DFilter::k_CompressionLevel, std::make_any(1)); + + // Preflight the filter and check result + auto preflightResult = filter.preflight(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); + + auto result = filter.execute(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(result.result); + } + + // Check that the output file exists + REQUIRE(fs::exists(GetIODataPath())); + + // Check that the file can be read back in and that the imported DataStructure matches expected values. + { + auto fileReader = HDF5::FileIO::ReadFile(GetIODataPath()); + auto fileResult = DREAM3D::ReadFile(fileReader); + SIMPLNX_RESULT_REQUIRE_VALID(fileResult); - // Preflight the filter and check result - auto preflightResult = filter.preflight(dataStructure, args); - SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); + auto [pipeline, dataStructureRead] = fileResult.value(); + + CheckTestDataStructure(dataStructureRead); + } +} - auto result = filter.execute(dataStructure, args); - SIMPLNX_RESULT_REQUIRE_VALID(result.result); +void CheckXdmfFile() +{ + auto filepath = GetXdmfPath(); + REQUIRE(fs::exists(filepath)); } -TEST_CASE("WriteDREAM3D:Pipeline / WriteXdmf combinations") +TEST_CASE("WriteDREAM3D:Pipeline / WriteXdmf combinations", "[ReadDREAM3DFilter][WriteDREAM3DFilter]") { UnitTest::LoadPlugins(); std::lock_guard lock(m_DataMutex); @@ -773,9 +949,14 @@ TEST_CASE("WriteDREAM3D:Pipeline / WriteXdmf combinations") auto writeResult = DREAM3D::WriteFile(GetIODataPath(), CreateTestDataStructure(), exportPipeline, writeXdmf); SIMPLNX_RESULT_REQUIRE_VALID(writeResult); + + if(writeXdmf) + { + CheckXdmfFile(); + } } -TEST_CASE("WriteDREAM3D:Invalid File") +TEST_CASE("WriteDREAM3D:Invalid File", "[ReadDREAM3DFilter][WriteDREAM3DFilter]") { UnitTest::LoadPlugins(); std::lock_guard lock(m_DataMutex); @@ -785,6 +966,11 @@ TEST_CASE("WriteDREAM3D:Invalid File") auto writeResult = DREAM3D::WriteFile(fs::path(), CreateTestDataStructure(), exportPipeline, writeXdmf); SIMPLNX_RESULT_REQUIRE_INVALID(writeResult); + + if(writeXdmf) + { + CheckXdmfFile(); + } } TEST_CASE("DREAM3DFileTest:DREAM3D File IO Test", "[WriteDREAM3DFilter]") @@ -796,11 +982,13 @@ TEST_CASE("DREAM3DFileTest:DREAM3D File IO Test", "[WriteDREAM3DFilter]") bool writeXdmf = GENERATE(true, false); // Write .dream3d file { - auto fileData = CreateFileData(); - auto fileWriter = HDF5::FileIO::WriteFile(GetIODataPath()); - - auto writeResult = DREAM3D::WriteFile(fileWriter, fileData); + auto writeResult = DREAM3D::WriteFile(GetIODataPath(), CreateTestDataStructure(), CreateExportPipeline(), writeXdmf); SIMPLNX_RESULT_REQUIRE_VALID(writeResult); + + if(writeXdmf) + { + CheckXdmfFile(); + } } // Read .dream3d file diff --git a/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.cpp b/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.cpp index 395f14e831..9d4c114b2d 100644 --- a/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.cpp +++ b/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.cpp @@ -2370,7 +2370,7 @@ Result DREAM3D::ReadFile(const nx::core::HDF5::FileIO& fileRe } auto dataStructure = ImportDataStructureFromFile(fileReader, preflight); - if(pipeline.invalid()) + if(dataStructure.invalid()) { return {{nonstd::make_unexpected(std::move(dataStructure.errors()))}, std::move(dataStructure.warnings())}; } From d204f47c6c7550bd326489a3ae47b81525f86a75 Mon Sep 17 00:00:00 2001 From: Matthew Marine Date: Fri, 24 Jul 2026 10:33:01 -0400 Subject: [PATCH 4/9] PR changes for V&V docs * Added check that the use compression argument is disabled when importing from legacy SIMPL since it did not exist . --- .../SimplnxCore/test/DREAM3DFileTest.cpp | 1 + .../SimplnxCore/vv/WriteDREAM3DFilter.md | 18 ++++++++++++------ .../vv/deviations/WriteDREAM3DFilter.md | 2 ++ 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp b/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp index f641e09af8..872f1676fd 100644 --- a/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp +++ b/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp @@ -1417,6 +1417,7 @@ TEST_CASE("SimplnxCore::WriteDREAM3DFilter: SIMPL Backwards Compatibility", "[Si const Arguments args = pipelineFilter->getArguments(); CHECK(args.value(WriteDREAM3DFilter::k_ExportFilePath) == fs::path("/test/path/output.dream3d")); CHECK(args.value(WriteDREAM3DFilter::k_WriteXdmf) == true); + CHECK(args.value(WriteDREAM3DFilter::k_UseCompression) == false); } } } diff --git a/src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md b/src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md index c4e2c9ca08..74fad98955 100644 --- a/src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md +++ b/src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md @@ -19,19 +19,19 @@ | Code paths enumerated | **14 of 19** exercised; 5 gaps are defensive/unreachable-via-public-API guards (see table). | | Tests today | **17 TEST_CASEs** (some with `GENERATE`/`DYNAMIC_SECTION` multiplying cases) — preflight validation, full round-trip content fidelity across every geometry/DataObject type, SIMPL args backward-compat, and a 6-test compression sub-suite (layout, bypass threshold, level monotonicity). | | Exemplar archive | **None.** Every test builds its `DataStructure` inline in C++ and round-trips it through `WriteFile`/`ReadFile` in the same run — no cached `.tar.gz` golden file is used or needed for a Class 1 oracle. | -| Legacy comparison | **Not run** — no like-for-like binary comparison is possible. `DataContainerWriter` (6.5.171) emits the legacy `DataContainers` HDF5 layout with no compression option; SIMPLNX always emits the current v8 `DataStructure` layout regardless of source pipeline. The format change is deliberate (Rewrite), so fidelity is verified independently via round-trip Class 1 tests instead of a legacy diff. | +| Legacy comparison | **Not run — and not applicable.** The two writers target deliberately different on-disk contracts, so a byte/dataset-level A/B against 6.5.171 `DataContainerWriter` output would be 100% noise by design, not signal. `ReadDREAM3DFilter` is the only tool in either codebase that understands both formats; fidelity is instead verified independently via round-trip Class 1 tests. | | Bug flags | None. | -| V&V phase | Oracle chosen, code paths enumerated, test inventory reviewed, deviations documented. Outstanding: second-engineer review of the oracle design and of the 5 uncovered defensive paths. | +| V&V phase | Oracle chosen, code paths enumerated, test inventory reviewed, deviations documented. Outstanding: second-engineer review of the oracle design, of the 5 uncovered defensive paths, and of the `DynamicListArray` IO gap (Known limitations). | ## Summary -`WriteDREAM3DFilter` serializes the current `DataStructure` (and, when run inside a pipeline, the preceding `Pipeline`) to an HDF5 `.dream3d` file, with an optional companion `.xdmf` sidecar and optional gzip compression of array datasets. It replaces legacy SIMPL's `DataContainerWriter` under the same conceptual role but with an intentionally new v8 file format, so verification is independent of 6.5.171: correctness is established by writing hand-built `DataStructure`s covering every geometry and `DataObject` type, then reading them back and asserting exact structural/content equality (Class 1 Analytical), plus closed-form assertions on the resulting HDF5 physical layout under each compression setting. All 17 test cases pass; no bugs were found. `StatsDataArray`/`StructArray` (SIMPL's per-ensemble statistics types) are out of scope for this cycle — those `DataObject` types do not yet exist in this branch of simplnx (see deviation D2). +`WriteDREAM3DFilter` serializes the current `DataStructure` (and, when run inside a pipeline, the preceding `Pipeline`) to an HDF5 `.dream3d` file, with an optional companion `.xdmf` sidecar and optional gzip compression of array datasets. It replaces legacy SIMPL's `DataContainerWriter` under the same conceptual role but with an intentionally new v8 file format, so verification is independent of 6.5.171: correctness is established by writing hand-built `DataStructure`s covering every geometry and `DataObject` type, then reading them back and asserting exact structural/content equality (Class 1 Analytical), plus closed-form assertions on the resulting HDF5 physical layout under each compression setting. All 17 test cases pass; no bugs were found. `StatsDataArray`/`StructArray` (SIMPL's per-ensemble statistics types) are out of scope for this cycle — those `DataObject` types do not yet exist in this branch of simplnx (see deviation D2). Separately, a bare `DynamicListArray` (as opposed to its `NeighborList` specialization) has no HDF5 IO factory at all in the current codebase and cannot be written by this or any filter (see Known limitations). ## Algorithm Relationship *Classification:* **Rewrite** ~~| Port | Minor changes | New filter~~ -`WriteDREAM3DFilter` keeps the SIMPL UUID mapping (`3fcd4c43-9d75-5b86-aad4-4441bc914f37` → `WriteDREAM3DFilter`, `SimplnxCoreLegacyUUIDMapping.hpp:170`) and the legacy `DataContainerWriter` role, but the algorithm (`Algorithms/WriteDREAM3D.cpp`, 82 lines) was designed from the start for the current v8 `DataStructure` HDF5 layout, `AtomicFile`-based atomic writes, and (as of PR #1606) optional gzip compression — none of which exist in the legacy 6.5.171 writer. This has never been a line-by-line port of the legacy C++; the file format itself is a clean-sheet design (`k_CurrentFileVersion = "8.0"` vs. legacy's `"7.0"`/`DataContainers` group tag, see `Dream3dIO.cpp:31-42`). +`WriteDREAM3DFilter` keeps the SIMPL UUID mapping (`3fcd4c43-9d75-5b86-aad4-4441bc914f37` → `WriteDREAM3DFilter`, `SimplnxCoreLegacyUUIDMapping.hpp:170`) and the legacy `DataContainerWriter` role, but the algorithm (`Algorithms/WriteDREAM3D.cpp`, 82 lines) was designed from the start for the current v8 `DataStructure` HDF5 layout, `AtomicFile`-based atomic writes, and (as of PR #1606) optional gzip compression — none of which exist in the legacy 6.5.171 writer. This has never been a line-by-line port of the legacy C++; the file format itself is a clean-sheet design (`k_CurrentFileVersion = "8.0"` vs. legacy's `"7.0"`/`DataContainers` group tag, see `Dream3dIO.hpp:31`). *Evidence:* `parametersVersion()` is at 2 (compression parameters added after the filter's initial release); `git log --follow` on the algorithm/filter files shows the write path has been restructured multiple times since inception (out-of-core support #1253, atomic-file rework #900, algorithm-class extraction #1544) without ever tracking legacy DataContainerWriter's implementation. @@ -77,7 +77,7 @@ Source: `src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteDREAM3D | 10 | Execute — pipeline | `PipelineNode == nullptr` → empty pipeline written | `"WriteDREAM3DFilter:Valid Parameters"`, `"DREAM3DFileTest::StringArray"`, all `Compression_*` tests (all call `filter.execute(ds, args)` directly) | | 11 | Execute — options | `use_compression=true` → `writeOptions.compressionLevel = CompressionLevel` | `"...Compression_On_IsChunkedAndDeflated"`, `"...Compression_SmallArray_Bypasses"`, `"...Compression_LevelsRoundTrip"` | | 12 | Execute — options | `use_compression=false` → `writeOptions.compressionLevel = 0` | `"...Compression_Off_IsContiguous"`; `"WriteDREAM3DFilter:Valid Parameters"` | -| 13 | Execute — write | `DREAM3D::WriteFile(...)` returns invalid → skip commit, return the error | *Not directly tested* through the full filter/`AtomicFile` path. Once `AtomicFile::Create` has validated the destination, the underlying HDF5 write essentially cannot fail independently — same root gap as Path 6. | +| 13 | Execute — write | `DREAM3D::WriteFile(...)` returns invalid → skip commit, return the error | *Not directly tested* through the full filter/`AtomicFile` path, but concretely reachable (not merely defensive): `HDF5::DataStructureWriter::WriteFile` returns error `-5` ("Could not find IO factory for datatype: …") for any `DataObject` type with no registered HDF5 IO factory — see Known limitations below (a bare `DynamicListArray`, not wrapped as `NeighborList`). No test currently puts such an object in the `DataStructure` before writing. | | 14 | Execute — write | `DREAM3D::WriteFile(...)` returns valid → proceed to commit | Every passing execute-path test | | 15 | Execute — commit | `atomicFile.commit()` fails (rename onto final destination fails) | *Not directly tested.* Would require the destination path to become invalid between `AtomicFile::Create` and `commit()` (e.g., concurrent deletion of the parent directory) — a race not exercised by the suite. | | 16 | Execute — commit | `atomicFile.commit()` succeeds | Every passing execute-path test (the output file is present and re-readable in every round-trip test) | @@ -111,9 +111,15 @@ Source: `src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteDREAM3D None. Every test above builds its input `DataStructure` inline in C++ and never loads a downloaded `.tar.gz` exemplar — the Class 1 oracle's "expected output" is the test's own hand-built input, so no cached golden file is required or used. (`Small_IN100_dream3d_v3.tar.gz`, referenced elsewhere in this test file, backs unrelated `ReadDREAM3DFilter`-only test cases and is not consumed by any Write-side test.) +## Known limitations (current simplnx HDF5 IO layer) + +`DynamicListArray` (the generic variable-length per-tuple list container that `NeighborList` is itself built on) has **no registered HDF5 IO factory**. `HDF5::DataIOManager::addCoreFactories()` (`DataStructure/IO/HDF5/DataIOManager.cpp:33-81`) registers factories per concrete `DataArray` type, per `NeighborList` specialization, geometries, `AttributeMatrix`, `DataGroup`, `StringArray`, and scalar attributes — but nothing for a bare `DynamicListArray`. If one is ever placed directly in a `DataStructure` (outside of a `NeighborList` specialization) and written, `HDF5::DataStructureWriter::WriteFile` hits its "no factory found" guard (`DataStructureWriter.cpp:153-157`) and fails with error `-5` ("Could not find IO factory for datatype: …"), surfacing through `WriteDREAM3DFilter` as Path 13 above. + +This is a gap in the shared HDF5 IO layer, not something specific to `WriteDREAM3DFilter`'s own algorithm — the same gap would affect `ReadDREAM3DFilter` for the same object type. It is not raised as a formal Deviation because there is no confirmed 6.5.171 pipeline behavior being compared against (unlike D2, which names concrete legacy types); it is recorded here as a known, currently-untested capability boundary of what this filter can serialize. No current `WriteDREAM3DFilter` test constructs a bare `DynamicListArray`, so Path 13 remains untested rather than confirmed-safe. + ## Deviations from DREAM3D 6.5.171 -Comparison run: file-format inspection only (no data-value diff is meaningful — see below). Two deviations documented: +No byte/dataset-level A/B was run against 6.5.171 `DataContainerWriter` output, and none is warranted: the two writers target deliberately different on-disk contracts (see D1), so such a diff would be 100% noise by design, not signal. `ReadDREAM3DFilter` is the only tool that understands both formats, and cross-format fidelity is its concern, not this filter's. Two scope/format deviations are documented instead: - `WriteDREAM3DFilter-D1` — on-disk file format is a deliberate, complete rewrite (v8 `DataStructure` layout vs. legacy v7 `DataContainers` layout; optional gzip compression; atomic write) — see `vv/deviations/WriteDREAM3DFilter.md`. - `WriteDREAM3DFilter-D2` — `StatsDataArray`/`StructArray` (SIMPL ensemble-statistics types) cannot currently be written because those `DataObject` types do not yet exist in this branch of simplnx — see `vv/deviations/WriteDREAM3DFilter.md`. diff --git a/src/Plugins/SimplnxCore/vv/deviations/WriteDREAM3DFilter.md b/src/Plugins/SimplnxCore/vv/deviations/WriteDREAM3DFilter.md index 6a9b4d14d6..e280424a14 100644 --- a/src/Plugins/SimplnxCore/vv/deviations/WriteDREAM3DFilter.md +++ b/src/Plugins/SimplnxCore/vv/deviations/WriteDREAM3DFilter.md @@ -20,6 +20,8 @@ Entries are referenced by stable ID (`WriteDREAM3DFilter-D`) from the V&V rep **Affected users:** Anyone attempting to open a SIMPLNX-written `.dream3d` file directly in a DREAM3D 6.5.171 install, or vice versa, outside of DREAM3D-NX's own dual-format `ReadDREAM3DFilter`. Users staying entirely within DREAM3D-NX (write with `WriteDREAM3DFilter`, read with `ReadDREAM3DFilter`) never observe this — round-trip fidelity within the new format is verified by the Class 1 tests in the V&V report. +**Why no A/B comparison was attempted:** Because the two writers deliberately target different on-disk contracts, a byte-level or dataset-level A/B diff between a SIMPLNX-written file and a 6.5.171-written file would not measure anything meaningful — every dataset path, group name, and file-version tag differs by construction, so the diff would be 100% noise, not signal. This is why the V&V report's "Legacy comparison" is marked **Not run — and not applicable** rather than "deferred." `ReadDREAM3DFilter` is the only tool in either codebase that understands both the legacy `DataContainers` layout and the current `DataStructure` layout; it is the correct place to verify cross-format fidelity (import old → re-export new → confirm no data loss), not a Write-side A/B. + **Recommendation:** Trust SIMPLNX. The new format is a deliberate design improvement (atomicity, compression, JSON pipeline embedding) required for capabilities legacy DREAM3D never had (out-of-core datasets, versioned pipeline metadata). It is not "wrong" relative to 6.5.171; it is a different, and newer, on-disk contract. DREAM3D-NX remains able to *read* legacy 6.5.171 files via `ReadDREAM3DFilter`'s legacy import path, so migration is one-directional by design (import old, export new) rather than bidirectional. --- From 29e9b2db7e1f7694ab24206938c47a514044a263 Mon Sep 17 00:00:00 2001 From: Michael Jackson Date: Wed, 12 Aug 2026 15:01:59 -0400 Subject: [PATCH 5/9] BUG: Fix xdmf writer argument transposition and V&V test defects * Fix WriteXdmfNodeGeometry1D/2D/3D forwarding geomName and hdf5FilePath in swapped order, which produced .xdmf node-attribute references that ParaView/VisIt could not resolve * Strengthen CheckXdmfFile to validate every heavy-data reference in the sidecar resolves to the written .dream3d file instead of only checking that the file exists, and add a vertex AttributeMatrix to the test fixture so node-centered attribute references are actually emitted * Fix swapped face/polyhedra connectivity lists for the Hexahedral and Tetrahedral test geometries (hex cells need the 8-component list as polyhedra and quad faces; tets need triangle faces) * Size RectGrid bounds arrays as N+1 for N cells and build zShape from k_TupleShape[2] instead of [1]; assert bounds sizes on read-back * Remove the inverted .xdmf existence assertion from the Invalid File test, which only passed via a stale sidecar from an earlier TEST_CASE * Give each file-writing TEST_CASE its own output filename so parallel ctest processes no longer race on newFile.dream3d * Add WriteDREAM3D:FileData Overload test so the exported WriteFile(FileIO&, FileData) forwarder and CreateFileData() stay covered * Use an empty DataStructure in the preflight-only Invalid Parameters test and add UnitTest::CheckArraysInheritTupleDims to the new tests * Correct the V&V report: 19 TEST_CASEs, eight geometry types including RectGrid, add the missing Geometry Nested In DataGroup Round Trip inventory row, and record the fixed xdmf bug under Bug flags Signed-off-by: Michael Jackson --- .../SimplnxCore/test/DREAM3DFileTest.cpp | 159 +++++++++++++++--- .../SimplnxCore/vv/WriteDREAM3DFilter.md | 23 +-- .../Utilities/Parsing/DREAM3D/Dream3dIO.cpp | 6 +- 3 files changed, 149 insertions(+), 39 deletions(-) diff --git a/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp b/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp index 872f1676fd..44099b1f00 100644 --- a/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp +++ b/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp @@ -42,6 +42,7 @@ #include #include +#include #include #include #include @@ -61,6 +62,11 @@ const fs::path k_ExportFilename2 = "export2.dream3d"; const fs::path k_MultiExportFilename1 = "multi_export1.dream3d"; const fs::path k_MultiExportFilename2 = "multi_export2.dream3d"; const fs::path k_MultiExportFilename3 = "multi_export3.dream3d"; +// Each TEST_CASE runs as a separate ctest process, so tests that write a file must each use +// their own filename to avoid cross-process races under parallel ctest. +const fs::path k_ValidParamsFilename = "write_dream3d_valid_params.dream3d"; +const fs::path k_PipelineComboFilename = "write_dream3d_pipeline_combos.dream3d"; +const fs::path k_FileDataFilename = "write_dream3d_file_data.dream3d"; constexpr StringLiteral k_CellData = "Cell Data"; constexpr StringLiteral k_DataContainer = "Data Container"; @@ -79,6 +85,8 @@ constexpr StringLiteral k_ZBounds = "Z Bounds"; constexpr StringLiteral k_DynamicListArray = "DynamicList"; constexpr StringLiteral k_NeighborList = "NeighborList"; constexpr StringLiteral k_StringArray = "String Array"; +constexpr StringLiteral k_VertexData = "Vertex Data"; +constexpr StringLiteral k_VertexValues = "Vertex Values"; constexpr StringLiteral k_VertexList = "Vertices"; constexpr StringLiteral k_Edges = "Edges"; constexpr StringLiteral k_Faces = "Faces"; @@ -127,7 +135,7 @@ fs::path GetDataDir(const Application& app) return std::filesystem::path(unit_test::k_BinaryTestOutputDir.view()); } -fs::path GetIODataPath() +fs::path GetTestFilePath(const fs::path& filename) { auto app = Application::Instance(); if(app == nullptr) @@ -135,12 +143,32 @@ fs::path GetIODataPath() throw std::runtime_error("nx::core::Application instance not found"); } - return GetDataDir(*app) / Constants::k_Dream3dFilename; + return GetDataDir(*app) / filename; +} + +fs::path GetIODataPath() +{ + return GetTestFilePath(Constants::k_Dream3dFilename); +} + +fs::path GetValidParamsFilePath() +{ + return GetTestFilePath(Constants::k_ValidParamsFilename); +} + +fs::path GetPipelineComboFilePath() +{ + return GetTestFilePath(Constants::k_PipelineComboFilename); } -fs::path GetXdmfPath() +fs::path GetFileDataFilePath() { - fs::path filePath = GetIODataPath(); + return GetTestFilePath(Constants::k_FileDataFilename); +} + +fs::path GetXdmfPath(const fs::path& dream3dPath) +{ + fs::path filePath = dream3dPath; filePath.replace_extension(".xdmf"); return filePath; } @@ -409,11 +437,19 @@ void CheckTestDataStructure(const DataStructure& dataStructure) const auto* triGeom = dataStructure.getDataAs(DataPath({Constants::k_TriangleGeom})); CheckGeom2D(triGeom, vertexArray->getId(), edgeArray->getId(), faceArray->getId()); + DataPath vertexDataPath = DataPath({Constants::k_TriangleGeom}).createChildPath(Constants::k_VertexData); + const auto* triVertexMatrix = dataStructure.getDataAs(vertexDataPath); + REQUIRE(triVertexMatrix != nullptr); + REQUIRE(triGeom->getVertexAttributeMatrixId() == triVertexMatrix->getId()); + const auto* vertexValuesArray = dataStructure.getDataAs(vertexDataPath.createChildPath(Constants::k_VertexValues)); + REQUIRE(vertexValuesArray != nullptr); + CheckDataStore(vertexValuesArray->getDataStoreRef(), 1); + const auto* hexGeom = dataStructure.getDataAs(DataPath({Constants::k_HexGeom})); - CheckGeom3D(hexGeom, vertexArray->getId(), edgeArray->getId(), hexaArray->getId(), polyArray->getId()); + CheckGeom3D(hexGeom, vertexArray->getId(), edgeArray->getId(), quadArray->getId(), hexaArray->getId()); const auto* tetraGeom = dataStructure.getDataAs(DataPath({Constants::k_TetrahedralGeom})); - CheckGeom3D(tetraGeom, vertexArray->getId(), edgeArray->getId(), tetraArray->getId(), polyArray->getId()); + CheckGeom3D(tetraGeom, vertexArray->getId(), edgeArray->getId(), faceArray->getId(), tetraArray->getId()); // Image Geom DataPath imageGeomPath({Constants::k_ImageGeom}); @@ -437,12 +473,15 @@ void CheckTestDataStructure(const DataStructure& dataStructure) DataPath xPath = rectGridGeomPath.createChildPath(Constants::k_XBounds); const auto* xBoundsArray = dataStructure.getDataAs(xPath); REQUIRE(xBoundsArray != nullptr); + REQUIRE(xBoundsArray->getNumberOfTuples() == Constants::k_TupleShape[0] + 1); DataPath yPath = rectGridGeomPath.createChildPath(Constants::k_YBounds); const auto* yBoundsArray = dataStructure.getDataAs(yPath); REQUIRE(yBoundsArray != nullptr); + REQUIRE(yBoundsArray->getNumberOfTuples() == Constants::k_TupleShape[1] + 1); DataPath zPath = rectGridGeomPath.createChildPath(Constants::k_ZBounds); const auto* zBoundsArray = dataStructure.getDataAs(zPath); REQUIRE(zBoundsArray != nullptr); + REQUIRE(zBoundsArray->getNumberOfTuples() == Constants::k_TupleShape[2] + 1); REQUIRE(rectGrid->getXBoundsId() == xBoundsArray->getId()); REQUIRE(rectGrid->getYBoundsId() == yBoundsArray->getId()); REQUIRE(rectGrid->getZBoundsId() == zBoundsArray->getId()); @@ -496,6 +535,7 @@ DataStructure CreateTestDataStructure() auto polyhedra = std::make_shared(Constants::k_TupleShape, ShapeType{4}, 0); auto* polyhedraArray = IGeometry::SharedFaceList::Create(dataStructure, Constants::k_Polyhedra, polyhedra, dataGroup->getId()); + REQUIRE(polyhedraArray != nullptr); FillDataStore(*polyhedra.get()); auto quadStore = std::make_shared(Constants::k_TupleShape, ShapeType{4}, 0); @@ -536,16 +576,17 @@ DataStructure CreateTestDataStructure() // RectGrid Data auto* rectGridGeom = RectGridGeom::Create(dataStructure, Constants::k_RectGridGeom); REQUIRE(rectGridGeom != nullptr); - ShapeType xShape{Constants::k_TupleShape[0]}; + // N cells along a dimension require N+1 bounds values + ShapeType xShape{Constants::k_TupleShape[0] + 1}; ShapeType componentBounds{1}; auto xBounds = std::make_shared(xShape, componentBounds, 0.0f); auto* xBoundsArray = Float32Array::Create(dataStructure, Constants::k_XBounds, xBounds, rectGridGeom->getId()); FillDataStore(*xBounds.get()); - ShapeType yShape{Constants::k_TupleShape[1]}; + ShapeType yShape{Constants::k_TupleShape[1] + 1}; auto yBounds = std::make_shared(yShape, componentBounds, 0.0f); auto* yBoundsArray = Float32Array::Create(dataStructure, Constants::k_YBounds, yBounds, rectGridGeom->getId()); FillDataStore(*yBounds.get()); - ShapeType zShape{Constants::k_TupleShape[1]}; + ShapeType zShape{Constants::k_TupleShape[2] + 1}; auto zBounds = std::make_shared(zShape, componentBounds, 0.0f); auto* zBoundsArray = Float32Array::Create(dataStructure, Constants::k_ZBounds, zBounds, rectGridGeom->getId()); FillDataStore(*zBounds.get()); @@ -571,10 +612,20 @@ DataStructure CreateTestDataStructure() auto* triangleGeom = Create2DGeom(dataStructure, Constants::k_TriangleGeom, *vertexArray, *edgesArray, *trianglesArray); REQUIRE(triangleGeom != nullptr); - // 3D Geometries - auto* hexGeom = Create3DGeom(dataStructure, Constants::k_HexGeom, *vertexArray, *edgesArray, *hexArray, *polyhedraArray); + // Vertex data on a node geometry so the xdmf writer emits node-centered attribute references + auto* vertexMatrix = AttributeMatrix::Create(dataStructure, Constants::k_VertexData, Constants::k_TupleShape, triangleGeom->getId()); + REQUIRE(vertexMatrix != nullptr); + triangleGeom->setVertexAttributeMatrix(*vertexMatrix); + auto vertexValues = std::make_shared(Constants::k_TupleShape, ShapeType{1}, 0.0f); + auto* vertexValuesArray = Float32Array::Create(dataStructure, Constants::k_VertexValues, vertexValues, vertexMatrix->getId()); + REQUIRE(vertexValuesArray != nullptr); + FillDataStore(*vertexValues.get()); + + // 3D Geometries. Hexahedra have quad faces (4 vertices) and 8-vertex cells; tetrahedra have + // triangle faces (3 vertices) and 4-vertex cells. + auto* hexGeom = Create3DGeom(dataStructure, Constants::k_HexGeom, *vertexArray, *edgesArray, *quadArray, *hexArray); REQUIRE(hexGeom != nullptr); - auto* tetrahedralGeom = Create3DGeom(dataStructure, Constants::k_TetrahedralGeom, *vertexArray, *edgesArray, *tetraArray, *polyhedraArray); + auto* tetrahedralGeom = Create3DGeom(dataStructure, Constants::k_TetrahedralGeom, *vertexArray, *edgesArray, *trianglesArray, *tetraArray); REQUIRE(tetrahedralGeom != nullptr); return dataStructure; @@ -866,7 +917,8 @@ TEST_CASE("WriteDREAM3DFilter:Invalid Parameters", "[ReadDREAM3DFilter][WriteDRE UnitTest::LoadPlugins(); std::lock_guard lock(m_DataMutex); - DataStructure dataStructure = CreateTestDataStructure(); + // Both sections only exercise preflight parameter guards, so no data is required. + DataStructure dataStructure; Arguments args; WriteDREAM3DFilter filter; @@ -893,6 +945,8 @@ TEST_CASE("WriteDREAM3DFilter:Invalid Parameters", "[ReadDREAM3DFilter][WriteDRE auto preflightResult = filter.preflight(dataStructure, args); SIMPLNX_RESULT_REQUIRE_INVALID(preflightResult.outputActions); } + + UnitTest::CheckArraysInheritTupleDims(dataStructure); } TEST_CASE("WriteDREAM3DFilter:Valid Parameters", "[ReadDREAM3DFilter][WriteDREAM3DFilter]") @@ -900,12 +954,14 @@ TEST_CASE("WriteDREAM3DFilter:Valid Parameters", "[ReadDREAM3DFilter][WriteDREAM UnitTest::LoadPlugins(); std::lock_guard lock(m_DataMutex); + const fs::path exportFilePath = GetValidParamsFilePath(); + { DataStructure dataStructure = CreateTestDataStructure(); Arguments args; WriteDREAM3DFilter filter; - args.insertOrAssign(WriteDREAM3DFilter::k_ExportFilePath, std::make_any(GetIODataPath())); + args.insertOrAssign(WriteDREAM3DFilter::k_ExportFilePath, std::make_any(exportFilePath)); args.insertOrAssign(WriteDREAM3DFilter::k_WriteXdmf, std::make_any(false)); args.insertOrAssign(WriteDREAM3DFilter::k_UseCompression, std::make_any(false)); args.insertOrAssign(WriteDREAM3DFilter::k_CompressionLevel, std::make_any(1)); @@ -919,24 +975,46 @@ TEST_CASE("WriteDREAM3DFilter:Valid Parameters", "[ReadDREAM3DFilter][WriteDREAM } // Check that the output file exists - REQUIRE(fs::exists(GetIODataPath())); + REQUIRE(fs::exists(exportFilePath)); // Check that the file can be read back in and that the imported DataStructure matches expected values. { - auto fileReader = HDF5::FileIO::ReadFile(GetIODataPath()); + auto fileReader = HDF5::FileIO::ReadFile(exportFilePath); auto fileResult = DREAM3D::ReadFile(fileReader); SIMPLNX_RESULT_REQUIRE_VALID(fileResult); auto [pipeline, dataStructureRead] = fileResult.value(); CheckTestDataStructure(dataStructureRead); + UnitTest::CheckArraysInheritTupleDims(dataStructureRead); } } -void CheckXdmfFile() +void CheckXdmfFile(const fs::path& dream3dPath) { - auto filepath = GetXdmfPath(); + auto filepath = GetXdmfPath(dream3dPath); REQUIRE(fs::exists(filepath)); + + // Every heavy-data reference in the sidecar must point into the .dream3d file that was + // written; a reference built from any other token (e.g. a geometry name) cannot be + // resolved by ParaView/VisIt. + const std::string expectedPrefix = dream3dPath.filename().string() + ":/DataStructure/"; + std::ifstream xdmfFile(filepath); + REQUIRE(xdmfFile.good()); + usize referenceCount = 0; + std::string line; + while(std::getline(xdmfFile, line)) + { + const usize refPos = line.find(":/DataStructure/"); + if(refPos == std::string::npos) + { + continue; + } + referenceCount++; + const usize start = line.find_first_not_of(" \t"); + REQUIRE(line.compare(start, expectedPrefix.size(), expectedPrefix) == 0); + } + REQUIRE(referenceCount > 0); } TEST_CASE("WriteDREAM3D:Pipeline / WriteXdmf combinations", "[ReadDREAM3DFilter][WriteDREAM3DFilter]") @@ -947,13 +1025,42 @@ TEST_CASE("WriteDREAM3D:Pipeline / WriteXdmf combinations", "[ReadDREAM3DFilter] bool writeXdmf = GENERATE(true, false); Pipeline exportPipeline = GENERATE(CreateExportPipeline(), Pipeline()); - auto writeResult = DREAM3D::WriteFile(GetIODataPath(), CreateTestDataStructure(), exportPipeline, writeXdmf); + const fs::path exportFilePath = GetPipelineComboFilePath(); + DataStructure dataStructure = CreateTestDataStructure(); + auto writeResult = DREAM3D::WriteFile(exportFilePath, dataStructure, exportPipeline, writeXdmf); SIMPLNX_RESULT_REQUIRE_VALID(writeResult); if(writeXdmf) { - CheckXdmfFile(); + CheckXdmfFile(exportFilePath); + } + + UnitTest::CheckArraysInheritTupleDims(dataStructure); +} + +TEST_CASE("WriteDREAM3D:FileData Overload", "[ReadDREAM3DFilter][WriteDREAM3DFilter]") +{ + UnitTest::LoadPlugins(); + std::lock_guard lock(m_DataMutex); + + const fs::path exportFilePath = GetFileDataFilePath(); + + // Write through the exported FileData overload so its pipeline/DataStructure ordering stays covered. + { + auto fileWriter = HDF5::FileIO::WriteFile(exportFilePath); + REQUIRE(fileWriter.isValid()); + auto writeResult = DREAM3D::WriteFile(fileWriter, CreateFileData()); + SIMPLNX_RESULT_REQUIRE_VALID(writeResult); } + + auto fileReader = HDF5::FileIO::ReadFile(exportFilePath); + auto fileResult = DREAM3D::ReadFile(fileReader); + SIMPLNX_RESULT_REQUIRE_VALID(fileResult); + + auto [pipeline, dataStructure] = fileResult.value(); + REQUIRE(pipeline.size() == CreateExportPipeline().size()); + CheckTestDataStructure(dataStructure); + UnitTest::CheckArraysInheritTupleDims(dataStructure); } TEST_CASE("WriteDREAM3D:Invalid File", "[ReadDREAM3DFilter][WriteDREAM3DFilter]") @@ -964,13 +1071,13 @@ TEST_CASE("WriteDREAM3D:Invalid File", "[ReadDREAM3DFilter][WriteDREAM3DFilter]" bool writeXdmf = GENERATE(true, false); Pipeline exportPipeline = GENERATE(CreateExportPipeline(), Pipeline()); - auto writeResult = DREAM3D::WriteFile(fs::path(), CreateTestDataStructure(), exportPipeline, writeXdmf); + // The write fails before any file (or .xdmf sidecar) can be produced; there is no + // target path whose sidecar could be checked. + DataStructure dataStructure = CreateTestDataStructure(); + auto writeResult = DREAM3D::WriteFile(fs::path(), dataStructure, exportPipeline, writeXdmf); SIMPLNX_RESULT_REQUIRE_INVALID(writeResult); - if(writeXdmf) - { - CheckXdmfFile(); - } + UnitTest::CheckArraysInheritTupleDims(dataStructure); } TEST_CASE("DREAM3DFileTest:DREAM3D File IO Test", "[WriteDREAM3DFilter]") @@ -987,7 +1094,7 @@ TEST_CASE("DREAM3DFileTest:DREAM3D File IO Test", "[WriteDREAM3DFilter]") if(writeXdmf) { - CheckXdmfFile(); + CheckXdmfFile(GetIODataPath()); } } diff --git a/src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md b/src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md index 74fad98955..a0ca60088b 100644 --- a/src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md +++ b/src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md @@ -15,17 +15,17 @@ | Aspect | Current state | |------------------------|---------------| | Algorithm Relationship | **Rewrite** — same UUID/role as legacy `DataContainerWriter`, but the on-disk format is entirely new (v8 `DataStructure` HDF5 layout + `AtomicFile` atomic-write + optional gzip compression), not a translation of the legacy writer's code. | -| Oracle (confirmed) | **Class 1 (Analytical)** — expected content is the hand-built in-memory `DataStructure`/`Pipeline` the test itself constructed; expected HDF5 physical layout (contiguous vs. chunked+deflate) is a closed-form function of array byte-size and the two compression parameters. 17 fixtures across `DREAM3DFileTest.cpp`, all pass. | +| Oracle (confirmed) | **Class 1 (Analytical)** — expected content is the hand-built in-memory `DataStructure`/`Pipeline` the test itself constructed; expected HDF5 physical layout (contiguous vs. chunked+deflate) is a closed-form function of array byte-size and the two compression parameters. 19 fixtures across `DREAM3DFileTest.cpp`, all pass. | | Code paths enumerated | **14 of 19** exercised; 5 gaps are defensive/unreachable-via-public-API guards (see table). | -| Tests today | **17 TEST_CASEs** (some with `GENERATE`/`DYNAMIC_SECTION` multiplying cases) — preflight validation, full round-trip content fidelity across every geometry/DataObject type, SIMPL args backward-compat, and a 6-test compression sub-suite (layout, bypass threshold, level monotonicity). | +| Tests today | **19 TEST_CASEs** (some with `GENERATE`/`DYNAMIC_SECTION` multiplying cases) — preflight validation, full round-trip content fidelity across every geometry/DataObject type, SIMPL args backward-compat, and a 6-test compression sub-suite (layout, bypass threshold, level monotonicity). | | Exemplar archive | **None.** Every test builds its `DataStructure` inline in C++ and round-trips it through `WriteFile`/`ReadFile` in the same run — no cached `.tar.gz` golden file is used or needed for a Class 1 oracle. | | Legacy comparison | **Not run — and not applicable.** The two writers target deliberately different on-disk contracts, so a byte/dataset-level A/B against 6.5.171 `DataContainerWriter` output would be 100% noise by design, not signal. `ReadDREAM3DFilter` is the only tool in either codebase that understands both formats; fidelity is instead verified independently via round-trip Class 1 tests. | -| Bug flags | None. | +| Bug flags | One, found and fixed during this V&V cycle: `WriteXdmfNodeGeometry1D/2D/3D` (`Dream3dIO.cpp`) forwarded to the next-lower writer with `geomName` and `hdf5FilePath` transposed (both `std::string_view`, so it compiled silently), producing `.xdmf` node-attribute references that ParaView/VisIt could not resolve. Fixed alongside a content-level `.xdmf` oracle (`CheckXdmfFile`) that would have caught it. | | V&V phase | Oracle chosen, code paths enumerated, test inventory reviewed, deviations documented. Outstanding: second-engineer review of the oracle design, of the 5 uncovered defensive paths, and of the `DynamicListArray` IO gap (Known limitations). | ## Summary -`WriteDREAM3DFilter` serializes the current `DataStructure` (and, when run inside a pipeline, the preceding `Pipeline`) to an HDF5 `.dream3d` file, with an optional companion `.xdmf` sidecar and optional gzip compression of array datasets. It replaces legacy SIMPL's `DataContainerWriter` under the same conceptual role but with an intentionally new v8 file format, so verification is independent of 6.5.171: correctness is established by writing hand-built `DataStructure`s covering every geometry and `DataObject` type, then reading them back and asserting exact structural/content equality (Class 1 Analytical), plus closed-form assertions on the resulting HDF5 physical layout under each compression setting. All 17 test cases pass; no bugs were found. `StatsDataArray`/`StructArray` (SIMPL's per-ensemble statistics types) are out of scope for this cycle — those `DataObject` types do not yet exist in this branch of simplnx (see deviation D2). Separately, a bare `DynamicListArray` (as opposed to its `NeighborList` specialization) has no HDF5 IO factory at all in the current codebase and cannot be written by this or any filter (see Known limitations). +`WriteDREAM3DFilter` serializes the current `DataStructure` (and, when run inside a pipeline, the preceding `Pipeline`) to an HDF5 `.dream3d` file, with an optional companion `.xdmf` sidecar and optional gzip compression of array datasets. It replaces legacy SIMPL's `DataContainerWriter` under the same conceptual role but with an intentionally new v8 file format, so verification is independent of 6.5.171: correctness is established by writing hand-built `DataStructure`s covering every geometry and `DataObject` type, then reading them back and asserting exact structural/content equality (Class 1 Analytical), plus closed-form assertions on the resulting HDF5 physical layout under each compression setting. All 19 test cases pass. One bug was found and fixed during this cycle: the `.xdmf` node-geometry writers in the shared `Dream3dIO` utility transposed the geometry name and HDF5 file path when forwarding between levels, producing sidecar attribute references ParaView could not resolve (see Bug flags above). `StatsDataArray`/`StructArray` (SIMPL's per-ensemble statistics types) are out of scope for this cycle — those `DataObject` types do not yet exist in this branch of simplnx (see deviation D2). Separately, a bare `DynamicListArray` (as opposed to its `NeighborList` specialization) has no HDF5 IO factory at all in the current codebase and cannot be written by this or any filter (see Known limitations). ## Algorithm Relationship @@ -49,11 +49,12 @@ *Applied:* Every test constructs its expected answer directly, without ever running the filter to "produce" the expected value: -- **Content fidelity:** each test builds a `DataStructure` in C++ (`CreateTestDataStructure()`, or an inline array/geometry), writes it, reads it back, and asserts the read-back content equals what was built — by construction, not by comparison to a previously-captured file. `CheckTestDataStructure()` walks every `DataObject` kind the filter must support (nested `DataGroup`s, `AttributeMatrix`, `NeighborList`, `StringArray`, and all seven geometry types: Vertex/Edge/Triangle/Quad/Tetrahedral/Hexahedral/Image) and asserts exact values against the hand-known fill pattern from `FillDataStore()`. +- **Content fidelity:** each test builds a `DataStructure` in C++ (`CreateTestDataStructure()`, or an inline array/geometry), writes it, reads it back, and asserts the read-back content equals what was built — by construction, not by comparison to a previously-captured file. `CheckTestDataStructure()` walks every `DataObject` kind the filter must support (nested `DataGroup`s, `AttributeMatrix`, `NeighborList`, `StringArray`, and all eight geometry types: Vertex/Edge/Triangle/Quad/Tetrahedral/Hexahedral/Image/RectGrid) and asserts exact values against the hand-known fill pattern from `FillDataStore()`. +- **Xdmf sidecar:** `CheckXdmfFile()` asserts the sidecar exists and that every heavy-data reference in it (`:/DataStructure/` DataItems) resolves to the `.dream3d` file that was written — a content-level check, not an existence-only check. The test fixture includes a vertex `AttributeMatrix` on a node geometry so node-centered attribute references are actually emitted and validated. - **HDF5 physical layout:** the filter's documented compression policy (`docs/WriteDREAM3DFilter.md`) states arrays under 16 KiB always stay contiguous/uncompressed regardless of settings, and any larger array is chunked+deflated at the requested level when compression is enabled. This is a closed-form predicate on `(array byte size, UseCompression, CompressionLevel)` — tests assert it directly via `UnitTest::ProbeHdf5Dataset` rather than trusting the filter's own claim about what it wrote. - **Pipeline embedding:** when run inside an actual `Pipeline`, the written file's embedded pipeline JSON must reproduce the exact filter sequence and count that was executed — asserted against the hand-built pipeline (`pipeline.size() == 3`, filter names checked by index). -**Encoded tests:** `src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp` (17 `TEST_CASE`s touching Write, several parameterized via `GENERATE`/`DYNAMIC_SECTION`) — all pass. See Test inventory below for the full list. +**Encoded tests:** `src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp` (19 `TEST_CASE`s touching Write, several parameterized via `GENERATE`/`DYNAMIC_SECTION`) — all pass. See Test inventory below for the full list. *Second-engineer review:* Outstanding. Recommended focus: the Class 1 boundary-of-scope claim in deviation D2 (StatsDataArray/StructArray) and the 5 uncovered defensive paths (Code path coverage below). @@ -89,10 +90,11 @@ Source: `src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteDREAM3D | Test case | Status | Notes | |-----------|--------|-------| -| `WriteDREAM3DFilter:Invalid Parameters` (§ Empty FilePath, § Bad Compression Level) | kept | Preflight-only, inline `DataStructure`. Covers Paths 1, 2. | -| `WriteDREAM3DFilter:Valid Parameters` | kept | Preflight + full execute, inline `DataStructure`, compression off. Covers Paths 5, 7, 10, 12, 14, 16, 19. | -| `WriteDREAM3D:Pipeline / WriteXdmf combinations` | kept | Calls `DREAM3D::WriteFile(path, ds, pipeline, writeXdmf)` directly (bypasses `AtomicFile`/the Algorithm class) across `writeXdmf × {real pipeline, empty pipeline}` (4 cases). Exercises the shared write utility the Algorithm depends on, not the Algorithm's own guards. | -| `WriteDREAM3D:Invalid File` | kept | Same free-function call with an empty path (4 cases via `GENERATE`); asserts the underlying `HDF5::FileIO::WriteFile` failure is surfaced, not `AtomicFile`'s. | +| `WriteDREAM3DFilter:Invalid Parameters` (§ Empty FilePath, § Bad Compression Level) | kept | Preflight-only, empty `DataStructure` (the guards never touch data). Covers Paths 1, 2. | +| `WriteDREAM3DFilter:Valid Parameters` | kept | Preflight + full execute, full `CreateTestDataStructure()` fixture, compression off, own output file. Covers Paths 5, 7, 10, 12, 14, 16, 19. | +| `WriteDREAM3D:Pipeline / WriteXdmf combinations` | kept | Calls `DREAM3D::WriteFile(path, ds, pipeline, writeXdmf)` directly (bypasses `AtomicFile`/the Algorithm class) across `writeXdmf × {real pipeline, empty pipeline}` (4 cases), own output file. Exercises the shared write utility the Algorithm depends on, not the Algorithm's own guards; validates the `.xdmf` sidecar's heavy-data references via `CheckXdmfFile`. | +| `WriteDREAM3D:FileData Overload` | new | Writes through the exported `DREAM3D::WriteFile(HDF5::FileIO&, const FileData&)` forwarder (via `CreateFileData()`) and round-trips the full fixture — keeps the overload's pipeline/DataStructure argument ordering covered. | +| `WriteDREAM3D:Invalid File` | kept | Same free-function call with an empty path (4 cases via `GENERATE`); asserts the underlying `HDF5::FileIO::WriteFile` failure is surfaced, not `AtomicFile`'s. No sidecar check — a failed write has no target whose sidecar could exist. | | `DREAM3DFileTest:DREAM3D File IO Test` | kept | The primary Class 1 content-fidelity test. Builds every geometry/`DataObject` type (`CreateTestDataStructure`), writes + reads back (`writeXdmf ∈ {true,false}`), and asserts full structural/content equality (`CheckTestDataStructure`) plus pipeline round-trip (`pipeline.size()==3`, filter names by index). | | `DREAM3DFileTest::StringArray` | kept | Round-trips a `StringArray` through the actual `WriteDREAM3DFilter`/`ReadDREAM3DFilter` classes (not the free function) — covers Path 10 with real filter execution. | | `DREAM3DFileTest:Import/Export DREAM3D Filter Test` | kept | Executes `WriteDREAM3DFilter` inside a real `Pipeline` (`exportPipeline.execute()`) — the only test exercising Path 9 (non-null `PipelineNode` with a real preceding pipeline). Also checks preflight-imported vs. executed array store types on the read side. | @@ -106,6 +108,7 @@ Source: `src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteDREAM3D | `WriteDREAM3DFilter: Compression_LevelsRoundTrip` | kept | Full filter execute at levels {1,5,9} on the same 1M-element pattern. Round-trips content at each level and asserts non-increasing file size as level rises (Class 4 companion invariant). | | `WriteDREAM3DFilter: Compression_Preflight_RejectsOutOfRangeLevel` | kept | Three sequential preflight-only checks: level=0 (invalid), level=10 (invalid), level=0 with compression off (valid — ignored). Covers Paths 2, 3, 4. | | `DREAM3DFileTest: PreflightCache avoids re-reading unchanged files` | kept | Uses `DREAM3D::WriteFile` only to create read-side fixture files; does not exercise `WriteDREAM3DFilter`'s own behavior. Listed for completeness since it shares source-file/tag space. | +| `DREAM3DFileTest: Geometry Nested In DataGroup Round Trip` | kept | Executes `WriteDREAM3DFilter` directly across all 8 geometry types, each at top level and nested inside a `DataGroup` (issue #1642 regression coverage), `write_xdmf_file=false`. Covers Paths 7, 10, 14, 16, 19 with per-geometry fixtures. | ## Exemplar archive diff --git a/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.cpp b/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.cpp index 9d4c114b2d..219a60b1db 100644 --- a/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.cpp +++ b/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.cpp @@ -633,7 +633,7 @@ void WriteXdmfNodeGeometry0D(std::ostream& out, const INodeGeometry0D& nodeGeom0 void WriteXdmfNodeGeometry1D(std::ostream& out, const INodeGeometry1D& nodeGeom1D, std::string_view geomName, std::string_view hdf5FilePath) { - WriteXdmfNodeGeometry0D(out, nodeGeom1D, hdf5FilePath, geomName); + WriteXdmfNodeGeometry0D(out, nodeGeom1D, geomName, hdf5FilePath); const AttributeMatrix* edgeData = nodeGeom1D.getEdgeAttributeMatrix(); if(edgeData == nullptr) @@ -645,7 +645,7 @@ void WriteXdmfNodeGeometry1D(std::ostream& out, const INodeGeometry1D& nodeGeom1 void WriteXdmfNodeGeometry2D(std::ostream& out, const INodeGeometry2D& nodeGeom2D, std::string_view geomName, std::string_view hdf5FilePath) { - WriteXdmfNodeGeometry1D(out, nodeGeom2D, hdf5FilePath, geomName); + WriteXdmfNodeGeometry1D(out, nodeGeom2D, geomName, hdf5FilePath); const AttributeMatrix* faceData = nodeGeom2D.getFaceAttributeMatrix(); if(faceData == nullptr) @@ -657,7 +657,7 @@ void WriteXdmfNodeGeometry2D(std::ostream& out, const INodeGeometry2D& nodeGeom2 void WriteXdmfNodeGeometry3D(std::ostream& out, const INodeGeometry3D& nodeGeom3D, std::string_view geomName, std::string_view hdf5FilePath) { - WriteXdmfNodeGeometry2D(out, nodeGeom3D, hdf5FilePath, geomName); + WriteXdmfNodeGeometry2D(out, nodeGeom3D, geomName, hdf5FilePath); const AttributeMatrix* polyhedraData = nodeGeom3D.getPolyhedraAttributeMatrix(); if(polyhedraData == nullptr) From f21d13cbf15b94d2c198309526e84be203120ba1 Mon Sep 17 00:00:00 2001 From: Michael Jackson Date: Wed, 19 Aug 2026 17:07:16 -0400 Subject: [PATCH 6/9] TEST: Cover the write-failure path and record the GridMontage IO gap The V&V report listed Path 13 (DREAM3D::WriteFile returns invalid, so the AtomicFile is never committed) as untested. A DataObject type with no registered HDF5 IO factory is the concrete way to reach it. * Add WriteDREAM3DFilter:Unwritable DataObject Type, which writes a DataStructure holding a GridMontage and asserts preflight still succeeds, execute fails with exactly one error of code -5, and no file is left at the destination * Record GridMontage alongside DynamicListArray under Known limitations: GridMontageIO is fully implemented but never passed to addFactory(), so the type is unwritable even though CreateGridMontageAction is public core API. Note that GridMontageIO::writeData also discards the group it creates and writes into the parent group, so registration and that defect have to be fixed together * Mark Path 13 covered and update the report's counts to 20 TEST_CASEs and 15 of 19 code paths exercised Signed-off-by: Michael Jackson --- .../SimplnxCore/test/DREAM3DFileTest.cpp | 48 +++++++++++++++++++ .../SimplnxCore/vv/WriteDREAM3DFilter.md | 21 ++++---- 2 files changed, 61 insertions(+), 8 deletions(-) diff --git a/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp b/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp index 44099b1f00..d4c0813867 100644 --- a/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp +++ b/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp @@ -23,6 +23,7 @@ #include "simplnx/DataStructure/IDataStore.hpp" #include "simplnx/DataStructure/IO/HDF5/DataStructureReader.hpp" #include "simplnx/DataStructure/ListStore.hpp" +#include "simplnx/DataStructure/Montage/GridMontage.hpp" #include "simplnx/DataStructure/ScalarData.hpp" #include "simplnx/Filter/Arguments.hpp" #include "simplnx/Filter/FilterHandle.hpp" @@ -67,6 +68,7 @@ const fs::path k_MultiExportFilename3 = "multi_export3.dream3d"; const fs::path k_ValidParamsFilename = "write_dream3d_valid_params.dream3d"; const fs::path k_PipelineComboFilename = "write_dream3d_pipeline_combos.dream3d"; const fs::path k_FileDataFilename = "write_dream3d_file_data.dream3d"; +const fs::path k_UnwritableTypeFilename = "write_dream3d_unwritable_type.dream3d"; constexpr StringLiteral k_CellData = "Cell Data"; constexpr StringLiteral k_DataContainer = "Data Container"; @@ -120,6 +122,7 @@ constexpr StringLiteral k_Group3Name = "Third-Level"; constexpr StringLiteral k_AttributeMatrixName = "AttributeMatrix"; constexpr StringLiteral k_ArrayName = "Test-Array"; constexpr StringLiteral k_Array2Name = "Test-Array2"; +constexpr StringLiteral k_GridMontageName = "GridMontage"; constexpr StringLiteral k_CreateDataFilterName = "Create Data Group"; constexpr StringLiteral k_ExportD3DFilterName = "Write DREAM3D-NX File"; @@ -166,6 +169,11 @@ fs::path GetFileDataFilePath() return GetTestFilePath(Constants::k_FileDataFilename); } +fs::path GetUnwritableTypeFilePath() +{ + return GetTestFilePath(Constants::k_UnwritableTypeFilename); +} + fs::path GetXdmfPath(const fs::path& dream3dPath) { fs::path filePath = dream3dPath; @@ -1080,6 +1088,46 @@ TEST_CASE("WriteDREAM3D:Invalid File", "[ReadDREAM3DFilter][WriteDREAM3DFilter]" UnitTest::CheckArraysInheritTupleDims(dataStructure); } +TEST_CASE("WriteDREAM3DFilter:Unwritable DataObject Type", "[ReadDREAM3DFilter][WriteDREAM3DFilter]") +{ + UnitTest::LoadPlugins(); + std::lock_guard lock(m_DataMutex); + + // GridMontage has a fully implemented GridMontageIO class, but that class is never registered + // by HDF5::DataIOManager::addCoreFactories(), so DataStructureWriter cannot resolve a factory + // for it and fails with error -5. This test pins that capability boundary, and covers the + // "DREAM3D::WriteFile returned invalid -> the AtomicFile is never committed" path: the write + // must fail *and* leave no file behind at the destination path. + const fs::path exportFilePath = GetUnwritableTypeFilePath(); + std::error_code removeError; + fs::remove(exportFilePath, removeError); + REQUIRE_FALSE(fs::exists(exportFilePath)); + + DataStructure dataStructure; + REQUIRE(GridMontage::Create(dataStructure, DataNames::k_GridMontageName) != nullptr); + + Arguments args; + WriteDREAM3DFilter filter; + args.insertOrAssign(WriteDREAM3DFilter::k_ExportFilePath, std::make_any(exportFilePath)); + args.insertOrAssign(WriteDREAM3DFilter::k_WriteXdmf, std::make_any(false)); + args.insertOrAssign(WriteDREAM3DFilter::k_UseCompression, std::make_any(false)); + args.insertOrAssign(WriteDREAM3DFilter::k_CompressionLevel, std::make_any(1)); + + // Preflight does not inspect DataObject types, so it must still succeed. + auto preflightResult = filter.preflight(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); + + auto executeResult = filter.execute(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_INVALID(executeResult.result); + REQUIRE(executeResult.result.errors().size() == 1); + REQUIRE(executeResult.result.errors()[0].code == -5); + + // The AtomicFile must not have been committed. + REQUIRE_FALSE(fs::exists(exportFilePath)); + + UnitTest::CheckArraysInheritTupleDims(dataStructure); +} + TEST_CASE("DREAM3DFileTest:DREAM3D File IO Test", "[WriteDREAM3DFilter]") { UnitTest::LoadPlugins(); diff --git a/src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md b/src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md index a0ca60088b..6123f49687 100644 --- a/src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md +++ b/src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md @@ -15,17 +15,17 @@ | Aspect | Current state | |------------------------|---------------| | Algorithm Relationship | **Rewrite** — same UUID/role as legacy `DataContainerWriter`, but the on-disk format is entirely new (v8 `DataStructure` HDF5 layout + `AtomicFile` atomic-write + optional gzip compression), not a translation of the legacy writer's code. | -| Oracle (confirmed) | **Class 1 (Analytical)** — expected content is the hand-built in-memory `DataStructure`/`Pipeline` the test itself constructed; expected HDF5 physical layout (contiguous vs. chunked+deflate) is a closed-form function of array byte-size and the two compression parameters. 19 fixtures across `DREAM3DFileTest.cpp`, all pass. | -| Code paths enumerated | **14 of 19** exercised; 5 gaps are defensive/unreachable-via-public-API guards (see table). | -| Tests today | **19 TEST_CASEs** (some with `GENERATE`/`DYNAMIC_SECTION` multiplying cases) — preflight validation, full round-trip content fidelity across every geometry/DataObject type, SIMPL args backward-compat, and a 6-test compression sub-suite (layout, bypass threshold, level monotonicity). | +| Oracle (confirmed) | **Class 1 (Analytical)** — expected content is the hand-built in-memory `DataStructure`/`Pipeline` the test itself constructed; expected HDF5 physical layout (contiguous vs. chunked+deflate) is a closed-form function of array byte-size and the two compression parameters. 20 fixtures across `DREAM3DFileTest.cpp`, all pass. | +| Code paths enumerated | **15 of 19** exercised; the 4 remaining gaps are defensive/unreachable-via-public-API guards (see table). | +| Tests today | **20 TEST_CASEs** (some with `GENERATE`/`DYNAMIC_SECTION` multiplying cases) — preflight validation, full round-trip content fidelity across every geometry/DataObject type, SIMPL args backward-compat, and a 6-test compression sub-suite (layout, bypass threshold, level monotonicity). | | Exemplar archive | **None.** Every test builds its `DataStructure` inline in C++ and round-trips it through `WriteFile`/`ReadFile` in the same run — no cached `.tar.gz` golden file is used or needed for a Class 1 oracle. | | Legacy comparison | **Not run — and not applicable.** The two writers target deliberately different on-disk contracts, so a byte/dataset-level A/B against 6.5.171 `DataContainerWriter` output would be 100% noise by design, not signal. `ReadDREAM3DFilter` is the only tool in either codebase that understands both formats; fidelity is instead verified independently via round-trip Class 1 tests. | | Bug flags | One, found and fixed during this V&V cycle: `WriteXdmfNodeGeometry1D/2D/3D` (`Dream3dIO.cpp`) forwarded to the next-lower writer with `geomName` and `hdf5FilePath` transposed (both `std::string_view`, so it compiled silently), producing `.xdmf` node-attribute references that ParaView/VisIt could not resolve. Fixed alongside a content-level `.xdmf` oracle (`CheckXdmfFile`) that would have caught it. | -| V&V phase | Oracle chosen, code paths enumerated, test inventory reviewed, deviations documented. Outstanding: second-engineer review of the oracle design, of the 5 uncovered defensive paths, and of the `DynamicListArray` IO gap (Known limitations). | +| V&V phase | Oracle chosen, code paths enumerated, test inventory reviewed, deviations documented. Outstanding: second-engineer review of the oracle design, of the 4 uncovered defensive paths, and of the `DynamicListArray`/`GridMontage` IO gaps (Known limitations). | ## Summary -`WriteDREAM3DFilter` serializes the current `DataStructure` (and, when run inside a pipeline, the preceding `Pipeline`) to an HDF5 `.dream3d` file, with an optional companion `.xdmf` sidecar and optional gzip compression of array datasets. It replaces legacy SIMPL's `DataContainerWriter` under the same conceptual role but with an intentionally new v8 file format, so verification is independent of 6.5.171: correctness is established by writing hand-built `DataStructure`s covering every geometry and `DataObject` type, then reading them back and asserting exact structural/content equality (Class 1 Analytical), plus closed-form assertions on the resulting HDF5 physical layout under each compression setting. All 19 test cases pass. One bug was found and fixed during this cycle: the `.xdmf` node-geometry writers in the shared `Dream3dIO` utility transposed the geometry name and HDF5 file path when forwarding between levels, producing sidecar attribute references ParaView could not resolve (see Bug flags above). `StatsDataArray`/`StructArray` (SIMPL's per-ensemble statistics types) are out of scope for this cycle — those `DataObject` types do not yet exist in this branch of simplnx (see deviation D2). Separately, a bare `DynamicListArray` (as opposed to its `NeighborList` specialization) has no HDF5 IO factory at all in the current codebase and cannot be written by this or any filter (see Known limitations). +`WriteDREAM3DFilter` serializes the current `DataStructure` (and, when run inside a pipeline, the preceding `Pipeline`) to an HDF5 `.dream3d` file, with an optional companion `.xdmf` sidecar and optional gzip compression of array datasets. It replaces legacy SIMPL's `DataContainerWriter` under the same conceptual role but with an intentionally new v8 file format, so verification is independent of 6.5.171: correctness is established by writing hand-built `DataStructure`s covering every geometry and `DataObject` type, then reading them back and asserting exact structural/content equality (Class 1 Analytical), plus closed-form assertions on the resulting HDF5 physical layout under each compression setting. All 20 test cases pass. One bug was found and fixed during this cycle: the `.xdmf` node-geometry writers in the shared `Dream3dIO` utility transposed the geometry name and HDF5 file path when forwarding between levels, producing sidecar attribute references ParaView could not resolve (see Bug flags above). `StatsDataArray`/`StructArray` (SIMPL's per-ensemble statistics types) are out of scope for this cycle — those `DataObject` types do not yet exist in this branch of simplnx (see deviation D2). Separately, two `DataObject` types have no registered HDF5 IO factory and therefore cannot be written by this or any filter: a bare `DynamicListArray` (as opposed to its `NeighborList` specialization), and `GridMontage` — whose IO class exists but is never registered even though `CreateGridMontageAction` is public core API. Both are recorded under Known limitations, and the resulting write-failure contract is now pinned by a dedicated test. ## Algorithm Relationship @@ -78,7 +78,7 @@ Source: `src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteDREAM3D | 10 | Execute — pipeline | `PipelineNode == nullptr` → empty pipeline written | `"WriteDREAM3DFilter:Valid Parameters"`, `"DREAM3DFileTest::StringArray"`, all `Compression_*` tests (all call `filter.execute(ds, args)` directly) | | 11 | Execute — options | `use_compression=true` → `writeOptions.compressionLevel = CompressionLevel` | `"...Compression_On_IsChunkedAndDeflated"`, `"...Compression_SmallArray_Bypasses"`, `"...Compression_LevelsRoundTrip"` | | 12 | Execute — options | `use_compression=false` → `writeOptions.compressionLevel = 0` | `"...Compression_Off_IsContiguous"`; `"WriteDREAM3DFilter:Valid Parameters"` | -| 13 | Execute — write | `DREAM3D::WriteFile(...)` returns invalid → skip commit, return the error | *Not directly tested* through the full filter/`AtomicFile` path, but concretely reachable (not merely defensive): `HDF5::DataStructureWriter::WriteFile` returns error `-5` ("Could not find IO factory for datatype: …") for any `DataObject` type with no registered HDF5 IO factory — see Known limitations below (a bare `DynamicListArray`, not wrapped as `NeighborList`). No test currently puts such an object in the `DataStructure` before writing. | +| 13 | Execute — write | `DREAM3D::WriteFile(...)` returns invalid → skip commit, return the error | `"WriteDREAM3DFilter:Unwritable DataObject Type"` — writes a `DataStructure` holding a `GridMontage`, a type with no registered HDF5 IO factory. Asserts preflight still succeeds, execute fails with exactly one error of code `-5` ("Could not find IO factory for datatype: …"), and the destination file does **not** exist afterward (i.e. the `AtomicFile` was never committed). See Known limitations below. | | 14 | Execute — write | `DREAM3D::WriteFile(...)` returns valid → proceed to commit | Every passing execute-path test | | 15 | Execute — commit | `atomicFile.commit()` fails (rename onto final destination fails) | *Not directly tested.* Would require the destination path to become invalid between `AtomicFile::Create` and `commit()` (e.g., concurrent deletion of the parent directory) — a race not exercised by the suite. | | 16 | Execute — commit | `atomicFile.commit()` succeeds | Every passing execute-path test (the output file is present and re-readable in every round-trip test) | @@ -95,6 +95,7 @@ Source: `src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteDREAM3D | `WriteDREAM3D:Pipeline / WriteXdmf combinations` | kept | Calls `DREAM3D::WriteFile(path, ds, pipeline, writeXdmf)` directly (bypasses `AtomicFile`/the Algorithm class) across `writeXdmf × {real pipeline, empty pipeline}` (4 cases), own output file. Exercises the shared write utility the Algorithm depends on, not the Algorithm's own guards; validates the `.xdmf` sidecar's heavy-data references via `CheckXdmfFile`. | | `WriteDREAM3D:FileData Overload` | new | Writes through the exported `DREAM3D::WriteFile(HDF5::FileIO&, const FileData&)` forwarder (via `CreateFileData()`) and round-trips the full fixture — keeps the overload's pipeline/DataStructure argument ordering covered. | | `WriteDREAM3D:Invalid File` | kept | Same free-function call with an empty path (4 cases via `GENERATE`); asserts the underlying `HDF5::FileIO::WriteFile` failure is surfaced, not `AtomicFile`'s. No sidecar check — a failed write has no target whose sidecar could exist. | +| `WriteDREAM3DFilter:Unwritable DataObject Type` | new for V&V | Full filter execute on a `DataStructure` containing a `GridMontage` (no registered IO factory). Pins the write-failure contract end to end: preflight valid, execute invalid with exactly one error of code `-5`, and no file left at the destination — the only test covering Path 13 and the `AtomicFile`-not-committed guarantee. Also the executable record of the GridMontage IO gap (see Known limitations). | | `DREAM3DFileTest:DREAM3D File IO Test` | kept | The primary Class 1 content-fidelity test. Builds every geometry/`DataObject` type (`CreateTestDataStructure`), writes + reads back (`writeXdmf ∈ {true,false}`), and asserts full structural/content equality (`CheckTestDataStructure`) plus pipeline round-trip (`pipeline.size()==3`, filter names by index). | | `DREAM3DFileTest::StringArray` | kept | Round-trips a `StringArray` through the actual `WriteDREAM3DFilter`/`ReadDREAM3DFilter` classes (not the free function) — covers Path 10 with real filter execution. | | `DREAM3DFileTest:Import/Export DREAM3D Filter Test` | kept | Executes `WriteDREAM3DFilter` inside a real `Pipeline` (`exportPipeline.execute()`) — the only test exercising Path 9 (non-null `PipelineNode` with a real preceding pipeline). Also checks preflight-imported vs. executed array store types on the read side. | @@ -116,9 +117,13 @@ None. Every test above builds its input `DataStructure` inline in C++ and never ## Known limitations (current simplnx HDF5 IO layer) -`DynamicListArray` (the generic variable-length per-tuple list container that `NeighborList` is itself built on) has **no registered HDF5 IO factory**. `HDF5::DataIOManager::addCoreFactories()` (`DataStructure/IO/HDF5/DataIOManager.cpp:33-81`) registers factories per concrete `DataArray` type, per `NeighborList` specialization, geometries, `AttributeMatrix`, `DataGroup`, `StringArray`, and scalar attributes — but nothing for a bare `DynamicListArray`. If one is ever placed directly in a `DataStructure` (outside of a `NeighborList` specialization) and written, `HDF5::DataStructureWriter::WriteFile` hits its "no factory found" guard (`DataStructureWriter.cpp:153-157`) and fails with error `-5` ("Could not find IO factory for datatype: …"), surfacing through `WriteDREAM3DFilter` as Path 13 above. +`HDF5::DataIOManager::addCoreFactories()` (`DataStructure/IO/HDF5/DataIOManager.cpp:33-81`) registers factories per concrete `DataArray` type, per `NeighborList` specialization, per `ScalarData` type, the eight geometries, `AttributeMatrix`, `DataGroup`, and `StringArray`. Two `DataObject` types reachable in the current codebase are **not** registered, and any `DataStructure` containing one cannot be written: `HDF5::DataStructureWriter::WriteFile` hits its "no factory found" guard (`DataStructureWriter.cpp:153-157`) and fails with error `-5` ("Could not find IO factory for datatype: …"), surfacing through `WriteDREAM3DFilter` as Path 13 above. -This is a gap in the shared HDF5 IO layer, not something specific to `WriteDREAM3DFilter`'s own algorithm — the same gap would affect `ReadDREAM3DFilter` for the same object type. It is not raised as a formal Deviation because there is no confirmed 6.5.171 pipeline behavior being compared against (unlike D2, which names concrete legacy types); it is recorded here as a known, currently-untested capability boundary of what this filter can serialize. No current `WriteDREAM3DFilter` test constructs a bare `DynamicListArray`, so Path 13 remains untested rather than confirmed-safe. +1. **`DynamicListArray`** — the generic variable-length per-tuple list container that `NeighborList` is itself built on. It has no IO class at all in `DataStructure/IO/HDF5/`, so nothing could be registered. Only reachable by constructing one directly; no shipped filter creates a bare `DynamicListArray` outside of a `NeighborList` specialization. + +2. **`GridMontage`** — unlike the above, this type *does* have a fully written IO class (`DataStructure/IO/HDF5/GridMontageIO.cpp`, implementing both `readData` and `writeData`), but that class is never passed to `addFactory<>()`, so it is dead code and the type is unwritable in practice. `GridMontage` is more reachable than `DynamicListArray`: `CreateGridMontageAction` (`src/simplnx/Filter/Actions/CreateGridMontageAction.cpp`) is public core API that any filter may use to create one. No filter in this repository currently does, so no shipped pipeline can hit this today — but a plugin filter using that public Action would produce a `DataStructure` that `WriteDREAM3DFilter` silently cannot save. Note also that `GridMontageIO::writeData` creates a `groupWriter` it never uses and then passes `parentGroup` to `WriteBaseGroupData`, so registering it as-is would write montage contents into the parent group; the registration fix and that defect must be addressed together. + +Both are gaps in the shared HDF5 IO layer, not in `WriteDREAM3DFilter`'s own algorithm — the same gaps affect `ReadDREAM3DFilter` for the same types. Neither is raised as a formal Deviation because there is no confirmed 6.5.171 pipeline behavior being compared against (unlike D2, which names concrete legacy types); they are recorded here as known capability boundaries of what this filter can serialize. The failure mode itself is no longer untested: `"WriteDREAM3DFilter:Unwritable DataObject Type"` pins it via `GridMontage`, asserting the `-5` error and that no partial file is committed. ## Deviations from DREAM3D 6.5.171 From b66135adb31d4483f7a7f6781c1eb77176b6076b Mon Sep 17 00:00:00 2001 From: Michael Jackson Date: Wed, 19 Aug 2026 18:07:30 -0400 Subject: [PATCH 7/9] STY: Correct the rationale comment on per-test output filenames The comment justified distinct output filenames as avoiding races under parallel ctest, but ctest is not run with -j in this project: tests share on-disk fixtures and TestFileSentinel archives that do not survive parallel invocation, and the CI test presets run serially. Restate the reason as test independence, which is what the change actually buys. Signed-off-by: Michael Jackson --- src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp b/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp index d4c0813867..a13d3a435d 100644 --- a/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp +++ b/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp @@ -63,8 +63,9 @@ const fs::path k_ExportFilename2 = "export2.dream3d"; const fs::path k_MultiExportFilename1 = "multi_export1.dream3d"; const fs::path k_MultiExportFilename2 = "multi_export2.dream3d"; const fs::path k_MultiExportFilename3 = "multi_export3.dream3d"; -// Each TEST_CASE runs as a separate ctest process, so tests that write a file must each use -// their own filename to avoid cross-process races under parallel ctest. +// Each TEST_CASE runs as a separate ctest process. Giving every file-writing case its own +// output filename keeps them independent, so one case can never observe or clobber a file +// another one wrote, and each can assert on its own output in isolation. const fs::path k_ValidParamsFilename = "write_dream3d_valid_params.dream3d"; const fs::path k_PipelineComboFilename = "write_dream3d_pipeline_combos.dream3d"; const fs::path k_FileDataFilename = "write_dream3d_file_data.dream3d"; From 72578ab0f386b57fc90e1d7def33f29ed42c009e Mon Sep 17 00:00:00 2001 From: Michael Jackson Date: Wed, 19 Aug 2026 18:17:31 -0400 Subject: [PATCH 8/9] DOC: Scope the GridMontage observation to current behavior only Montage support in SIMPLNX is an open design question. The previous wording implied the remedy was to register GridMontageIO and repair its writeData, which presumes a design decision that has not been made. * Record GridMontage as an observation about current behavior, not a defect with a known fix, and note that the existing GridMontage / GridMontageIO / CreateGridMontageAction code is an unfinished sketch inherited from the legacy SIMPL montage design rather than a foundation to switch on * State that the on-disk representation, and whether GridMontage is the right in-memory abstraction, need designing before the code is revived * Reframe the test comment so the write-failure contract is clearly the subject and GridMontage only the vehicle, with a note to re-point the test at another unwritable type if montages later become writable Signed-off-by: Michael Jackson --- src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp | 15 ++++++++++----- src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md | 10 ++++++---- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp b/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp index a13d3a435d..b1e5b93f41 100644 --- a/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp +++ b/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp @@ -1094,11 +1094,16 @@ TEST_CASE("WriteDREAM3DFilter:Unwritable DataObject Type", "[ReadDREAM3DFilter][ UnitTest::LoadPlugins(); std::lock_guard lock(m_DataMutex); - // GridMontage has a fully implemented GridMontageIO class, but that class is never registered - // by HDF5::DataIOManager::addCoreFactories(), so DataStructureWriter cannot resolve a factory - // for it and fails with error -5. This test pins that capability boundary, and covers the - // "DREAM3D::WriteFile returned invalid -> the AtomicFile is never committed" path: the write - // must fail *and* leave no file behind at the destination path. + // The subject of this test is the write-failure contract, not GridMontage: when + // DREAM3D::WriteFile returns invalid, the AtomicFile must never be committed, so the write + // fails *and* leaves no file behind at the destination path. + // + // GridMontage is merely a convenient way to reach that path. Its GridMontageIO class is never + // registered by HDF5::DataIOManager::addCoreFactories(), so DataStructureWriter cannot resolve + // a factory for it and fails with error -5. That is current behavior only: montage support in + // SIMPLNX is an open design question, and this test deliberately makes no claim about how + // montages ought to behave. If montages later become writable, re-point this test at another + // unwritable type rather than deleting it -- the write-failure contract still needs coverage. const fs::path exportFilePath = GetUnwritableTypeFilePath(); std::error_code removeError; fs::remove(exportFilePath, removeError); diff --git a/src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md b/src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md index 6123f49687..f939c0b91b 100644 --- a/src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md +++ b/src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md @@ -21,11 +21,11 @@ | Exemplar archive | **None.** Every test builds its `DataStructure` inline in C++ and round-trips it through `WriteFile`/`ReadFile` in the same run — no cached `.tar.gz` golden file is used or needed for a Class 1 oracle. | | Legacy comparison | **Not run — and not applicable.** The two writers target deliberately different on-disk contracts, so a byte/dataset-level A/B against 6.5.171 `DataContainerWriter` output would be 100% noise by design, not signal. `ReadDREAM3DFilter` is the only tool in either codebase that understands both formats; fidelity is instead verified independently via round-trip Class 1 tests. | | Bug flags | One, found and fixed during this V&V cycle: `WriteXdmfNodeGeometry1D/2D/3D` (`Dream3dIO.cpp`) forwarded to the next-lower writer with `geomName` and `hdf5FilePath` transposed (both `std::string_view`, so it compiled silently), producing `.xdmf` node-attribute references that ParaView/VisIt could not resolve. Fixed alongside a content-level `.xdmf` oracle (`CheckXdmfFile`) that would have caught it. | -| V&V phase | Oracle chosen, code paths enumerated, test inventory reviewed, deviations documented. Outstanding: second-engineer review of the oracle design, of the 4 uncovered defensive paths, and of the `DynamicListArray`/`GridMontage` IO gaps (Known limitations). | +| V&V phase | Oracle chosen, code paths enumerated, test inventory reviewed, deviations documented. Outstanding: second-engineer review of the oracle design, of the 4 uncovered defensive paths, and of the `DynamicListArray`/`GridMontage` serialization boundaries (Known limitations). Montage support is an open design question and is explicitly out of scope for this cycle. | ## Summary -`WriteDREAM3DFilter` serializes the current `DataStructure` (and, when run inside a pipeline, the preceding `Pipeline`) to an HDF5 `.dream3d` file, with an optional companion `.xdmf` sidecar and optional gzip compression of array datasets. It replaces legacy SIMPL's `DataContainerWriter` under the same conceptual role but with an intentionally new v8 file format, so verification is independent of 6.5.171: correctness is established by writing hand-built `DataStructure`s covering every geometry and `DataObject` type, then reading them back and asserting exact structural/content equality (Class 1 Analytical), plus closed-form assertions on the resulting HDF5 physical layout under each compression setting. All 20 test cases pass. One bug was found and fixed during this cycle: the `.xdmf` node-geometry writers in the shared `Dream3dIO` utility transposed the geometry name and HDF5 file path when forwarding between levels, producing sidecar attribute references ParaView could not resolve (see Bug flags above). `StatsDataArray`/`StructArray` (SIMPL's per-ensemble statistics types) are out of scope for this cycle — those `DataObject` types do not yet exist in this branch of simplnx (see deviation D2). Separately, two `DataObject` types have no registered HDF5 IO factory and therefore cannot be written by this or any filter: a bare `DynamicListArray` (as opposed to its `NeighborList` specialization), and `GridMontage` — whose IO class exists but is never registered even though `CreateGridMontageAction` is public core API. Both are recorded under Known limitations, and the resulting write-failure contract is now pinned by a dedicated test. +`WriteDREAM3DFilter` serializes the current `DataStructure` (and, when run inside a pipeline, the preceding `Pipeline`) to an HDF5 `.dream3d` file, with an optional companion `.xdmf` sidecar and optional gzip compression of array datasets. It replaces legacy SIMPL's `DataContainerWriter` under the same conceptual role but with an intentionally new v8 file format, so verification is independent of 6.5.171: correctness is established by writing hand-built `DataStructure`s covering every geometry and `DataObject` type, then reading them back and asserting exact structural/content equality (Class 1 Analytical), plus closed-form assertions on the resulting HDF5 physical layout under each compression setting. All 20 test cases pass. One bug was found and fixed during this cycle: the `.xdmf` node-geometry writers in the shared `Dream3dIO` utility transposed the geometry name and HDF5 file path when forwarding between levels, producing sidecar attribute references ParaView could not resolve (see Bug flags above). `StatsDataArray`/`StructArray` (SIMPL's per-ensemble statistics types) are out of scope for this cycle — those `DataObject` types do not yet exist in this branch of simplnx (see deviation D2). Separately, two `DataObject` types have no registered HDF5 IO factory and therefore cannot be written by this or any filter: a bare `DynamicListArray` (as opposed to its `NeighborList` specialization), and `GridMontage` — whose IO class exists but is never registered even though `CreateGridMontageAction` is public core API. Both are recorded under Known limitations as observations about current behavior; neither is presented as a defect with a known fix, and montage support in particular is an unsettled design question this report takes no position on. The resulting write-failure contract is pinned by a dedicated test. ## Algorithm Relationship @@ -95,7 +95,7 @@ Source: `src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteDREAM3D | `WriteDREAM3D:Pipeline / WriteXdmf combinations` | kept | Calls `DREAM3D::WriteFile(path, ds, pipeline, writeXdmf)` directly (bypasses `AtomicFile`/the Algorithm class) across `writeXdmf × {real pipeline, empty pipeline}` (4 cases), own output file. Exercises the shared write utility the Algorithm depends on, not the Algorithm's own guards; validates the `.xdmf` sidecar's heavy-data references via `CheckXdmfFile`. | | `WriteDREAM3D:FileData Overload` | new | Writes through the exported `DREAM3D::WriteFile(HDF5::FileIO&, const FileData&)` forwarder (via `CreateFileData()`) and round-trips the full fixture — keeps the overload's pipeline/DataStructure argument ordering covered. | | `WriteDREAM3D:Invalid File` | kept | Same free-function call with an empty path (4 cases via `GENERATE`); asserts the underlying `HDF5::FileIO::WriteFile` failure is surfaced, not `AtomicFile`'s. No sidecar check — a failed write has no target whose sidecar could exist. | -| `WriteDREAM3DFilter:Unwritable DataObject Type` | new for V&V | Full filter execute on a `DataStructure` containing a `GridMontage` (no registered IO factory). Pins the write-failure contract end to end: preflight valid, execute invalid with exactly one error of code `-5`, and no file left at the destination — the only test covering Path 13 and the `AtomicFile`-not-committed guarantee. Also the executable record of the GridMontage IO gap (see Known limitations). | +| `WriteDREAM3DFilter:Unwritable DataObject Type` | new for V&V | Full filter execute on a `DataStructure` containing a `GridMontage` (no registered IO factory). Pins the write-failure contract end to end: preflight valid, execute invalid with exactly one error of code `-5`, and no file left at the destination — the only test covering Path 13 and the `AtomicFile`-not-committed guarantee. GridMontage is only the vehicle for reaching that path, not the subject of the test (see Known limitations). | | `DREAM3DFileTest:DREAM3D File IO Test` | kept | The primary Class 1 content-fidelity test. Builds every geometry/`DataObject` type (`CreateTestDataStructure`), writes + reads back (`writeXdmf ∈ {true,false}`), and asserts full structural/content equality (`CheckTestDataStructure`) plus pipeline round-trip (`pipeline.size()==3`, filter names by index). | | `DREAM3DFileTest::StringArray` | kept | Round-trips a `StringArray` through the actual `WriteDREAM3DFilter`/`ReadDREAM3DFilter` classes (not the free function) — covers Path 10 with real filter execution. | | `DREAM3DFileTest:Import/Export DREAM3D Filter Test` | kept | Executes `WriteDREAM3DFilter` inside a real `Pipeline` (`exportPipeline.execute()`) — the only test exercising Path 9 (non-null `PipelineNode` with a real preceding pipeline). Also checks preflight-imported vs. executed array store types on the read side. | @@ -121,7 +121,9 @@ None. Every test above builds its input `DataStructure` inline in C++ and never 1. **`DynamicListArray`** — the generic variable-length per-tuple list container that `NeighborList` is itself built on. It has no IO class at all in `DataStructure/IO/HDF5/`, so nothing could be registered. Only reachable by constructing one directly; no shipped filter creates a bare `DynamicListArray` outside of a `NeighborList` specialization. -2. **`GridMontage`** — unlike the above, this type *does* have a fully written IO class (`DataStructure/IO/HDF5/GridMontageIO.cpp`, implementing both `readData` and `writeData`), but that class is never passed to `addFactory<>()`, so it is dead code and the type is unwritable in practice. `GridMontage` is more reachable than `DynamicListArray`: `CreateGridMontageAction` (`src/simplnx/Filter/Actions/CreateGridMontageAction.cpp`) is public core API that any filter may use to create one. No filter in this repository currently does, so no shipped pipeline can hit this today — but a plugin filter using that public Action would produce a `DataStructure` that `WriteDREAM3DFilter` silently cannot save. Note also that `GridMontageIO::writeData` creates a `groupWriter` it never uses and then passes `parentGroup` to `WriteBaseGroupData`, so registering it as-is would write montage contents into the parent group; the registration fix and that defect must be addressed together. +2. **`GridMontage`** — unlike the above, this type *does* have an IO class (`DataStructure/IO/HDF5/GridMontageIO.cpp`, with both `readData` and `writeData` bodies), but that class is never passed to `addFactory<>()`, so it is unreachable and the type is unwritable in practice. `GridMontage` is more reachable than `DynamicListArray`: `CreateGridMontageAction` (`src/simplnx/Filter/Actions/CreateGridMontageAction.cpp`) is public core API that any filter may use to create one. No filter in this repository currently does, so no shipped pipeline can hit this today — but a plugin filter using that public Action would produce a `DataStructure` that `WriteDREAM3DFilter` cannot save. + + **This is recorded as an observation, not as a defect with a known fix.** Montage support in SIMPLNX is an open design question that has deliberately not been settled yet, and the existing `GridMontage`/`GridMontageIO`/`CreateGridMontageAction` code predates that decision — it should be treated as an unfinished sketch inherited from the legacy SIMPL montage design, not as a foundation that merely needs switching on. Simply calling `addFactory()` would *not* be a correct fix: `GridMontageIO::writeData` already creates a `groupWriter` it never uses and then passes `parentGroup` to `WriteBaseGroupData`, so registering it as-is would write montage contents into the wrong group. How montages should be represented on disk (and whether `GridMontage` is even the right in-memory abstraction) needs to be designed before any of this code is revived. This V&V cycle's only claim is the one its test makes: **today**, a `DataStructure` containing a `GridMontage` fails to write with error `-5` and leaves no partial file behind. Both are gaps in the shared HDF5 IO layer, not in `WriteDREAM3DFilter`'s own algorithm — the same gaps affect `ReadDREAM3DFilter` for the same types. Neither is raised as a formal Deviation because there is no confirmed 6.5.171 pipeline behavior being compared against (unlike D2, which names concrete legacy types); they are recorded here as known capability boundaries of what this filter can serialize. The failure mode itself is no longer untested: `"WriteDREAM3DFilter:Unwritable DataObject Type"` pins it via `GridMontage`, asserting the `-5` error and that no partial file is committed. From 2ffa506ed610417acffee9480837c0d5fe143268 Mon Sep 17 00:00:00 2001 From: Michael Jackson Date: Thu, 20 Aug 2026 10:18:14 -0400 Subject: [PATCH 9/9] DOC: Second-engineer sign-off and template conformance for the WriteDREAM3DFilter V&V report Records the second-engineer review as complete and brings the report into strict conformance with docs/vv_templates. Sign-off: * Status COMPLETE - 2026-08-20; Sign-off names the V&V author and the second engineer with the PR under which the review was performed. * The Oracle section's second-engineer entry replaces "Outstanding" with what the review actually covered across its three passes: oracle design and non-circularity, the Class 4 companion invariant, the D2 scope boundary confirmed by inspection, the four unreachable defensive paths, and the two bugs found and fixed along the way. Template conformance: * Folded the non-template "Known limitations" section into Code path coverage, where Path 13 already describes the failure mode it documents. The report now carries exactly the eight template sections in template order. * Test inventory uses only the canonical kept / new-for-V&V / retired status values. * Reconciled counts that disagreed with each other and with the source: the Oracle section claimed 19 Write-related TEST_CASEs against an inventory of 20, and the second-engineer note claimed 5 uncovered defensive paths against a table showing 4. * Named the three ReadDREAM3DFilter-only TEST_CASEs that are deliberately excluded from this filter's inventory, so the exclusion is auditable rather than a silent omission from a shared test file. Dual-build verification, which the Test inventory gate requires: the DREAM3D file IO tests pass 32/32 in both the in-core and out-of-core Release builds, and the full SimplnxCore:: suite passes 979/979 in-core, at the rebased head. Co-Authored-By: Claude Opus 5 --- .../SimplnxCore/vv/WriteDREAM3DFilter.md | 57 +++++++++++-------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md b/src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md index f939c0b91b..d1c963e035 100644 --- a/src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md +++ b/src/Plugins/SimplnxCore/vv/WriteDREAM3DFilter.md @@ -7,25 +7,25 @@ | SIMPLNX Human Name | Write DREAM3D-NX File | | DREAM3D 6.5.171 equivalent | `DataContainerWriter` — SIMPL UUID `3fcd4c43-9d75-5b86-aad4-4441bc914f37` | | Verified commit | ** | -| Status | **READY FOR REVIEW** (second-engineer review outstanding — see V&V phase) | -| Sign-off | *pending second-engineer review* | +| Status | COMPLETE — 2026-08-20 | +| Sign-off | Matthew Marine (V&V author, PR #1683). Second engineer: Michael A. Jackson , 2026-08-20 (PR #1683 review). | ## At a glance | Aspect | Current state | |------------------------|---------------| | Algorithm Relationship | **Rewrite** — same UUID/role as legacy `DataContainerWriter`, but the on-disk format is entirely new (v8 `DataStructure` HDF5 layout + `AtomicFile` atomic-write + optional gzip compression), not a translation of the legacy writer's code. | -| Oracle (confirmed) | **Class 1 (Analytical)** — expected content is the hand-built in-memory `DataStructure`/`Pipeline` the test itself constructed; expected HDF5 physical layout (contiguous vs. chunked+deflate) is a closed-form function of array byte-size and the two compression parameters. 20 fixtures across `DREAM3DFileTest.cpp`, all pass. | +| Oracle (confirmed) | **Class 1 (Analytical)** — expected content is the hand-built in-memory `DataStructure`/`Pipeline` the test itself constructed; expected HDF5 physical layout (contiguous vs. chunked+deflate) is a closed-form function of array byte-size and the two compression parameters. 20 Write-related fixtures across `DREAM3DFileTest.cpp`, all pass. | | Code paths enumerated | **15 of 19** exercised; the 4 remaining gaps are defensive/unreachable-via-public-API guards (see table). | -| Tests today | **20 TEST_CASEs** (some with `GENERATE`/`DYNAMIC_SECTION` multiplying cases) — preflight validation, full round-trip content fidelity across every geometry/DataObject type, SIMPL args backward-compat, and a 6-test compression sub-suite (layout, bypass threshold, level monotonicity). | +| Tests today | **20 Write-related `TEST_CASE`s of the 23 in `DREAM3DFileTest.cpp` / 30 ctest entries**, all passing (some with `GENERATE`/`DYNAMIC_SECTION` multiplying cases) — preflight validation, full round-trip content fidelity across every geometry/DataObject type, SIMPL args backward-compat, and a 6-test compression sub-suite (layout, bypass threshold, level monotonicity). | | Exemplar archive | **None.** Every test builds its `DataStructure` inline in C++ and round-trips it through `WriteFile`/`ReadFile` in the same run — no cached `.tar.gz` golden file is used or needed for a Class 1 oracle. | | Legacy comparison | **Not run — and not applicable.** The two writers target deliberately different on-disk contracts, so a byte/dataset-level A/B against 6.5.171 `DataContainerWriter` output would be 100% noise by design, not signal. `ReadDREAM3DFilter` is the only tool in either codebase that understands both formats; fidelity is instead verified independently via round-trip Class 1 tests. | | Bug flags | One, found and fixed during this V&V cycle: `WriteXdmfNodeGeometry1D/2D/3D` (`Dream3dIO.cpp`) forwarded to the next-lower writer with `geomName` and `hdf5FilePath` transposed (both `std::string_view`, so it compiled silently), producing `.xdmf` node-attribute references that ParaView/VisIt could not resolve. Fixed alongside a content-level `.xdmf` oracle (`CheckXdmfFile`) that would have caught it. | -| V&V phase | Oracle chosen, code paths enumerated, test inventory reviewed, deviations documented. Outstanding: second-engineer review of the oracle design, of the 4 uncovered defensive paths, and of the `DynamicListArray`/`GridMontage` serialization boundaries (Known limitations). Montage support is an open design question and is explicitly out of scope for this cycle. | +| V&V phase | Discovery, algorithm relationship, oracle design, code-path enumeration, test inventory, deviations, and the bug fixes found along the way — **complete**. Second-engineer review of the oracle design, the 4 uncovered defensive paths, and the `DynamicListArray`/`GridMontage` serialization boundaries **signed off by Michael A. Jackson, 2026-08-20** (PR #1683). No legacy A/B is applicable (see Legacy comparison). Montage support remains an open design question, explicitly out of scope for this cycle and recorded as a capability boundary rather than a defect. **Nothing outstanding.** | ## Summary -`WriteDREAM3DFilter` serializes the current `DataStructure` (and, when run inside a pipeline, the preceding `Pipeline`) to an HDF5 `.dream3d` file, with an optional companion `.xdmf` sidecar and optional gzip compression of array datasets. It replaces legacy SIMPL's `DataContainerWriter` under the same conceptual role but with an intentionally new v8 file format, so verification is independent of 6.5.171: correctness is established by writing hand-built `DataStructure`s covering every geometry and `DataObject` type, then reading them back and asserting exact structural/content equality (Class 1 Analytical), plus closed-form assertions on the resulting HDF5 physical layout under each compression setting. All 20 test cases pass. One bug was found and fixed during this cycle: the `.xdmf` node-geometry writers in the shared `Dream3dIO` utility transposed the geometry name and HDF5 file path when forwarding between levels, producing sidecar attribute references ParaView could not resolve (see Bug flags above). `StatsDataArray`/`StructArray` (SIMPL's per-ensemble statistics types) are out of scope for this cycle — those `DataObject` types do not yet exist in this branch of simplnx (see deviation D2). Separately, two `DataObject` types have no registered HDF5 IO factory and therefore cannot be written by this or any filter: a bare `DynamicListArray` (as opposed to its `NeighborList` specialization), and `GridMontage` — whose IO class exists but is never registered even though `CreateGridMontageAction` is public core API. Both are recorded under Known limitations as observations about current behavior; neither is presented as a defect with a known fix, and montage support in particular is an unsettled design question this report takes no position on. The resulting write-failure contract is pinned by a dedicated test. +`WriteDREAM3DFilter` serializes the current `DataStructure` (and, when run inside a pipeline, the preceding `Pipeline`) to an HDF5 `.dream3d` file, with an optional companion `.xdmf` sidecar and optional gzip compression of array datasets. It replaces legacy SIMPL's `DataContainerWriter` under the same conceptual role but with an intentionally new v8 file format, so verification is independent of 6.5.171: correctness is established by writing hand-built `DataStructure`s covering every geometry and `DataObject` type, then reading them back and asserting exact structural/content equality (Class 1 Analytical), plus closed-form assertions on the resulting HDF5 physical layout under each compression setting. All 20 test cases pass. One bug was found and fixed during this cycle: the `.xdmf` node-geometry writers in the shared `Dream3dIO` utility transposed the geometry name and HDF5 file path when forwarding between levels, producing sidecar attribute references ParaView could not resolve (see Bug flags above). `StatsDataArray`/`StructArray` (SIMPL's per-ensemble statistics types) are out of scope for this cycle — those `DataObject` types do not yet exist in this branch of simplnx (see deviation D2). Separately, two `DataObject` types have no registered HDF5 IO factory and therefore cannot be written by this or any filter: a bare `DynamicListArray` (as opposed to its `NeighborList` specialization), and `GridMontage` — whose IO class exists but is never registered even though `CreateGridMontageAction` is public core API. Both are recorded under Code path coverage as observations about current behavior; neither is presented as a defect with a known fix, and montage support in particular is an unsettled design question this report takes no position on. The resulting write-failure contract is pinned by a dedicated test. ## Algorithm Relationship @@ -54,9 +54,15 @@ - **HDF5 physical layout:** the filter's documented compression policy (`docs/WriteDREAM3DFilter.md`) states arrays under 16 KiB always stay contiguous/uncompressed regardless of settings, and any larger array is chunked+deflated at the requested level when compression is enabled. This is a closed-form predicate on `(array byte size, UseCompression, CompressionLevel)` — tests assert it directly via `UnitTest::ProbeHdf5Dataset` rather than trusting the filter's own claim about what it wrote. - **Pipeline embedding:** when run inside an actual `Pipeline`, the written file's embedded pipeline JSON must reproduce the exact filter sequence and count that was executed — asserted against the hand-built pipeline (`pipeline.size() == 3`, filter names checked by index). -**Encoded tests:** `src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp` (19 `TEST_CASE`s touching Write, several parameterized via `GENERATE`/`DYNAMIC_SECTION`) — all pass. See Test inventory below for the full list. +**Encoded tests:** `src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp` (20 of the file's 23 `TEST_CASE`s touch Write, several parameterized via `GENERATE`/`DYNAMIC_SECTION`) — all pass. See Test inventory below for the full list and for the three `ReadDREAM3DFilter`-only cases that are deliberately out of scope. -*Second-engineer review:* Outstanding. Recommended focus: the Class 1 boundary-of-scope claim in deviation D2 (StatsDataArray/StructArray) and the 5 uncovered defensive paths (Code path coverage below). +*Second-engineer review:* **Signed off by Michael A. Jackson , 2026-08-20** (PR #1683 review, per sign-off convention). The V&V work was authored by Matthew Marine, so the review is independent of the author. Reviewed across three passes (2026-07-23, 2026-08-12, and this closing pass): + +- **Oracle design.** Confirmed Class 1 is correct and non-circular: every fixture builds its own `DataStructure` in C++ and asserts the read-back against the hand-known fill pattern, so no previously-captured file is ever the source of truth. The HDF5 physical-layout assertions are a genuine closed-form predicate on `(byte size, UseCompression, CompressionLevel)` checked via `ProbeHdf5Dataset`, not a restatement of what the filter reported writing. +- **Class 4 companion.** The file-size monotonicity invariant in `Compression_LevelsRoundTrip` is a legitimate invariant check, not a substitute for the per-level content round-trip that runs alongside it. +- **Scope boundary (D2).** Confirmed by inspection that `StatsDataArray`/`StructArray` do not exist in this branch, so the exclusion is a real scope boundary rather than an untested path. +- **The 4 uncovered defensive paths.** Each was independently confirmed unreachable through the public API, and each appears as its own row in the code-path table rather than being omitted. +- **Bugs found during review.** Three transposed `std::string_view` arguments in `WriteXdmfNodeGeometry1D/2D/3D` and a wrong-`Result` test in `DREAM3D::ReadFile` were found and fixed during these passes; both are recorded under Bug flags with the tests that now pin them. ## Code path coverage @@ -78,7 +84,7 @@ Source: `src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteDREAM3D | 10 | Execute — pipeline | `PipelineNode == nullptr` → empty pipeline written | `"WriteDREAM3DFilter:Valid Parameters"`, `"DREAM3DFileTest::StringArray"`, all `Compression_*` tests (all call `filter.execute(ds, args)` directly) | | 11 | Execute — options | `use_compression=true` → `writeOptions.compressionLevel = CompressionLevel` | `"...Compression_On_IsChunkedAndDeflated"`, `"...Compression_SmallArray_Bypasses"`, `"...Compression_LevelsRoundTrip"` | | 12 | Execute — options | `use_compression=false` → `writeOptions.compressionLevel = 0` | `"...Compression_Off_IsContiguous"`; `"WriteDREAM3DFilter:Valid Parameters"` | -| 13 | Execute — write | `DREAM3D::WriteFile(...)` returns invalid → skip commit, return the error | `"WriteDREAM3DFilter:Unwritable DataObject Type"` — writes a `DataStructure` holding a `GridMontage`, a type with no registered HDF5 IO factory. Asserts preflight still succeeds, execute fails with exactly one error of code `-5` ("Could not find IO factory for datatype: …"), and the destination file does **not** exist afterward (i.e. the `AtomicFile` was never committed). See Known limitations below. | +| 13 | Execute — write | `DREAM3D::WriteFile(...)` returns invalid → skip commit, return the error | `"WriteDREAM3DFilter:Unwritable DataObject Type"` — writes a `DataStructure` holding a `GridMontage`, a type with no registered HDF5 IO factory. Asserts preflight still succeeds, execute fails with exactly one error of code `-5` ("Could not find IO factory for datatype: …"), and the destination file does **not** exist afterward (i.e. the `AtomicFile` was never committed). See the capability-boundary note at the end of this section. | | 14 | Execute — write | `DREAM3D::WriteFile(...)` returns valid → proceed to commit | Every passing execute-path test | | 15 | Execute — commit | `atomicFile.commit()` fails (rename onto final destination fails) | *Not directly tested.* Would require the destination path to become invalid between `AtomicFile::Create` and `commit()` (e.g., concurrent deletion of the parent directory) — a race not exercised by the suite. | | 16 | Execute — commit | `atomicFile.commit()` succeeds | Every passing execute-path test (the output file is present and re-readable in every round-trip test) | @@ -86,6 +92,19 @@ Source: `src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteDREAM3D | 18 | Execute — xdmf | `write_xdmf_file=true`, rename fails → `MakeErrorResult` with system error message | *Not directly tested.* Would require the `.xdmf` destination to become unwritable between the HDF5 write succeeding and the rename — not portably reproducible in the current suite. | | 19 | Execute — xdmf | `write_xdmf_file=false` → skip rename, return `WriteFile`'s result directly | Most `Compression_*` tests, `"DREAM3DFileTest::StringArray"`, `"WriteDREAM3DFilter:Valid Parameters"` | + +**Capability boundary behind Path 13 — `DataObject` types the shared HDF5 IO layer cannot write.** + +`HDF5::DataIOManager::addCoreFactories()` (`DataStructure/IO/HDF5/DataIOManager.cpp:33-81`) registers factories per concrete `DataArray` type, per `NeighborList` specialization, per `ScalarData` type, the eight geometries, `AttributeMatrix`, `DataGroup`, and `StringArray`. Two `DataObject` types reachable in the current codebase are **not** registered, and any `DataStructure` containing one cannot be written: `HDF5::DataStructureWriter::WriteFile` hits its "no factory found" guard (`DataStructureWriter.cpp:153-157`) and fails with error `-5` ("Could not find IO factory for datatype: …"), surfacing through `WriteDREAM3DFilter` as Path 13 above. + +1. **`DynamicListArray`** — the generic variable-length per-tuple list container that `NeighborList` is itself built on. It has no IO class at all in `DataStructure/IO/HDF5/`, so nothing could be registered. Only reachable by constructing one directly; no shipped filter creates a bare `DynamicListArray` outside of a `NeighborList` specialization. + +2. **`GridMontage`** — unlike the above, this type *does* have an IO class (`DataStructure/IO/HDF5/GridMontageIO.cpp`, with both `readData` and `writeData` bodies), but that class is never passed to `addFactory<>()`, so it is unreachable and the type is unwritable in practice. `GridMontage` is more reachable than `DynamicListArray`: `CreateGridMontageAction` (`src/simplnx/Filter/Actions/CreateGridMontageAction.cpp`) is public core API that any filter may use to create one. No filter in this repository currently does, so no shipped pipeline can hit this today — but a plugin filter using that public Action would produce a `DataStructure` that `WriteDREAM3DFilter` cannot save. + + **This is recorded as an observation, not as a defect with a known fix.** Montage support in SIMPLNX is an open design question that has deliberately not been settled yet, and the existing `GridMontage`/`GridMontageIO`/`CreateGridMontageAction` code predates that decision — it should be treated as an unfinished sketch inherited from the legacy SIMPL montage design, not as a foundation that merely needs switching on. Simply calling `addFactory()` would *not* be a correct fix: `GridMontageIO::writeData` already creates a `groupWriter` it never uses and then passes `parentGroup` to `WriteBaseGroupData`, so registering it as-is would write montage contents into the wrong group. How montages should be represented on disk (and whether `GridMontage` is even the right in-memory abstraction) needs to be designed before any of this code is revived. This V&V cycle's only claim is the one its test makes: **today**, a `DataStructure` containing a `GridMontage` fails to write with error `-5` and leaves no partial file behind. + +Both are gaps in the shared HDF5 IO layer, not in `WriteDREAM3DFilter`'s own algorithm — the same gaps affect `ReadDREAM3DFilter` for the same types. Neither is raised as a formal Deviation because there is no confirmed 6.5.171 pipeline behavior being compared against (unlike D2, which names concrete legacy types); they are recorded here as known capability boundaries of what this filter can serialize. The failure mode itself is no longer untested: `"WriteDREAM3DFilter:Unwritable DataObject Type"` pins it via `GridMontage`, asserting the `-5` error and that no partial file is committed. + ## Test inventory | Test case | Status | Notes | @@ -93,9 +112,9 @@ Source: `src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteDREAM3D | `WriteDREAM3DFilter:Invalid Parameters` (§ Empty FilePath, § Bad Compression Level) | kept | Preflight-only, empty `DataStructure` (the guards never touch data). Covers Paths 1, 2. | | `WriteDREAM3DFilter:Valid Parameters` | kept | Preflight + full execute, full `CreateTestDataStructure()` fixture, compression off, own output file. Covers Paths 5, 7, 10, 12, 14, 16, 19. | | `WriteDREAM3D:Pipeline / WriteXdmf combinations` | kept | Calls `DREAM3D::WriteFile(path, ds, pipeline, writeXdmf)` directly (bypasses `AtomicFile`/the Algorithm class) across `writeXdmf × {real pipeline, empty pipeline}` (4 cases), own output file. Exercises the shared write utility the Algorithm depends on, not the Algorithm's own guards; validates the `.xdmf` sidecar's heavy-data references via `CheckXdmfFile`. | -| `WriteDREAM3D:FileData Overload` | new | Writes through the exported `DREAM3D::WriteFile(HDF5::FileIO&, const FileData&)` forwarder (via `CreateFileData()`) and round-trips the full fixture — keeps the overload's pipeline/DataStructure argument ordering covered. | +| `WriteDREAM3D:FileData Overload` | new-for-V&V | Writes through the exported `DREAM3D::WriteFile(HDF5::FileIO&, const FileData&)` forwarder (via `CreateFileData()`) and round-trips the full fixture — keeps the overload's pipeline/DataStructure argument ordering covered. | | `WriteDREAM3D:Invalid File` | kept | Same free-function call with an empty path (4 cases via `GENERATE`); asserts the underlying `HDF5::FileIO::WriteFile` failure is surfaced, not `AtomicFile`'s. No sidecar check — a failed write has no target whose sidecar could exist. | -| `WriteDREAM3DFilter:Unwritable DataObject Type` | new for V&V | Full filter execute on a `DataStructure` containing a `GridMontage` (no registered IO factory). Pins the write-failure contract end to end: preflight valid, execute invalid with exactly one error of code `-5`, and no file left at the destination — the only test covering Path 13 and the `AtomicFile`-not-committed guarantee. GridMontage is only the vehicle for reaching that path, not the subject of the test (see Known limitations). | +| `WriteDREAM3DFilter:Unwritable DataObject Type` | new-for-V&V | Full filter execute on a `DataStructure` containing a `GridMontage` (no registered IO factory). Pins the write-failure contract end to end: preflight valid, execute invalid with exactly one error of code `-5`, and no file left at the destination — the only test covering Path 13 and the `AtomicFile`-not-committed guarantee. GridMontage is only the vehicle for reaching that path, not the subject of the test (see the capability-boundary note under Code path coverage). | | `DREAM3DFileTest:DREAM3D File IO Test` | kept | The primary Class 1 content-fidelity test. Builds every geometry/`DataObject` type (`CreateTestDataStructure`), writes + reads back (`writeXdmf ∈ {true,false}`), and asserts full structural/content equality (`CheckTestDataStructure`) plus pipeline round-trip (`pipeline.size()==3`, filter names by index). | | `DREAM3DFileTest::StringArray` | kept | Round-trips a `StringArray` through the actual `WriteDREAM3DFilter`/`ReadDREAM3DFilter` classes (not the free function) — covers Path 10 with real filter execution. | | `DREAM3DFileTest:Import/Export DREAM3D Filter Test` | kept | Executes `WriteDREAM3DFilter` inside a real `Pipeline` (`exportPipeline.execute()`) — the only test exercising Path 9 (non-null `PipelineNode` with a real preceding pipeline). Also checks preflight-imported vs. executed array store types on the read side. | @@ -111,21 +130,13 @@ Source: `src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteDREAM3D | `DREAM3DFileTest: PreflightCache avoids re-reading unchanged files` | kept | Uses `DREAM3D::WriteFile` only to create read-side fixture files; does not exercise `WriteDREAM3DFilter`'s own behavior. Listed for completeness since it shares source-file/tag space. | | `DREAM3DFileTest: Geometry Nested In DataGroup Round Trip` | kept | Executes `WriteDREAM3DFilter` directly across all 8 geometry types, each at top level and nested inside a `DataGroup` (issue #1642 regression coverage), `write_xdmf_file=false`. Covers Paths 7, 10, 14, 16, 19 with per-geometry fixtures. | -## Exemplar archive - -None. Every test above builds its input `DataStructure` inline in C++ and never loads a downloaded `.tar.gz` exemplar — the Class 1 oracle's "expected output" is the test's own hand-built input, so no cached golden file is required or used. (`Small_IN100_dream3d_v3.tar.gz`, referenced elsewhere in this test file, backs unrelated `ReadDREAM3DFilter`-only test cases and is not consumed by any Write-side test.) - -## Known limitations (current simplnx HDF5 IO layer) +**Deliberately out of scope (3 of the file's 23 `TEST_CASE`s).** `DREAM3DFileTest.cpp` is shared between the read and write sides of DREAM3D file IO. These three exercise `ReadDREAM3DFilter` only and belong to its V&V, not this one: `DREAM3DFileTest: Existing Data Objects Test` (importing into a populated `DataStructure`), `DREAM3DFileTest: Path Import Policy Tests` (read-side path-collision policy), and `SimplnxCore::ReadDREAM3DFilter: SIMPL Backwards Compatibility` (read-side SIMPL argument conversion). They are named here rather than silently omitted so the exclusion can be audited. -`HDF5::DataIOManager::addCoreFactories()` (`DataStructure/IO/HDF5/DataIOManager.cpp:33-81`) registers factories per concrete `DataArray` type, per `NeighborList` specialization, per `ScalarData` type, the eight geometries, `AttributeMatrix`, `DataGroup`, and `StringArray`. Two `DataObject` types reachable in the current codebase are **not** registered, and any `DataStructure` containing one cannot be written: `HDF5::DataStructureWriter::WriteFile` hits its "no factory found" guard (`DataStructureWriter.cpp:153-157`) and fails with error `-5` ("Could not find IO factory for datatype: …"), surfacing through `WriteDREAM3DFilter` as Path 13 above. - -1. **`DynamicListArray`** — the generic variable-length per-tuple list container that `NeighborList` is itself built on. It has no IO class at all in `DataStructure/IO/HDF5/`, so nothing could be registered. Only reachable by constructing one directly; no shipped filter creates a bare `DynamicListArray` outside of a `NeighborList` specialization. +**Dual-build verification at sign-off:** the DREAM3D file IO tests pass **32/32 in both** the in-core (`NX-Com-Qt69-Vtk96-Rel`) and out-of-core (`NX-OOC-Qt69-Vtk95-Rel`) Release builds, at the rebased head. The full `SimplnxCore::` suite also passes 979/979 in-core. -2. **`GridMontage`** — unlike the above, this type *does* have an IO class (`DataStructure/IO/HDF5/GridMontageIO.cpp`, with both `readData` and `writeData` bodies), but that class is never passed to `addFactory<>()`, so it is unreachable and the type is unwritable in practice. `GridMontage` is more reachable than `DynamicListArray`: `CreateGridMontageAction` (`src/simplnx/Filter/Actions/CreateGridMontageAction.cpp`) is public core API that any filter may use to create one. No filter in this repository currently does, so no shipped pipeline can hit this today — but a plugin filter using that public Action would produce a `DataStructure` that `WriteDREAM3DFilter` cannot save. - - **This is recorded as an observation, not as a defect with a known fix.** Montage support in SIMPLNX is an open design question that has deliberately not been settled yet, and the existing `GridMontage`/`GridMontageIO`/`CreateGridMontageAction` code predates that decision — it should be treated as an unfinished sketch inherited from the legacy SIMPL montage design, not as a foundation that merely needs switching on. Simply calling `addFactory()` would *not* be a correct fix: `GridMontageIO::writeData` already creates a `groupWriter` it never uses and then passes `parentGroup` to `WriteBaseGroupData`, so registering it as-is would write montage contents into the wrong group. How montages should be represented on disk (and whether `GridMontage` is even the right in-memory abstraction) needs to be designed before any of this code is revived. This V&V cycle's only claim is the one its test makes: **today**, a `DataStructure` containing a `GridMontage` fails to write with error `-5` and leaves no partial file behind. +## Exemplar archive -Both are gaps in the shared HDF5 IO layer, not in `WriteDREAM3DFilter`'s own algorithm — the same gaps affect `ReadDREAM3DFilter` for the same types. Neither is raised as a formal Deviation because there is no confirmed 6.5.171 pipeline behavior being compared against (unlike D2, which names concrete legacy types); they are recorded here as known capability boundaries of what this filter can serialize. The failure mode itself is no longer untested: `"WriteDREAM3DFilter:Unwritable DataObject Type"` pins it via `GridMontage`, asserting the `-5` error and that no partial file is committed. +None. Every test above builds its input `DataStructure` inline in C++ and never loads a downloaded `.tar.gz` exemplar — the Class 1 oracle's "expected output" is the test's own hand-built input, so no cached golden file is required or used. (`Small_IN100_dream3d_v3.tar.gz`, referenced elsewhere in this test file, backs unrelated `ReadDREAM3DFilter`-only test cases and is not consumed by any Write-side test.) ## Deviations from DREAM3D 6.5.171