Fix/gr ndsp 125 duplicate keys in data contract details

src/dve/core_engine/message.py

@@ -239,7 +239,7 @@ def from_pydantic_error(
         record: Record,
         error: ValidationError,
         error_details: Optional[
-            dict[FieldName, dict[ErrorCategory, DataContractErrorDetail]]
+            dict[EntityName, dict[FieldName, dict[ErrorCategory, DataContractErrorDetail]]]
         ] = None,
     ) -> Messages:
         """Create messages from a `pydantic` validation error."""
@@ -255,9 +255,13 @@ def from_pydantic_error(
 
             error_field = ".".join([idx for idx in error_dict["loc"] if not isinstance(idx, int)])
 
-            error_detail: DataContractErrorDetail = error_details.get(  # type: ignore
-                error_field, DEFAULT_ERROR_DETAIL
-            ).get(category)
+            error_detail_entity = error_details.get(entity)  # type: ignore
+            if error_detail_entity is None:
+                error_detail = DEFAULT_ERROR_DETAIL.get(category)  # type: ignore
+            else:
+                error_detail: DataContractErrorDetail = error_detail_entity.get(  # type: ignore
+                    error_field, DEFAULT_ERROR_DETAIL
+                ).get(category)
 
             messages.append(
                 cls(

src/dve/core_engine/validation.py

@@ -38,7 +38,7 @@ def __init__(
         self._model: Optional[BaseModel] = None
         self._error_info = error_info or {}
         self._error_details: Optional[
-            dict[FieldName, dict[ErrorCategory, DataContractErrorDetail]]
+            dict[EntityName, dict[FieldName, dict[ErrorCategory, DataContractErrorDetail]]]
         ] = None
 
     def __reduce__(self):  # Don't attempt to pickle Pydantic models.
@@ -63,17 +63,21 @@ def model(self) -> BaseModel:
         return self._model
 
     @property
-    def error_details(self) -> dict[FieldName, dict[ErrorCategory, DataContractErrorDetail]]:
+    def error_details(self) -> dict[
+        EntityName, dict[FieldName, dict[ErrorCategory, DataContractErrorDetail]]
+    ]:
         """Custom error code and message mapping for contract phase"""
         if not self._error_details:
-            _error_details = {
-                field: {
-                    err_type: DataContractErrorDetail(**detail)
-                    for err_type, detail in err_details.items()
+            _temp_dict = {}
+            for entity_name, fields in self._error_info.items():
+                _temp_dict[entity_name] = {
+                    field: {
+                        err_type: DataContractErrorDetail(**detail)
+                        for err_type, detail in err_details.items()
+                    }
+                    for field, err_details in fields.items()
                 }
-                for field, err_details in self._error_info.items()
-            }
-            self._error_details = _error_details
+            self._error_details = _temp_dict
         return self._error_details
 
     def __call__(self, record: Record) -> tuple[Optional[Record], Messages]:

tests/test_core_engine/test_backends/fixtures.py

@@ -565,25 +565,37 @@ def nested_all_string_parquet_w_errors(temp_dir,
 def nested_parquet_custom_dc_err_details(temp_dir):
     file_path = Path(temp_dir).joinpath("nested_parquet_data_contract_codes.json")
     err_details = {
-        "id": {
-            "Blank": {"error_code": "TESTIDBLANK",
-                      "error_message": "id cannot be null",
-                      "reporting_entity": "test_rename"},
-            "Bad value": {"error_code": "TESTIDBAD",
-                          "error_message": "id is invalid: id - {{id}}",
-                          "reporting_entity": "test_rename"}
+        "nested_model": {
+            "id": {
+                "Blank": {
+                    "error_code": "TESTIDBLANK",
+                    "error_message": "id cannot be null",
+                    "reporting_entity": "test_rename"
                 },
-        "datetimefield": {
-            "Bad value": {"error_code": "TESTDTFIELDBAD",
-                          "error_message": "datetimefield is invalid: id - {{id}}, datetimefield - {{datetimefield}}"}
-        },
-        "subfield.id": {
-            "Blank": {"error_code": "SUBFIELDTESTIDBLANK",
-                      "error_message": "subfield id cannot be null"},
-            "Bad value": {"error_code": "SUBFIELDTESTIDBAD",
-                          "error_message": "subfield id is invalid: subfield.id - {{__error_value}}"}
+                "Bad value": {
+                    "error_code": "TESTIDBAD",
+                    "error_message": "id is invalid: id - {{id}}",
+                    "reporting_entity": "test_rename"
+                }
+            },
+            "datetimefield": {
+                "Bad value": {
+                    "error_code": "TESTDTFIELDBAD",
+                    "error_message": "datetimefield is invalid: id - {{id}}, datetimefield - {{datetimefield}}"
+                }
+            },
+            "subfield.id": {
+                "Blank": {
+                    "error_code": "SUBFIELDTESTIDBLANK",
+                    "error_message": "subfield id cannot be null"
                 },
-            }
+                "Bad value": {
+                    "error_code": "SUBFIELDTESTIDBAD",
+                    "error_message": "subfield id is invalid: subfield.id - {{__error_value}}"
+                }
+            },
+        }
+    }
     with open(file_path, mode="w") as fle:
         json.dump(err_details, fle)
 

tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_data_contract.py

@@ -372,7 +372,9 @@ def test_duckdb_data_contract_custom_error_details(nested_all_string_parquet_w_e
         reporting_fields={"nested_model": ["id"]},
     )
 
-    entities, feedback_errors_uri, stage_successful = data_contract.apply_data_contract(get_parent(parquet_uri), entities, {"nested_model": parquet_uri}, dc_meta)
+    entities, feedback_errors_uri, stage_successful = data_contract.apply_data_contract(
+        get_parent(parquet_uri), entities, {"nested_model": parquet_uri}, dc_meta
+    )
     assert stage_successful
     messages: list[UserMessage] = [msg for msg in load_feedback_messages(feedback_errors_uri)]
     assert len(messages) == 2
@@ -381,4 +383,4 @@ def test_duckdb_data_contract_custom_error_details(nested_all_string_parquet_w_e
     assert messages[0].ErrorMessage == "subfield id is invalid: subfield.id - WRONG"
     assert messages[1].ErrorCode == "TESTIDBAD"
     assert messages[1].ErrorMessage == "id is invalid: id - WRONG"
-    assert messages[1].Entity == "test_rename"
+    assert messages[1].Entity == "test_rename"

tests/test_core_engine/test_message.py

@@ -183,46 +183,56 @@ class TestModel(BaseModel):
         unimportant_field: Annotated[int, Field(default=None)]
 
     custom_error_details: str = """
-    {"idx": {"Blank": {"error_code": "IDBLANKERRCODE",
+    {"test_entity": {"idx": {"Blank": {"error_code": "IDBLANKERRCODE",
                       "error_message": "idx is a mandatory field",
                       "is_informational": true},
             "Bad value": {"error_code": "IDDODGYVALCODE",
                           "error_message": "idx value is dodgy: {{idx}}",
                           "error_level": "submission"}},
      "date_field": {"Bad value": {"error_code": "DATEDODGYVALCODE",
-                                  "error_message": "date_field value is dodgy: idx: {{idx}}, date_field: {{date_field}}"}}}    
+                                  "error_message": "date_field value is dodgy: idx: {{idx}}, date_field: {{date_field}}"}}}
+    }
     """
-    error_details: Dict[str, Dict[str, DataContractErrorDetail]] = {field: {err_type: DataContractErrorDetail(**detail) 
-                                                                            for err_type, detail in err_details.items()} 
-                                                                    for field, err_details in json.loads(custom_error_details).items()}
-
+    error_details: Dict[str, Dict[str, Dict[str, DataContractErrorDetail]]] = {
+        entity_name: {
+            field: {
+                err_type: DataContractErrorDetail(**detail)
+                for err_type, detail in err_details.items()
+            }
+            for field, err_details in fields.items()
+        }
+        for entity_name, fields in json.loads(custom_error_details).items()
+    }
+
     _bad_value_data = {"idx": "ABC", "str_field": "test", "date_field": "terry", "unimportant_field": "dog"}
     _blank_value_data = {}
-    
+
     try:
         TestModel(**_bad_value_data)
     except ValidationError as e:
         _error_bad_value = e
-    
+
     try:
         TestModel(**_blank_value_data)
     except ValidationError as e:
         _error_blank = e
-           
+
     msgs_bad= FeedbackMessage.from_pydantic_error(entity="test_entity",
                                                   record = _bad_value_data,
                                                   error=_error_bad_value,
                                                   error_details=error_details)
 
     msgs_bad = sorted(msgs_bad, key=lambda x: x.error_location)
+
+    entity_error_details = error_details["test_entity"]
 
     assert len(msgs_bad) == 3
-    assert msgs_bad[0].error_code == error_details.get("date_field").get("Bad value").error_code
-    assert msgs_bad[0].error_message == error_details.get("date_field").get("Bad value").template_message(_bad_value_data)
+    assert msgs_bad[0].error_code == entity_error_details.get("date_field").get("Bad value").error_code
+    assert msgs_bad[0].error_message == entity_error_details.get("date_field").get("Bad value").template_message(_bad_value_data)
     assert msgs_bad[0].failure_type == "record"
     assert not msgs_bad[0].is_informational
-    assert msgs_bad[1].error_code == error_details.get("idx").get("Bad value").error_code
-    assert msgs_bad[1].error_message == error_details.get("idx").get("Bad value").template_message(_bad_value_data)
+    assert msgs_bad[1].error_code == entity_error_details.get("idx").get("Bad value").error_code
+    assert msgs_bad[1].error_message == entity_error_details.get("idx").get("Bad value").template_message(_bad_value_data)
     assert msgs_bad[1].failure_type == "submission"
     assert not msgs_bad[1].is_informational
     assert msgs_bad[2].error_code == bad_val_default.error_code
@@ -239,8 +249,8 @@ class TestModel(BaseModel):
     msgs_blank = sorted(msgs_blank, key=lambda x: x.error_location)
 
     assert len(msgs_blank) == 2
-    assert msgs_blank[0].error_code == error_details.get("idx").get("Blank").error_code
-    assert msgs_blank[0].error_message == error_details.get("idx").get("Blank").template_message(_blank_value_data)
+    assert msgs_blank[0].error_code == entity_error_details.get("idx").get("Blank").error_code
+    assert msgs_blank[0].error_message == entity_error_details.get("idx").get("Blank").template_message(_blank_value_data)
     assert msgs_blank[0].is_informational
     assert msgs_blank[1].error_code == blank_default.error_code
     assert msgs_blank[1].error_message == blank_default.error_message
@@ -271,11 +281,18 @@ class TestModel(BaseModel):
                                   }
                                  ]
                    }
-    custom_error_details: str = """{"sub_field.nested_field_2.test_date": {"Bad value": {"error_code": "DATEDODGYVALCODE",
-                                  "error_message": "date_field value is dodgy: a_field: {{a_field}}, date_field: {{__error_value}}"}}}"""   
-    error_details: Dict[str, Dict[str, DataContractErrorDetail]] = {field: {err_type: DataContractErrorDetail(**detail) 
-                                                                            for err_type, detail in err_details.items()} 
-                                                                    for field, err_details in json.loads(custom_error_details).items()}
+    custom_error_details: str = """{"test_entity": {"sub_field.nested_field_2.test_date": {"Bad value": {"error_code": "DATEDODGYVALCODE",
+                                  "error_message": "date_field value is dodgy: a_field: {{a_field}}, date_field: {{__error_value}}"}}}}"""   
+    error_details: Dict[str, Dict[str, Dict[str, DataContractErrorDetail]]] = {
+        entity_name: {
+            field: {
+                err_type: DataContractErrorDetail(**detail)
+                for err_type, detail in err_details.items()
+            }
+            for field, err_details in fields.items()
+        }
+        for entity_name, fields in json.loads(custom_error_details).items()
+    }
     try:
         TestModel(**test_record)
     except ValidationError as err:

tests/testdata/flights/flights_data_contract_error_details.json

@@ -1,14 +1,18 @@
 {
-    "country_id": {
-        "Blank": {
-            "error_code": "CountryIdIsMissing",
-            "error_message": "Record Rejected - Country is missing an id"
+    "country": {
+        "country_id": {
+            "Blank": {
+                "error_code": "CountryIdIsMissing",
+                "error_message": "Record Rejected - Country is missing an id"
+            }
         }
     },
-    "airport_id": {
-        "Blank": {
-            "error_code": "AirportIdIsMissing",
-            "error_message": "Record Rejected - Airport is missing an id"
+    "airport": {
+        "airport_id": {
+            "Blank": {
+                "error_code": "AirportIdIsMissing",
+                "error_message": "Record Rejected - Airport is missing an id"
+            }
         }
     }
 }

tests/testdata/movies/movies_contract_error_details.json

@@ -1,27 +1,29 @@
 {
-    "title": {
-        "Blank": {
-            "error_code": "BLANKTITLE",
-            "error_message": "title should not be blank",
-            "error_level": "submission"
-        }
-    },
-    "year": {
-        "Blank": {
-            "error_code": "BLANKYEAR",
-            "error_message": "year not provided",
-            "is_informational": true
+    "movies": {
+        "title": {
+            "Blank": {
+                "error_code": "BLANKTITLE",
+                "error_message": "title should not be blank",
+                "error_level": "submission"
+            }
         },
-        "Bad value": {
-            "error_code": "DODGYYEAR",
-            "error_message": "year value ({{year}}) is invalid",
-            "reporting_entity": "movies_rename_test"
-        }
-    },
-    "cast.date_joined": {
-        "Bad value": {
-            "error_code": "DODGYDATE",
-            "error_message": "date_joined value is not valid: {{__error_value}}"
+        "year": {
+            "Blank": {
+                "error_code": "BLANKYEAR",
+                "error_message": "year not provided",
+                "is_informational": true
+            },
+            "Bad value": {
+                "error_code": "DODGYYEAR",
+                "error_message": "year value ({{year}}) is invalid",
+                "reporting_entity": "movies_rename_test"
+            }
+        },
+        "cast.date_joined": {
+            "Bad value": {
+                "error_code": "DODGYDATE",
+                "error_message": "date_joined value is not valid: {{__error_value}}"
+            }
         }
     }
 }