Skip to content

Commit 7b5a6a7

Browse files
georgeRobertsongeorgeRobertson
committed
fix: add entity into data contract details to deal with duplicate column definitions
1 parent b63c2ac commit 7b5a6a7

7 files changed

Lines changed: 127 additions & 82 deletions

File tree

‎src/dve/core_engine/message.py‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,7 @@ def from_pydantic_error(
239239
record: Record,
240240
error: ValidationError,
241241
error_details: Optional[
242-
dict[FieldName, dict[ErrorCategory, DataContractErrorDetail]]
242+
dict[EntityName, dict[FieldName, dict[ErrorCategory, DataContractErrorDetail]]]
243243
] = None,
244244
) -> Messages:
245245
"""Create messages from a `pydantic` validation error."""
@@ -255,9 +255,13 @@ def from_pydantic_error(
255255

256256
error_field = ".".join([idx for idx in error_dict["loc"] if not isinstance(idx, int)])
257257

258-
error_detail: DataContractErrorDetail = error_details.get( # type: ignore
259-
error_field, DEFAULT_ERROR_DETAIL
260-
).get(category)
258+
error_detail_entity = error_details.get(entity) # type: ignore
259+
if error_detail_entity is None:
260+
error_detail = DEFAULT_ERROR_DETAIL.get(category) # type: ignore
261+
else:
262+
error_detail: DataContractErrorDetail = error_detail_entity.get( # type: ignore
263+
error_field, DEFAULT_ERROR_DETAIL
264+
).get(category)
261265

262266
messages.append(
263267
cls(

‎src/dve/core_engine/validation.py‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ def __init__(
3838
self._model: Optional[BaseModel] = None
3939
self._error_info = error_info or {}
4040
self._error_details: Optional[
41-
dict[FieldName, dict[ErrorCategory, DataContractErrorDetail]]
41+
dict[EntityName, dict[FieldName, dict[ErrorCategory, DataContractErrorDetail]]]
4242
] = None
4343

4444
def __reduce__(self): # Don't attempt to pickle Pydantic models.
@@ -63,17 +63,21 @@ def model(self) -> BaseModel:
6363
return self._model
6464

6565
@property
66-
def error_details(self) -> dict[FieldName, dict[ErrorCategory, DataContractErrorDetail]]:
66+
def error_details(self) -> dict[
67+
EntityName, dict[FieldName, dict[ErrorCategory, DataContractErrorDetail]]
68+
]:
6769
"""Custom error code and message mapping for contract phase"""
6870
if not self._error_details:
69-
_error_details = {
70-
field: {
71-
err_type: DataContractErrorDetail(**detail)
72-
for err_type, detail in err_details.items()
71+
_temp_dict = {}
72+
for entity_name, fields in self._error_info.items():
73+
_temp_dict[entity_name] = {
74+
field: {
75+
err_type: DataContractErrorDetail(**detail)
76+
for err_type, detail in err_details.items()
77+
}
78+
for field, err_details in fields.items()
7379
}
74-
for field, err_details in self._error_info.items()
75-
}
76-
self._error_details = _error_details
80+
self._error_details = _temp_dict
7781
return self._error_details
7882

7983
def __call__(self, record: Record) -> tuple[Optional[Record], Messages]:

‎tests/test_core_engine/test_backends/fixtures.py‎

Lines changed: 29 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -565,25 +565,37 @@ def nested_all_string_parquet_w_errors(temp_dir,
565565
def nested_parquet_custom_dc_err_details(temp_dir):
566566
file_path = Path(temp_dir).joinpath("nested_parquet_data_contract_codes.json")
567567
err_details = {
568-
"id": {
569-
"Blank": {"error_code": "TESTIDBLANK",
570-
"error_message": "id cannot be null",
571-
"reporting_entity": "test_rename"},
572-
"Bad value": {"error_code": "TESTIDBAD",
573-
"error_message": "id is invalid: id - {{id}}",
574-
"reporting_entity": "test_rename"}
568+
"nested_model": {
569+
"id": {
570+
"Blank": {
571+
"error_code": "TESTIDBLANK",
572+
"error_message": "id cannot be null",
573+
"reporting_entity": "test_rename"
575574
},
576-
"datetimefield": {
577-
"Bad value": {"error_code": "TESTDTFIELDBAD",
578-
"error_message": "datetimefield is invalid: id - {{id}}, datetimefield - {{datetimefield}}"}
579-
},
580-
"subfield.id": {
581-
"Blank": {"error_code": "SUBFIELDTESTIDBLANK",
582-
"error_message": "subfield id cannot be null"},
583-
"Bad value": {"error_code": "SUBFIELDTESTIDBAD",
584-
"error_message": "subfield id is invalid: subfield.id - {{__error_value}}"}
575+
"Bad value": {
576+
"error_code": "TESTIDBAD",
577+
"error_message": "id is invalid: id - {{id}}",
578+
"reporting_entity": "test_rename"
579+
}
580+
},
581+
"datetimefield": {
582+
"Bad value": {
583+
"error_code": "TESTDTFIELDBAD",
584+
"error_message": "datetimefield is invalid: id - {{id}}, datetimefield - {{datetimefield}}"
585+
}
586+
},
587+
"subfield.id": {
588+
"Blank": {
589+
"error_code": "SUBFIELDTESTIDBLANK",
590+
"error_message": "subfield id cannot be null"
585591
},
586-
}
592+
"Bad value": {
593+
"error_code": "SUBFIELDTESTIDBAD",
594+
"error_message": "subfield id is invalid: subfield.id - {{__error_value}}"
595+
}
596+
},
597+
}
598+
}
587599
with open(file_path, mode="w") as fle:
588600
json.dump(err_details, fle)
589601

‎tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_data_contract.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -372,7 +372,9 @@ def test_duckdb_data_contract_custom_error_details(nested_all_string_parquet_w_e
372372
reporting_fields={"nested_model": ["id"]},
373373
)
374374

375-
entities, feedback_errors_uri, stage_successful = data_contract.apply_data_contract(get_parent(parquet_uri), entities, {"nested_model": parquet_uri}, dc_meta)
375+
entities, feedback_errors_uri, stage_successful = data_contract.apply_data_contract(
376+
get_parent(parquet_uri), entities, {"nested_model": parquet_uri}, dc_meta
377+
)
376378
assert stage_successful
377379
messages: list[UserMessage] = [msg for msg in load_feedback_messages(feedback_errors_uri)]
378380
assert len(messages) == 2
@@ -381,4 +383,4 @@ def test_duckdb_data_contract_custom_error_details(nested_all_string_parquet_w_e
381383
assert messages[0].ErrorMessage == "subfield id is invalid: subfield.id - WRONG"
382384
assert messages[1].ErrorCode == "TESTIDBAD"
383385
assert messages[1].ErrorMessage == "id is invalid: id - WRONG"
384-
assert messages[1].Entity == "test_rename"
386+
assert messages[1].Entity == "test_rename"

‎tests/test_core_engine/test_message.py‎

Lines changed: 37 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -183,46 +183,56 @@ class TestModel(BaseModel):
183183
unimportant_field: Annotated[int, Field(default=None)]
184184

185185
custom_error_details: str = """
186-
{"idx": {"Blank": {"error_code": "IDBLANKERRCODE",
186+
{"test_entity": {"idx": {"Blank": {"error_code": "IDBLANKERRCODE",
187187
"error_message": "idx is a mandatory field",
188188
"is_informational": true},
189189
"Bad value": {"error_code": "IDDODGYVALCODE",
190190
"error_message": "idx value is dodgy: {{idx}}",
191191
"error_level": "submission"}},
192192
"date_field": {"Bad value": {"error_code": "DATEDODGYVALCODE",
193-
"error_message": "date_field value is dodgy: idx: {{idx}}, date_field: {{date_field}}"}}}
193+
"error_message": "date_field value is dodgy: idx: {{idx}}, date_field: {{date_field}}"}}}
194+
}
194195
"""
195-
error_details: Dict[str, Dict[str, DataContractErrorDetail]] = {field: {err_type: DataContractErrorDetail(**detail)
196-
for err_type, detail in err_details.items()}
197-
for field, err_details in json.loads(custom_error_details).items()}
198-
196+
error_details: Dict[str, Dict[str, Dict[str, DataContractErrorDetail]]] = {
197+
entity_name: {
198+
field: {
199+
err_type: DataContractErrorDetail(**detail)
200+
for err_type, detail in err_details.items()
201+
}
202+
for field, err_details in fields.items()
203+
}
204+
for entity_name, fields in json.loads(custom_error_details).items()
205+
}
206+
199207
_bad_value_data = {"idx": "ABC", "str_field": "test", "date_field": "terry", "unimportant_field": "dog"}
200208
_blank_value_data = {}
201-
209+
202210
try:
203211
TestModel(**_bad_value_data)
204212
except ValidationError as e:
205213
_error_bad_value = e
206-
214+
207215
try:
208216
TestModel(**_blank_value_data)
209217
except ValidationError as e:
210218
_error_blank = e
211-
219+
212220
msgs_bad= FeedbackMessage.from_pydantic_error(entity="test_entity",
213221
record = _bad_value_data,
214222
error=_error_bad_value,
215223
error_details=error_details)
216224

217225
msgs_bad = sorted(msgs_bad, key=lambda x: x.error_location)
226+
227+
entity_error_details = error_details["test_entity"]
218228

219229
assert len(msgs_bad) == 3
220-
assert msgs_bad[0].error_code == error_details.get("date_field").get("Bad value").error_code
221-
assert msgs_bad[0].error_message == error_details.get("date_field").get("Bad value").template_message(_bad_value_data)
230+
assert msgs_bad[0].error_code == entity_error_details.get("date_field").get("Bad value").error_code
231+
assert msgs_bad[0].error_message == entity_error_details.get("date_field").get("Bad value").template_message(_bad_value_data)
222232
assert msgs_bad[0].failure_type == "record"
223233
assert not msgs_bad[0].is_informational
224-
assert msgs_bad[1].error_code == error_details.get("idx").get("Bad value").error_code
225-
assert msgs_bad[1].error_message == error_details.get("idx").get("Bad value").template_message(_bad_value_data)
234+
assert msgs_bad[1].error_code == entity_error_details.get("idx").get("Bad value").error_code
235+
assert msgs_bad[1].error_message == entity_error_details.get("idx").get("Bad value").template_message(_bad_value_data)
226236
assert msgs_bad[1].failure_type == "submission"
227237
assert not msgs_bad[1].is_informational
228238
assert msgs_bad[2].error_code == bad_val_default.error_code
@@ -239,8 +249,8 @@ class TestModel(BaseModel):
239249
msgs_blank = sorted(msgs_blank, key=lambda x: x.error_location)
240250

241251
assert len(msgs_blank) == 2
242-
assert msgs_blank[0].error_code == error_details.get("idx").get("Blank").error_code
243-
assert msgs_blank[0].error_message == error_details.get("idx").get("Blank").template_message(_blank_value_data)
252+
assert msgs_blank[0].error_code == entity_error_details.get("idx").get("Blank").error_code
253+
assert msgs_blank[0].error_message == entity_error_details.get("idx").get("Blank").template_message(_blank_value_data)
244254
assert msgs_blank[0].is_informational
245255
assert msgs_blank[1].error_code == blank_default.error_code
246256
assert msgs_blank[1].error_message == blank_default.error_message
@@ -271,11 +281,18 @@ class TestModel(BaseModel):
271281
}
272282
]
273283
}
274-
custom_error_details: str = """{"sub_field.nested_field_2.test_date": {"Bad value": {"error_code": "DATEDODGYVALCODE",
275-
"error_message": "date_field value is dodgy: a_field: {{a_field}}, date_field: {{__error_value}}"}}}"""
276-
error_details: Dict[str, Dict[str, DataContractErrorDetail]] = {field: {err_type: DataContractErrorDetail(**detail)
277-
for err_type, detail in err_details.items()}
278-
for field, err_details in json.loads(custom_error_details).items()}
284+
custom_error_details: str = """{"test_entity": {"sub_field.nested_field_2.test_date": {"Bad value": {"error_code": "DATEDODGYVALCODE",
285+
"error_message": "date_field value is dodgy: a_field: {{a_field}}, date_field: {{__error_value}}"}}}}"""
286+
error_details: Dict[str, Dict[str, Dict[str, DataContractErrorDetail]]] = {
287+
entity_name: {
288+
field: {
289+
err_type: DataContractErrorDetail(**detail)
290+
for err_type, detail in err_details.items()
291+
}
292+
for field, err_details in fields.items()
293+
}
294+
for entity_name, fields in json.loads(custom_error_details).items()
295+
}
279296
try:
280297
TestModel(**test_record)
281298
except ValidationError as err:

‎tests/testdata/flights/flights_data_contract_error_details.json‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,18 @@
11
{
2-
"country_id": {
3-
"Blank": {
4-
"error_code": "CountryIdIsMissing",
5-
"error_message": "Record Rejected - Country is missing an id"
2+
"country": {
3+
"country_id": {
4+
"Blank": {
5+
"error_code": "CountryIdIsMissing",
6+
"error_message": "Record Rejected - Country is missing an id"
7+
}
68
}
79
},
8-
"airport_id": {
9-
"Blank": {
10-
"error_code": "AirportIdIsMissing",
11-
"error_message": "Record Rejected - Airport is missing an id"
10+
"airport": {
11+
"airport_id": {
12+
"Blank": {
13+
"error_code": "AirportIdIsMissing",
14+
"error_message": "Record Rejected - Airport is missing an id"
15+
}
1216
}
1317
}
1418
}

‎tests/testdata/movies/movies_contract_error_details.json‎

Lines changed: 24 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,29 @@
11
{
2-
"title": {
3-
"Blank": {
4-
"error_code": "BLANKTITLE",
5-
"error_message": "title should not be blank",
6-
"error_level": "submission"
7-
}
8-
},
9-
"year": {
10-
"Blank": {
11-
"error_code": "BLANKYEAR",
12-
"error_message": "year not provided",
13-
"is_informational": true
2+
"movies": {
3+
"title": {
4+
"Blank": {
5+
"error_code": "BLANKTITLE",
6+
"error_message": "title should not be blank",
7+
"error_level": "submission"
8+
}
149
},
15-
"Bad value": {
16-
"error_code": "DODGYYEAR",
17-
"error_message": "year value ({{year}}) is invalid",
18-
"reporting_entity": "movies_rename_test"
19-
}
20-
},
21-
"cast.date_joined": {
22-
"Bad value": {
23-
"error_code": "DODGYDATE",
24-
"error_message": "date_joined value is not valid: {{__error_value}}"
10+
"year": {
11+
"Blank": {
12+
"error_code": "BLANKYEAR",
13+
"error_message": "year not provided",
14+
"is_informational": true
15+
},
16+
"Bad value": {
17+
"error_code": "DODGYYEAR",
18+
"error_message": "year value ({{year}}) is invalid",
19+
"reporting_entity": "movies_rename_test"
20+
}
21+
},
22+
"cast.date_joined": {
23+
"Bad value": {
24+
"error_code": "DODGYDATE",
25+
"error_message": "date_joined value is not valid: {{__error_value}}"
26+
}
2527
}
2628
}
2729
}

0 commit comments

Comments
 (0)